diff --git a/main/src/main/java/sbt/internal/FlatLoader.java b/main/src/main/java/sbt/internal/FlatLoader.java index e1586d8be..0c0b9682f 100644 --- a/main/src/main/java/sbt/internal/FlatLoader.java +++ b/main/src/main/java/sbt/internal/FlatLoader.java @@ -10,7 +10,7 @@ package sbt.internal; import java.net.URL; import java.net.URLClassLoader; -class FlatLoader extends URLClassLoader { +final class FlatLoader extends URLClassLoader { static { ClassLoader.registerAsParallelCapable(); } diff --git a/main/src/main/java/sbt/internal/LayeredClassLoader.java b/main/src/main/java/sbt/internal/LayeredClassLoader.java index 241f94a05..a66f1d488 100644 --- a/main/src/main/java/sbt/internal/LayeredClassLoader.java +++ b/main/src/main/java/sbt/internal/LayeredClassLoader.java @@ -8,16 +8,14 @@ package sbt.internal; import java.io.File; -import scala.collection.immutable.Map; import scala.collection.Seq; -class LayeredClassLoader extends LayeredClassLoaderImpl { +final class LayeredClassLoader extends LayeredClassLoaderImpl { LayeredClassLoader( final Seq classpath, final ClassLoader parent, - final Map resources, final File tempDir) { - super(classpath, parent, resources, tempDir); + super(classpath, parent, tempDir); } static { diff --git a/main/src/main/java/sbt/internal/ResourceLoader.java b/main/src/main/java/sbt/internal/ResourceLoader.java index b408252d7..9d7993c64 100644 --- a/main/src/main/java/sbt/internal/ResourceLoader.java +++ b/main/src/main/java/sbt/internal/ResourceLoader.java @@ -11,7 +11,7 @@ import java.io.File; import scala.collection.immutable.Map; import scala.collection.Seq; -class ResourceLoader extends ResourceLoaderImpl { +final class ResourceLoader extends ResourceLoaderImpl { ResourceLoader( final Seq classpath, final ClassLoader parent, final Map resources) { super(classpath, parent, resources); diff --git a/main/src/main/java/sbt/internal/ScalaLibraryClassLoader.java b/main/src/main/java/sbt/internal/ScalaLibraryClassLoader.java new file mode 100644 index 000000000..7d9d4f679 --- /dev/null +++ b/main/src/main/java/sbt/internal/ScalaLibraryClassLoader.java @@ -0,0 +1,34 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal; + +import java.net.URL; +import java.net.URLClassLoader; + +final class ScalaLibraryClassLoader extends URLClassLoader { + static { + ClassLoader.registerAsParallelCapable(); + } + + private final URL[] jars; + + ScalaLibraryClassLoader(final URL[] jars, final ClassLoader parent) { + super(jars, parent); + this.jars = jars; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < jars.length; ++ i) { + builder.append(jars[i].toString()); + if (i < jars.length - 2) builder.append(", "); + } + return "ScalaLibraryClassLoader(" + builder + " parent = " + getParent() + ")"; + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f430d0685..e6327c609 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -833,25 +833,25 @@ object Defaults extends BuildCommon { // ((streams in test, loadedTestFrameworks, testLoader, testGrouping in test, testExecution in test, fullClasspath in test, javaHome in test, testForkedParallel, javaOptions in test) flatMap allTestGroupsTask).value, testResultLogger in (Test, test) :== TestResultLogger.SilentWhenNoTests, // https://github.com/sbt/sbt/issues/1185 test := { - val close = testLoader.value match { - case u: URLClassLoader => Some(() => u.close()) - case c: ClasspathFilter => Some(() => c.close()) - case _ => None - } val trl = (testResultLogger in (Test, test)).value val taskName = Project.showContextKey(state.value).show(resolvedScoped.value) - val currentLoader = Thread.currentThread.getContextClassLoader - try { - Thread.currentThread.setContextClassLoader(testLoader.value) - trl.run(streams.value.log, executeTests.value, taskName) - } finally { - Thread.currentThread.setContextClassLoader(currentLoader) - close.foreach(_.apply()) - } + try trl.run(streams.value.log, executeTests.value, taskName) + finally close(testLoader.value) }, - testOnly := inputTests(testOnly).evaluated, - testQuick := inputTests(testQuick).evaluated + testOnly := { + try inputTests(testOnly).evaluated + finally close(testLoader.value) + }, + testQuick := { + try inputTests(testQuick).evaluated + finally close(testLoader.value) + } ) + private def close(sbtLoader: ClassLoader): Unit = sbtLoader match { + case u: AutoCloseable => u.close() + case c: ClasspathFilter => c.close() + case _ => + } /** * A scope whose task axis is set to Zero. @@ -1010,13 +1010,7 @@ object Defaults extends BuildCommon { ) val taskName = display.show(resolvedScoped.value) val trl = testResultLogger.value - val currentLoader = Thread.currentThread.getContextClassLoader - try { - Thread.currentThread.setContextClassLoader(testLoader.value) - output.map(out => trl.run(s.log, out, taskName)) - } finally { - Thread.currentThread.setContextClassLoader(currentLoader) - } + output.map(out => trl.run(s.log, out, taskName)) } } diff --git a/main/src/main/scala/sbt/internal/ClassLoaders.scala b/main/src/main/scala/sbt/internal/ClassLoaders.scala index f4f57275c..1650efc9e 100644 --- a/main/src/main/scala/sbt/internal/ClassLoaders.scala +++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala @@ -80,7 +80,7 @@ private[sbt] object ClassLoaders { val newLoader = (classpath: Seq[File]) => { val mappings = classpath.map(f => f.getName -> f).toMap - val transformedDependencies = allDeps.map(f => mappings.get(f.getName).getOrElse(f)) + val transformedDependencies = allDeps.map(f => mappings.getOrElse(f.getName, f)) buildLayers( strategy = classLoaderLayeringStrategy.value: @sbtUnchecked, si = instance, @@ -133,7 +133,13 @@ private[sbt] object ClassLoaders { case _: AllLibraryJars => true case _ => false } - val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp) + val scalaLibraryLayer = { + cache.apply( + si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList, + interfaceLoader, + () => new ScalaLibraryClassLoader(si.libraryJars.map(_.toURI.toURL), interfaceLoader) + ) + } val cpFiles = fullCP.map(_._1) val scalaReflectJar = allDependencies.collectFirst { @@ -158,8 +164,13 @@ private[sbt] object ClassLoaders { // layer 3 (optional if in the test config and the runtime layer is not shared) val dependencyLayer = - if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp) - else resourceLayer + if (layerDependencies && allDependencies.nonEmpty) { + cache( + allDependencies.toList.map(f => f -> IO.getModifiedTimeOrZero(f)), + resourceLayer, + () => new ReverseLookupClassLoaderHolder(allDependencies, resourceLayer) + ) + } else resourceLayer val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet // layer 4 @@ -167,7 +178,15 @@ private[sbt] object ClassLoaders { if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar else Set(si.libraryJars ++ scalaReflectJar: _*) val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName)) - new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp) + dependencyLayer match { + case dl: ReverseLookupClassLoaderHolder => + dl.checkout(dynamicClasspath, tmp) + case cl => + cl.getParent match { + case dl: ReverseLookupClassLoaderHolder => dl.checkout(dynamicClasspath, tmp) + case _ => new LayeredClassLoader(dynamicClasspath, cl, tmp) + } + } } } @@ -175,27 +194,6 @@ private[sbt] object ClassLoaders { key: sbt.TaskKey[Seq[Attributed[File]]] ): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar"))) - // Creates a one or two layered classloader for the provided classpaths depending on whether - // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer - // above the regular jar layer. This allows the snapshot layer to be invalidated without - // invalidating the regular jar layer. If the classpath is empty, it just returns the parent - // loader. - private def layer( - classpath: Seq[File], - parent: ClassLoader, - cache: ClassLoaderCache, - resources: Map[String, String], - tmp: File - ): ClassLoader = { - if (classpath.nonEmpty) { - cache( - classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)), - parent, - () => new LayeredClassLoader(classpath, parent, resources, tmp) - ) - } else parent - } - // Creates a one or two layered classloader for the provided classpaths depending on whether // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer // above the regular jar layer. This allows the snapshot layer to be invalidated without @@ -209,11 +207,8 @@ private[sbt] object ClassLoaders { resourceMap: Map[String, String] ): ClassLoader = { if (resources.nonEmpty) { - cache( - resources.toList, - parent, - () => new ResourceLoader(classpath, parent, resourceMap) - ) + val mkLoader = () => new ResourceLoader(classpath, parent, resourceMap) + cache(resources.toList, parent, mkLoader) } else parent } diff --git a/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala b/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala index d366dc875..74ec041f8 100644 --- a/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala +++ b/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala @@ -10,119 +10,253 @@ package sbt.internal import java.io.File import java.net.URLClassLoader import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.{ AtomicBoolean, AtomicReference } import sbt.internal.inc.classpath._ import sbt.io.IO import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -private[sbt] class LayeredClassLoaderImpl( +/** + * A simple ClassLoader that copies native libraries to a temporary directory before loading them. + * Otherwise the same as a normal URLClassLoader. + * @param classpath the classpath of the url + * @param parent the parent loader + * @param tempDir the directory into which native libraries are copied before loading + */ +private[internal] class LayeredClassLoaderImpl( classpath: Seq[File], parent: ClassLoader, - override protected val resources: Map[String, String], - tempDir: File, -) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent) - with RawResources - with NativeCopyLoader - with AutoCloseable { - private[this] val nativeLibs = new java.util.HashSet[File]().asScala - override protected val config = new NativeCopyConfig( - tempDir, - classpath, - IO.parseClasspath(System.getProperty("java.library.path", "")) - ) - override def findLibrary(name: String): String = { - super.findLibrary(name) match { - case null => null - case l => - nativeLibs += new File(l) - l + tempDir: File +) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) + with NativeLoader { + setTempDir(tempDir) +} + +/** + * This classloader doesn't load any classes. It is able to create a two layer bundled ClassLoader + * that is able to load the full project classpath. The top layer is able to directly load the + * project dependencies. The bottom layer can load the full classpath of the run or test task. + * If the top layer needs to load a class from the bottom layer via java reflection, we facilitate + * that with the `ReverseLookupClassLoader`. + * + * + * This holder caches the ReverseLookupClassLoader, which is the top loader in this hierarchy. The + * checkout method will get the RevereLookupClassLoader from the cache or make a new one if + * none is available. It will only cache at most one so if multiple concurrently tasks have the + * same dependency classpath, multiple instances of ReverseLookupClassLoader will be created for + * the classpath. If the ReverseLookupClassLoader makes a lookup in the BottomClassLoader, it + * invalidates itself and will not be cached when it is returned. + * + * The reason it is a ClassLoader even though it can't load any classes is so its + * lifecycle -- and therefore the lifecycle of its cache entry -- is managed by the + * ClassLoaderCache, allowing the cache entry to be evicted under memory pressure. + * + * @param classpath the dependency classpath of the managed loader + * @param parent the parent ClassLoader of the managed loader + */ +private[internal] final class ReverseLookupClassLoaderHolder( + val classpath: Seq[File], + val parent: ClassLoader +) extends URLClassLoader(Array.empty, null) { + private[this] val cached: AtomicReference[ReverseLookupClassLoader] = new AtomicReference + private[this] val closed = new AtomicBoolean(false) + + /** + * Get a classloader. If there is a loader available in the cache, it will use that loader, + * otherwise it makes a new classloader. + * + * @return a ClassLoader + */ + def checkout(dependencyClasspath: Seq[File], tempDir: File): ClassLoader = { + if (closed.get()) { + val msg = "Tried to extract class loader from closed ReverseLookupClassLoaderHolder. " + + "Try running the `clearCaches` command and re-trying." + throw new IllegalStateException(msg) + } + val reverseLookupClassLoader = cached.getAndSet(null) match { + case null => new ReverseLookupClassLoader + case c => c + } + reverseLookupClassLoader.setTempDir(tempDir) + new BottomClassLoader(dependencyClasspath, reverseLookupClassLoader, tempDir) + } + + private def checkin(reverseLookupClassLoader: ReverseLookupClassLoader): Unit = { + if (reverseLookupClassLoader.isDirty) reverseLookupClassLoader.close() + else { + if (closed.get()) reverseLookupClassLoader.close() + else + cached.getAndSet(reverseLookupClassLoader) match { + case null => if (closed.get) reverseLookupClassLoader.close() + case c => c.close() + } + } + } + override def close(): Unit = { + closed.set(true) + cached.get() match { + case null => + case c => c.close() } } - private[this] val loaded = new ConcurrentHashMap[String, Class[_]] - private[this] val classLocks = new ConcurrentHashMap[String, AnyRef]() - /* - * Override findClass to both memoize its result and look down the class hierarchy to attempt to - * load a missing class from a descendant loader. If we didn't cache the loaded classes, - * then it would be possible for this class loader to load a different version of the class than - * the descendant, which would likely cause a crash. The search for the class in the descendants - * allows java reflection to work in cases where the class to load via reflection is not directly - * visible to the class that is attempting to load it. + /** + * A ClassLoader for the dependency layer of a run or test task. It is almost a normal + * URLClassLoader except that it has the ability to look one level down the classloading + * hierarchy to load a class via reflection that is not directly available to it. The ClassLoader + * that is below it in the hierarchy must be registered via setDescendant. If it ever loads a + * class from its descendant, then it cannot be used in a subsequent run because it will not be + * possible to reload that class. + * + * The descendant classloader checks it in and out via [[checkout]] and [[checkin]]. When it + * returns the loader via [[checkin]], if the loader is dirty, we close it. Otherwise we + * cache it if there is no existing cache entry. + * + * Performance degrades if loadClass is constantly looking back up to the provided + * BottomClassLoader so we provide an alternate loadClass definition that takes a reverseLookup + * boolean parameter. Because the [[BottomClassLoader]] knows what loader is calling into, when + * it delegates its search to the ReverseLookupClassLoader, it passes false for the reverseLookup + * flag. By default the flag is true. Most of the time, the default loadClass will only be + * invoked by java reflection calls. Even then, there's some chance that the class being loaded + * by java reflection is _not_ available on the bottom classpath so it is not guaranteed that + * performing the reverse lookup will invalidate this loader. + * */ - override def findClass(name: String): Class[_] = loaded.get(name) match { - case null => - val newLock = new AnyRef - val lock = classLocks.putIfAbsent(name, newLock) match { - case null => newLock - case l => l - } - lock.synchronized { - try { - val clazz = super.findClass(name) - loaded.putIfAbsent(name, clazz) match { - case null => clazz - case c => c + private class ReverseLookupClassLoader + extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) + with NativeLoader { + private[this] val directDescendant: AtomicReference[BottomClassLoader] = + new AtomicReference + private[this] val dirty = new AtomicBoolean(false) + def isDirty: Boolean = dirty.get() + def setDescendant(classLoader: BottomClassLoader): Unit = directDescendant.set(classLoader) + def loadClass(name: String, resolve: Boolean, reverseLookup: Boolean): Class[_] = { + try super.loadClass(name, resolve) + catch { + case e: ClassNotFoundException if reverseLookup => + directDescendant.get match { + case null => throw e + case cl => + val res = cl.lookupClass(name) + dirty.set(true) + res } - } catch { - case e: ClassNotFoundException => - /* - * If new threads are spawned, they inherit the context class loader of the parent - * This means that if a test or run task spawns background threads to do work, then the - * same context class loader is available on all of the background threads. In the test - * and run tasks, we temporarily set the context class loader of the main sbt thread to - * be the classloader generated by ClassLoaders.getLayers. This creates an environment - * that looks somewhat like a forked jvm with the app classloader set to be the - * generated class loader. If the test or run main changes the thread context class - * loader, this search might fail even if it would have passed on the initial entry - * into the method. Applications generally only modify the context classloader if they - * are manually loading classes. It's likely that if an application generated - * ClassLoader needs access to the classes in the sbt classpath, then it would be using - * the original context class loader as the parent of the new context class loader - * anyway. - * - * If we wanted to make this change permanent so that the user could not - * override the global context classloader, we would possibly need to intercept the - * classloading of java.lang.Thread itself to return a custom Thread class that mirrors - * the java.lang.Thread api, but stores the context class loader in a custom field. - * - */ - var currentLoader: ClassLoader = Thread.currentThread.getContextClassLoader - val loaders = new ListBuffer[LayeredClassLoader] - do { - currentLoader match { - case cl: LayeredClassLoader if cl != this => loaders.prepend(cl) - case _ => - } - currentLoader = currentLoader.getParent - } while (currentLoader != null && currentLoader != this) - if (currentLoader == this && loaders.nonEmpty) { - val resourceName = name.replace('.', '/').concat(".class") - loaders - .collectFirst { - case l if l.findResource(resourceName) != null => l.findClass(name) - } - .getOrElse(throw e) - } else throw e - } } - case c => c + } + override def loadClass(name: String, resolve: Boolean): Class[_] = + loadClass(name, resolve, reverseLookup = true) + } + + /** + * The bottom most layer in our layering hierarchy. This layer should never be cached. The + * dependency layer may need access to classes only available at this layer using java + * reflection. To make this work, we register this loader with the parent in its + * constructor. We also add the lookupClass method which gives ReverseLookupClassLoader + * a public interface to findClass. + * + * To improve performance, when loading classes from the parent, we call the loadClass + * method with the reverseLookup flag set to false. This prevents the ReverseLookupClassLoader + * from trying to call back into this loader when it can't find a particular class. + * + * @param dynamicClasspath the classpath for the run or test task excluding the dependency jars + * @param parent the ReverseLookupClassLoader with which this loader needs to register itself + * so that reverse lookups required by java reflection will work + * @param tempDir the temp directory to copy native libraries + */ + private class BottomClassLoader( + dynamicClasspath: Seq[File], + parent: ReverseLookupClassLoader, + tempDir: File + ) extends URLClassLoader(dynamicClasspath.map(_.toURI.toURL).toArray, parent) + with NativeLoader { + parent.setDescendant(this) + setTempDir(tempDir) + + final def lookupClass(name: String): Class[_] = findClass(name) + + override def findClass(name: String): Class[_] = findLoadedClass(name) match { + case null => super.findClass(name) + case c => c + } + override def loadClass(name: String, resolve: Boolean): Class[_] = { + val clazz = findLoadedClass(name) match { + case null => + val c = try parent.loadClass(name, resolve = false, reverseLookup = false) + catch { case _: ClassNotFoundException => findClass(name) } + if (resolve) resolveClass(c) + c + case c => c + } + if (resolve) resolveClass(clazz) + clazz + } + override def close(): Unit = { + checkin(parent) + super.close() + } + } +} + +/** + * This is more or less copied from the NativeCopyLoader in zinc. It differs from the zinc + * NativeCopyLoader in that it doesn't allow any explicit mappings and it allows the tempDir + * to be dynamically reset. The explicit mappings feature isn't used by sbt. The dynamic + * temp directory use case is needed in some layered class loading scenarios. + */ +private trait NativeLoader extends ClassLoader with AutoCloseable { + private[this] val mapped = new ConcurrentHashMap[String, String] + private[this] val searchPaths = + sys.props.get("java.library.path").map(IO.parseClasspath).getOrElse(Nil) + private[this] val tempDir = new AtomicReference(new File("/dev/null")) + + abstract override def close(): Unit = { + setTempDir(new File("/dev/null")) + super.close() + } + override protected def findLibrary(name: String): String = synchronized { + mapped.get(name) match { + case null => + findLibrary0(name) match { + case null => null + case n => + mapped.put(name, n) + NativeLibs.addNativeLib(n) + n + } + case n => n + } + } + private[internal] def setTempDir(file: File): Unit = { + deleteNativeLibs() + tempDir.set(file) + } + private[this] def deleteNativeLibs(): Unit = { + mapped.values().forEach(NativeLibs.delete) + mapped.clear() + } + private[this] def findLibrary0(name: String): String = { + val mappedName = System.mapLibraryName(name) + val search = searchPaths.toStream flatMap relativeLibrary(mappedName) + search.headOption.map(copy).orNull + } + private[this] def relativeLibrary(mappedName: String)(base: File): Seq[File] = { + val f = new File(base, mappedName) + if (f.isFile) f :: Nil else Nil + } + private[this] def copy(f: File): String = { + val target = new File(tempDir.get(), f.getName) + IO.copyFile(f, target) + target.getAbsolutePath } - override def close(): Unit = nativeLibs.foreach(NativeLibs.delete) - override def toString: String = s"""LayeredClassLoader( - | classpath = - | ${classpath mkString "\n "} - | parent = - | ${parent.toString.linesIterator.mkString("\n ")} - |)""".stripMargin } private[internal] class ResourceLoaderImpl( classpath: Seq[File], parent: ClassLoader, - resources: Map[String, String] -) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) { + override val resources: Map[String, String] +) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) + with RawResources { override def findClass(name: String): Class[_] = throw new ClassNotFoundException(name) override def loadClass(name: String, resolve: Boolean): Class[_] = { val clazz = parent.loadClass(name) @@ -143,7 +277,8 @@ private[internal] object NativeLibs { nativeLibs.add(new File(lib)) () } - def delete(file: File): Unit = { + def delete(lib: String): Unit = { + val file = new File(lib) nativeLibs.remove(file) file.delete() ()