mirror of https://github.com/sbt/sbt.git
Overhaul dependency layer for java reflection
My first attempt, cc8c66c66d, at making
java reflection work with a layered classloader with a dependency jar
layer was a failure. It would generally work ok the first time the user
ran test, but was likely to fail on a second run.
There were a number of problems with the strategy:
1) It relied on the thread's context class loader to determine where to
attempt the reverse lookup.
2) It is not possible to ever reload classes loaded by a classloader.
Consider the classloading hierarchy a <- b, where
the arrow implies that a is the parent of b. I incorrectly thought
that a's loadClass method would be called every time a class loaded
by a made a call to Class.forName(name: String). This turns out to
not be the case. As a result, the second time the dependency layer
was used, where now the hierarchy is a <- c, that same Class.forName
call could return a Class from b which causes a nasty crash.
It isn't possible to work around the limitation in 2 so the only option
that allows both caching and java reflection across layers to work is to
cache the dependency layer, but invalidate when cross layer reflection
occurs. This turns out to be straightforward to implement. The
performance looks very similar to the ScalaLibrary strategy when java
reflection is used, which makes sense because the scala library and
scala reflect layers are still reused when the dependency layer is
invalidated.
I also stopped passing around the resource map to all of the layers.
Resource loading is hierarchical and the resource layer comes above the
dependency layer in the stack so there was no need for the bottom layers
to also be RawResource loaders.
This commit is contained in:
Ethan Atkins
parent
76f3bb271e
commit
286e52793c
|
|
@ -8,16 +8,14 @@
|
|||
package sbt.internal;
|
||||
|
||||
import java.io.File;
|
||||
import scala.collection.immutable.Map;
|
||||
import scala.collection.Seq;
|
||||
|
||||
final class LayeredClassLoader extends LayeredClassLoaderImpl {
|
||||
LayeredClassLoader(
|
||||
final Seq<File> classpath,
|
||||
final ClassLoader parent,
|
||||
final Map<String, String> resources,
|
||||
final File tempDir) {
|
||||
super(classpath, parent, resources, tempDir);
|
||||
super(classpath, parent, tempDir);
|
||||
}
|
||||
|
||||
static {
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ private[sbt] object ClassLoaders {
|
|||
val newLoader =
|
||||
(classpath: Seq[File]) => {
|
||||
val mappings = classpath.map(f => f.getName -> f).toMap
|
||||
val transformedDependencies = allDeps.map(f => mappings.get(f.getName).getOrElse(f))
|
||||
val transformedDependencies = allDeps.map(f => mappings.getOrElse(f.getName, f))
|
||||
buildLayers(
|
||||
strategy = classLoaderLayeringStrategy.value: @sbtUnchecked,
|
||||
si = instance,
|
||||
|
|
@ -164,8 +164,13 @@ private[sbt] object ClassLoaders {
|
|||
|
||||
// layer 3 (optional if in the test config and the runtime layer is not shared)
|
||||
val dependencyLayer =
|
||||
if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp)
|
||||
else resourceLayer
|
||||
if (layerDependencies && allDependencies.nonEmpty) {
|
||||
cache(
|
||||
allDependencies.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
|
||||
resourceLayer,
|
||||
() => new ReverseLookupClassLoaderHolder(allDependencies, resourceLayer)
|
||||
)
|
||||
} else resourceLayer
|
||||
|
||||
val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet
|
||||
// layer 4
|
||||
|
|
@ -173,7 +178,15 @@ private[sbt] object ClassLoaders {
|
|||
if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar
|
||||
else Set(si.libraryJars ++ scalaReflectJar: _*)
|
||||
val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName))
|
||||
new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp)
|
||||
dependencyLayer match {
|
||||
case dl: ReverseLookupClassLoaderHolder =>
|
||||
dl.checkout(dynamicClasspath, tmp)
|
||||
case cl =>
|
||||
cl.getParent match {
|
||||
case dl: ReverseLookupClassLoaderHolder => dl.checkout(dynamicClasspath, tmp)
|
||||
case _ => new LayeredClassLoader(dynamicClasspath, cl, tmp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -181,27 +194,6 @@ private[sbt] object ClassLoaders {
|
|||
key: sbt.TaskKey[Seq[Attributed[File]]]
|
||||
): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))
|
||||
|
||||
// Creates a one or two layered classloader for the provided classpaths depending on whether
|
||||
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
|
||||
// above the regular jar layer. This allows the snapshot layer to be invalidated without
|
||||
// invalidating the regular jar layer. If the classpath is empty, it just returns the parent
|
||||
// loader.
|
||||
private def layer(
|
||||
classpath: Seq[File],
|
||||
parent: ClassLoader,
|
||||
cache: ClassLoaderCache,
|
||||
resources: Map[String, String],
|
||||
tmp: File
|
||||
): ClassLoader = {
|
||||
if (classpath.nonEmpty) {
|
||||
cache(
|
||||
classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
|
||||
parent,
|
||||
() => new LayeredClassLoader(classpath, parent, resources, tmp)
|
||||
)
|
||||
} else parent
|
||||
}
|
||||
|
||||
// Creates a one or two layered classloader for the provided classpaths depending on whether
|
||||
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
|
||||
// above the regular jar layer. This allows the snapshot layer to be invalidated without
|
||||
|
|
@ -215,11 +207,8 @@ private[sbt] object ClassLoaders {
|
|||
resourceMap: Map[String, String]
|
||||
): ClassLoader = {
|
||||
if (resources.nonEmpty) {
|
||||
cache(
|
||||
resources.toList,
|
||||
parent,
|
||||
() => new ResourceLoader(classpath, parent, resourceMap)
|
||||
)
|
||||
val mkLoader = () => new ResourceLoader(classpath, parent, resourceMap)
|
||||
cache(resources.toList, parent, mkLoader)
|
||||
} else parent
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,119 +10,253 @@ package sbt.internal
|
|||
import java.io.File
|
||||
import java.net.URLClassLoader
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.{ AtomicBoolean, AtomicReference }
|
||||
|
||||
import sbt.internal.inc.classpath._
|
||||
import sbt.io.IO
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable.ListBuffer
|
||||
|
||||
private[sbt] class LayeredClassLoaderImpl(
|
||||
/**
|
||||
* A simple ClassLoader that copies native libraries to a temporary directory before loading them.
|
||||
* Otherwise the same as a normal URLClassLoader.
|
||||
* @param classpath the classpath of the url
|
||||
* @param parent the parent loader
|
||||
* @param tempDir the directory into which native libraries are copied before loading
|
||||
*/
|
||||
private[internal] class LayeredClassLoaderImpl(
|
||||
classpath: Seq[File],
|
||||
parent: ClassLoader,
|
||||
override protected val resources: Map[String, String],
|
||||
tempDir: File,
|
||||
) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent)
|
||||
with RawResources
|
||||
with NativeCopyLoader
|
||||
with AutoCloseable {
|
||||
private[this] val nativeLibs = new java.util.HashSet[File]().asScala
|
||||
override protected val config = new NativeCopyConfig(
|
||||
tempDir,
|
||||
classpath,
|
||||
IO.parseClasspath(System.getProperty("java.library.path", ""))
|
||||
)
|
||||
override def findLibrary(name: String): String = {
|
||||
super.findLibrary(name) match {
|
||||
case null => null
|
||||
case l =>
|
||||
nativeLibs += new File(l)
|
||||
l
|
||||
tempDir: File
|
||||
) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
|
||||
with NativeLoader {
|
||||
setTempDir(tempDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* This classloader doesn't load any classes. It is able to create a two layer bundled ClassLoader
|
||||
* that is able to load the full project classpath. The top layer is able to directly load the
|
||||
* project dependencies. The bottom layer can load the full classpath of the run or test task.
|
||||
* If the top layer needs to load a class from the bottom layer via java reflection, we facilitate
|
||||
* that with the `ReverseLookupClassLoader`.
|
||||
*
|
||||
*
|
||||
* This holder caches the ReverseLookupClassLoader, which is the top loader in this hierarchy. The
|
||||
* checkout method will get the RevereLookupClassLoader from the cache or make a new one if
|
||||
* none is available. It will only cache at most one so if multiple concurrently tasks have the
|
||||
* same dependency classpath, multiple instances of ReverseLookupClassLoader will be created for
|
||||
* the classpath. If the ReverseLookupClassLoader makes a lookup in the BottomClassLoader, it
|
||||
* invalidates itself and will not be cached when it is returned.
|
||||
*
|
||||
* The reason it is a ClassLoader even though it can't load any classes is so its
|
||||
* lifecycle -- and therefore the lifecycle of its cache entry -- is managed by the
|
||||
* ClassLoaderCache, allowing the cache entry to be evicted under memory pressure.
|
||||
*
|
||||
* @param classpath the dependency classpath of the managed loader
|
||||
* @param parent the parent ClassLoader of the managed loader
|
||||
*/
|
||||
private[internal] final class ReverseLookupClassLoaderHolder(
|
||||
val classpath: Seq[File],
|
||||
val parent: ClassLoader
|
||||
) extends URLClassLoader(Array.empty, null) {
|
||||
private[this] val cached: AtomicReference[ReverseLookupClassLoader] = new AtomicReference
|
||||
private[this] val closed = new AtomicBoolean(false)
|
||||
|
||||
/**
|
||||
* Get a classloader. If there is a loader available in the cache, it will use that loader,
|
||||
* otherwise it makes a new classloader.
|
||||
*
|
||||
* @return a ClassLoader
|
||||
*/
|
||||
def checkout(dependencyClasspath: Seq[File], tempDir: File): ClassLoader = {
|
||||
if (closed.get()) {
|
||||
val msg = "Tried to extract class loader from closed ReverseLookupClassLoaderHolder. " +
|
||||
"Try running the `clearCaches` command and re-trying."
|
||||
throw new IllegalStateException(msg)
|
||||
}
|
||||
val reverseLookupClassLoader = cached.getAndSet(null) match {
|
||||
case null => new ReverseLookupClassLoader
|
||||
case c => c
|
||||
}
|
||||
reverseLookupClassLoader.setTempDir(tempDir)
|
||||
new BottomClassLoader(dependencyClasspath, reverseLookupClassLoader, tempDir)
|
||||
}
|
||||
|
||||
private def checkin(reverseLookupClassLoader: ReverseLookupClassLoader): Unit = {
|
||||
if (reverseLookupClassLoader.isDirty) reverseLookupClassLoader.close()
|
||||
else {
|
||||
if (closed.get()) reverseLookupClassLoader.close()
|
||||
else
|
||||
cached.getAndSet(reverseLookupClassLoader) match {
|
||||
case null => if (closed.get) reverseLookupClassLoader.close()
|
||||
case c => c.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
override def close(): Unit = {
|
||||
closed.set(true)
|
||||
cached.get() match {
|
||||
case null =>
|
||||
case c => c.close()
|
||||
}
|
||||
}
|
||||
|
||||
private[this] val loaded = new ConcurrentHashMap[String, Class[_]]
|
||||
private[this] val classLocks = new ConcurrentHashMap[String, AnyRef]()
|
||||
/*
|
||||
* Override findClass to both memoize its result and look down the class hierarchy to attempt to
|
||||
* load a missing class from a descendant loader. If we didn't cache the loaded classes,
|
||||
* then it would be possible for this class loader to load a different version of the class than
|
||||
* the descendant, which would likely cause a crash. The search for the class in the descendants
|
||||
* allows java reflection to work in cases where the class to load via reflection is not directly
|
||||
* visible to the class that is attempting to load it.
|
||||
/**
|
||||
* A ClassLoader for the dependency layer of a run or test task. It is almost a normal
|
||||
* URLClassLoader except that it has the ability to look one level down the classloading
|
||||
* hierarchy to load a class via reflection that is not directly available to it. The ClassLoader
|
||||
* that is below it in the hierarchy must be registered via setDescendant. If it ever loads a
|
||||
* class from its descendant, then it cannot be used in a subsequent run because it will not be
|
||||
* possible to reload that class.
|
||||
*
|
||||
* The descendant classloader checks it in and out via [[checkout]] and [[checkin]]. When it
|
||||
* returns the loader via [[checkin]], if the loader is dirty, we close it. Otherwise we
|
||||
* cache it if there is no existing cache entry.
|
||||
*
|
||||
* Performance degrades if loadClass is constantly looking back up to the provided
|
||||
* BottomClassLoader so we provide an alternate loadClass definition that takes a reverseLookup
|
||||
* boolean parameter. Because the [[BottomClassLoader]] knows what loader is calling into, when
|
||||
* it delegates its search to the ReverseLookupClassLoader, it passes false for the reverseLookup
|
||||
* flag. By default the flag is true. Most of the time, the default loadClass will only be
|
||||
* invoked by java reflection calls. Even then, there's some chance that the class being loaded
|
||||
* by java reflection is _not_ available on the bottom classpath so it is not guaranteed that
|
||||
* performing the reverse lookup will invalidate this loader.
|
||||
*
|
||||
*/
|
||||
override def findClass(name: String): Class[_] = loaded.get(name) match {
|
||||
case null =>
|
||||
val newLock = new AnyRef
|
||||
val lock = classLocks.putIfAbsent(name, newLock) match {
|
||||
case null => newLock
|
||||
case l => l
|
||||
}
|
||||
lock.synchronized {
|
||||
try {
|
||||
val clazz = super.findClass(name)
|
||||
loaded.putIfAbsent(name, clazz) match {
|
||||
case null => clazz
|
||||
case c => c
|
||||
private class ReverseLookupClassLoader
|
||||
extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
|
||||
with NativeLoader {
|
||||
private[this] val directDescendant: AtomicReference[BottomClassLoader] =
|
||||
new AtomicReference
|
||||
private[this] val dirty = new AtomicBoolean(false)
|
||||
def isDirty: Boolean = dirty.get()
|
||||
def setDescendant(classLoader: BottomClassLoader): Unit = directDescendant.set(classLoader)
|
||||
def loadClass(name: String, resolve: Boolean, reverseLookup: Boolean): Class[_] = {
|
||||
try super.loadClass(name, resolve)
|
||||
catch {
|
||||
case e: ClassNotFoundException if reverseLookup =>
|
||||
directDescendant.get match {
|
||||
case null => throw e
|
||||
case cl =>
|
||||
val res = cl.lookupClass(name)
|
||||
dirty.set(true)
|
||||
res
|
||||
}
|
||||
} catch {
|
||||
case e: ClassNotFoundException =>
|
||||
/*
|
||||
* If new threads are spawned, they inherit the context class loader of the parent
|
||||
* This means that if a test or run task spawns background threads to do work, then the
|
||||
* same context class loader is available on all of the background threads. In the test
|
||||
* and run tasks, we temporarily set the context class loader of the main sbt thread to
|
||||
* be the classloader generated by ClassLoaders.getLayers. This creates an environment
|
||||
* that looks somewhat like a forked jvm with the app classloader set to be the
|
||||
* generated class loader. If the test or run main changes the thread context class
|
||||
* loader, this search might fail even if it would have passed on the initial entry
|
||||
* into the method. Applications generally only modify the context classloader if they
|
||||
* are manually loading classes. It's likely that if an application generated
|
||||
* ClassLoader needs access to the classes in the sbt classpath, then it would be using
|
||||
* the original context class loader as the parent of the new context class loader
|
||||
* anyway.
|
||||
*
|
||||
* If we wanted to make this change permanent so that the user could not
|
||||
* override the global context classloader, we would possibly need to intercept the
|
||||
* classloading of java.lang.Thread itself to return a custom Thread class that mirrors
|
||||
* the java.lang.Thread api, but stores the context class loader in a custom field.
|
||||
*
|
||||
*/
|
||||
var currentLoader: ClassLoader = Thread.currentThread.getContextClassLoader
|
||||
val loaders = new ListBuffer[LayeredClassLoader]
|
||||
do {
|
||||
currentLoader match {
|
||||
case cl: LayeredClassLoader if cl != this => loaders.prepend(cl)
|
||||
case _ =>
|
||||
}
|
||||
currentLoader = currentLoader.getParent
|
||||
} while (currentLoader != null && currentLoader != this)
|
||||
if (currentLoader == this && loaders.nonEmpty) {
|
||||
val resourceName = name.replace('.', '/').concat(".class")
|
||||
loaders
|
||||
.collectFirst {
|
||||
case l if l.findResource(resourceName) != null => l.findClass(name)
|
||||
}
|
||||
.getOrElse(throw e)
|
||||
} else throw e
|
||||
}
|
||||
}
|
||||
case c => c
|
||||
}
|
||||
override def loadClass(name: String, resolve: Boolean): Class[_] =
|
||||
loadClass(name, resolve, reverseLookup = true)
|
||||
}
|
||||
|
||||
/**
|
||||
* The bottom most layer in our layering hierarchy. This layer should never be cached. The
|
||||
* dependency layer may need access to classes only available at this layer using java
|
||||
* reflection. To make this work, we register this loader with the parent in its
|
||||
* constructor. We also add the lookupClass method which gives ReverseLookupClassLoader
|
||||
* a public interface to findClass.
|
||||
*
|
||||
* To improve performance, when loading classes from the parent, we call the loadClass
|
||||
* method with the reverseLookup flag set to false. This prevents the ReverseLookupClassLoader
|
||||
* from trying to call back into this loader when it can't find a particular class.
|
||||
*
|
||||
* @param dynamicClasspath the classpath for the run or test task excluding the dependency jars
|
||||
* @param parent the ReverseLookupClassLoader with which this loader needs to register itself
|
||||
* so that reverse lookups required by java reflection will work
|
||||
* @param tempDir the temp directory to copy native libraries
|
||||
*/
|
||||
private class BottomClassLoader(
|
||||
dynamicClasspath: Seq[File],
|
||||
parent: ReverseLookupClassLoader,
|
||||
tempDir: File
|
||||
) extends URLClassLoader(dynamicClasspath.map(_.toURI.toURL).toArray, parent)
|
||||
with NativeLoader {
|
||||
parent.setDescendant(this)
|
||||
setTempDir(tempDir)
|
||||
|
||||
final def lookupClass(name: String): Class[_] = findClass(name)
|
||||
|
||||
override def findClass(name: String): Class[_] = findLoadedClass(name) match {
|
||||
case null => super.findClass(name)
|
||||
case c => c
|
||||
}
|
||||
override def loadClass(name: String, resolve: Boolean): Class[_] = {
|
||||
val clazz = findLoadedClass(name) match {
|
||||
case null =>
|
||||
val c = try parent.loadClass(name, resolve = false, reverseLookup = false)
|
||||
catch { case _: ClassNotFoundException => findClass(name) }
|
||||
if (resolve) resolveClass(c)
|
||||
c
|
||||
case c => c
|
||||
}
|
||||
if (resolve) resolveClass(clazz)
|
||||
clazz
|
||||
}
|
||||
override def close(): Unit = {
|
||||
checkin(parent)
|
||||
super.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is more or less copied from the NativeCopyLoader in zinc. It differs from the zinc
|
||||
* NativeCopyLoader in that it doesn't allow any explicit mappings and it allows the tempDir
|
||||
* to be dynamically reset. The explicit mappings feature isn't used by sbt. The dynamic
|
||||
* temp directory use case is needed in some layered class loading scenarios.
|
||||
*/
|
||||
private trait NativeLoader extends ClassLoader with AutoCloseable {
|
||||
private[this] val mapped = new ConcurrentHashMap[String, String]
|
||||
private[this] val searchPaths =
|
||||
sys.props.get("java.library.path").map(IO.parseClasspath).getOrElse(Nil)
|
||||
private[this] val tempDir = new AtomicReference(new File("/dev/null"))
|
||||
|
||||
abstract override def close(): Unit = {
|
||||
setTempDir(new File("/dev/null"))
|
||||
super.close()
|
||||
}
|
||||
override protected def findLibrary(name: String): String = synchronized {
|
||||
mapped.get(name) match {
|
||||
case null =>
|
||||
findLibrary0(name) match {
|
||||
case null => null
|
||||
case n =>
|
||||
mapped.put(name, n)
|
||||
NativeLibs.addNativeLib(n)
|
||||
n
|
||||
}
|
||||
case n => n
|
||||
}
|
||||
}
|
||||
private[internal] def setTempDir(file: File): Unit = {
|
||||
deleteNativeLibs()
|
||||
tempDir.set(file)
|
||||
}
|
||||
private[this] def deleteNativeLibs(): Unit = {
|
||||
mapped.values().forEach(NativeLibs.delete)
|
||||
mapped.clear()
|
||||
}
|
||||
private[this] def findLibrary0(name: String): String = {
|
||||
val mappedName = System.mapLibraryName(name)
|
||||
val search = searchPaths.toStream flatMap relativeLibrary(mappedName)
|
||||
search.headOption.map(copy).orNull
|
||||
}
|
||||
private[this] def relativeLibrary(mappedName: String)(base: File): Seq[File] = {
|
||||
val f = new File(base, mappedName)
|
||||
if (f.isFile) f :: Nil else Nil
|
||||
}
|
||||
private[this] def copy(f: File): String = {
|
||||
val target = new File(tempDir.get(), f.getName)
|
||||
IO.copyFile(f, target)
|
||||
target.getAbsolutePath
|
||||
}
|
||||
override def close(): Unit = nativeLibs.foreach(NativeLibs.delete)
|
||||
override def toString: String = s"""LayeredClassLoader(
|
||||
| classpath =
|
||||
| ${classpath mkString "\n "}
|
||||
| parent =
|
||||
| ${parent.toString.linesIterator.mkString("\n ")}
|
||||
|)""".stripMargin
|
||||
}
|
||||
|
||||
private[internal] class ResourceLoaderImpl(
|
||||
classpath: Seq[File],
|
||||
parent: ClassLoader,
|
||||
resources: Map[String, String]
|
||||
) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
|
||||
override val resources: Map[String, String]
|
||||
) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
|
||||
with RawResources {
|
||||
override def findClass(name: String): Class[_] = throw new ClassNotFoundException(name)
|
||||
override def loadClass(name: String, resolve: Boolean): Class[_] = {
|
||||
val clazz = parent.loadClass(name)
|
||||
|
|
@ -143,7 +277,8 @@ private[internal] object NativeLibs {
|
|||
nativeLibs.add(new File(lib))
|
||||
()
|
||||
}
|
||||
def delete(file: File): Unit = {
|
||||
def delete(lib: String): Unit = {
|
||||
val file = new File(lib)
|
||||
nativeLibs.remove(file)
|
||||
file.delete()
|
||||
()
|
||||
|
|
|
|||
Loading…
Reference in New Issue