luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4780 from eatkins/java-reflection-v2 

Java reflection v2
This commit is contained in:
eugene yokota 2019-06-07 07:53:14 +02:00 committed by GitHub
parent efa26d6eb5 286e52793c
commit fd0f078c73
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 309 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
main/src/main/java/sbt/internal/FlatLoader.java
View File

@ -10,7 +10,7 @@ package sbt.internal;
import java.net.URL;
import java.net.URLClassLoader;
class FlatLoader extends URLClassLoader {
final class FlatLoader extends URLClassLoader {
static {
ClassLoader.registerAsParallelCapable();
}

6
main/src/main/java/sbt/internal/LayeredClassLoader.java
View File

@ -8,16 +8,14 @@
package sbt.internal;
import java.io.File;
import scala.collection.immutable.Map;
import scala.collection.Seq;
class LayeredClassLoader extends LayeredClassLoaderImpl {
final class LayeredClassLoader extends LayeredClassLoaderImpl {
LayeredClassLoader(
final Seq<File> classpath,
final ClassLoader parent,
final Map<String, String> resources,
final File tempDir) {
super(classpath, parent, resources, tempDir);
super(classpath, parent, tempDir);
}
static {

2
main/src/main/java/sbt/internal/ResourceLoader.java
View File

@ -11,7 +11,7 @@ import java.io.File;
import scala.collection.immutable.Map;
import scala.collection.Seq;
class ResourceLoader extends ResourceLoaderImpl {
final class ResourceLoader extends ResourceLoaderImpl {
ResourceLoader(
final Seq<File> classpath, final ClassLoader parent, final Map<String, String> resources) {
super(classpath, parent, resources);

34
main/src/main/java/sbt/internal/ScalaLibraryClassLoader.java Normal file
View File

@ -0,0 +1,34 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.net.URL;
import java.net.URLClassLoader;
final class ScalaLibraryClassLoader extends URLClassLoader {
static {
ClassLoader.registerAsParallelCapable();
}
private final URL[] jars;
ScalaLibraryClassLoader(final URL[] jars, final ClassLoader parent) {
super(jars, parent);
this.jars = jars;
}
@Override
public String toString() {
final StringBuilder builder = new StringBuilder();
for (int i = 0; i < jars.length; ++ i) {
builder.append(jars[i].toString());
if (i < jars.length - 2) builder.append(", ");
}
return "ScalaLibraryClassLoader(" + builder + " parent = " + getParent() + ")";
}
}

38
main/src/main/scala/sbt/Defaults.scala
View File

@ -833,25 +833,25 @@ object Defaults extends BuildCommon {
// ((streams in test, loadedTestFrameworks, testLoader, testGrouping in test, testExecution in test, fullClasspath in test, javaHome in test, testForkedParallel, javaOptions in test) flatMap allTestGroupsTask).value,
testResultLogger in (Test, test) :== TestResultLogger.SilentWhenNoTests, // https://github.com/sbt/sbt/issues/1185
test := {
val close = testLoader.value match {
case u: URLClassLoader => Some(() => u.close())
case c: ClasspathFilter => Some(() => c.close())
case _ => None
}
val trl = (testResultLogger in (Test, test)).value
val taskName = Project.showContextKey(state.value).show(resolvedScoped.value)
val currentLoader = Thread.currentThread.getContextClassLoader
try {
Thread.currentThread.setContextClassLoader(testLoader.value)
trl.run(streams.value.log, executeTests.value, taskName)
} finally {
Thread.currentThread.setContextClassLoader(currentLoader)
close.foreach(_.apply())
}
try trl.run(streams.value.log, executeTests.value, taskName)
finally close(testLoader.value)
},
testOnly := inputTests(testOnly).evaluated,
testQuick := inputTests(testQuick).evaluated
testOnly := {
try inputTests(testOnly).evaluated
finally close(testLoader.value)
},
testQuick := {
try inputTests(testQuick).evaluated
finally close(testLoader.value)
}
)
private def close(sbtLoader: ClassLoader): Unit = sbtLoader match {
case u: AutoCloseable => u.close()
case c: ClasspathFilter => c.close()
case _ =>
}
/**
* A scope whose task axis is set to Zero.
@ -1010,13 +1010,7 @@ object Defaults extends BuildCommon {
)
val taskName = display.show(resolvedScoped.value)
val trl = testResultLogger.value
val currentLoader = Thread.currentThread.getContextClassLoader
try {
Thread.currentThread.setContextClassLoader(testLoader.value)
output.map(out => trl.run(s.log, out, taskName))
} finally {
Thread.currentThread.setContextClassLoader(currentLoader)
}
output.map(out => trl.run(s.log, out, taskName))
}
}

57
main/src/main/scala/sbt/internal/ClassLoaders.scala
View File

@ -80,7 +80,7 @@ private[sbt] object ClassLoaders {
val newLoader =
(classpath: Seq[File]) => {
val mappings = classpath.map(f => f.getName -> f).toMap
val transformedDependencies = allDeps.map(f => mappings.get(f.getName).getOrElse(f))
val transformedDependencies = allDeps.map(f => mappings.getOrElse(f.getName, f))
buildLayers(
strategy = classLoaderLayeringStrategy.value: @sbtUnchecked,
si = instance,
@ -133,7 +133,13 @@ private[sbt] object ClassLoaders {
case _: AllLibraryJars => true
case _ => false
}
val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp)
val scalaLibraryLayer = {
cache.apply(
si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
interfaceLoader,
() => new ScalaLibraryClassLoader(si.libraryJars.map(_.toURI.toURL), interfaceLoader)
)
}
val cpFiles = fullCP.map(_._1)
val scalaReflectJar = allDependencies.collectFirst {
@ -158,8 +164,13 @@ private[sbt] object ClassLoaders {
// layer 3 (optional if in the test config and the runtime layer is not shared)
val dependencyLayer =
if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp)
else resourceLayer
if (layerDependencies && allDependencies.nonEmpty) {
cache(
allDependencies.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
resourceLayer,
() => new ReverseLookupClassLoaderHolder(allDependencies, resourceLayer)
)
} else resourceLayer
val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet
// layer 4
@ -167,7 +178,15 @@ private[sbt] object ClassLoaders {
if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar
else Set(si.libraryJars ++ scalaReflectJar: _*)
val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName))
new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp)
dependencyLayer match {
case dl: ReverseLookupClassLoaderHolder =>
dl.checkout(dynamicClasspath, tmp)
case cl =>
cl.getParent match {
case dl: ReverseLookupClassLoaderHolder => dl.checkout(dynamicClasspath, tmp)
case _ => new LayeredClassLoader(dynamicClasspath, cl, tmp)
}
}
}
}
@ -175,27 +194,6 @@ private[sbt] object ClassLoaders {
key: sbt.TaskKey[Seq[Attributed[File]]]
): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
// invalidating the regular jar layer. If the classpath is empty, it just returns the parent
// loader.
private def layer(
classpath: Seq[File],
parent: ClassLoader,
cache: ClassLoaderCache,
resources: Map[String, String],
tmp: File
): ClassLoader = {
if (classpath.nonEmpty) {
cache(
classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
parent,
() => new LayeredClassLoader(classpath, parent, resources, tmp)
)
} else parent
}
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
@ -209,11 +207,8 @@ private[sbt] object ClassLoaders {
resourceMap: Map[String, String]
): ClassLoader = {
if (resources.nonEmpty) {
cache(
resources.toList,
parent,
() => new ResourceLoader(classpath, parent, resourceMap)
)
val mkLoader = () => new ResourceLoader(classpath, parent, resourceMap)
cache(resources.toList, parent, mkLoader)
} else parent
}

323
main/src/main/scala/sbt/internal/LayeredClassLoaders.scala
View File

@ -10,119 +10,253 @@ package sbt.internal
import java.io.File
import java.net.URLClassLoader
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.{ AtomicBoolean, AtomicReference }
import sbt.internal.inc.classpath._
import sbt.io.IO
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
private[sbt] class LayeredClassLoaderImpl(
/**
* A simple ClassLoader that copies native libraries to a temporary directory before loading them.
* Otherwise the same as a normal URLClassLoader.
* @param classpath the classpath of the url
* @param parent the parent loader
* @param tempDir the directory into which native libraries are copied before loading
*/
private[internal] class LayeredClassLoaderImpl(
classpath: Seq[File],
parent: ClassLoader,
override protected val resources: Map[String, String],
tempDir: File,
) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent)
with RawResources
with NativeCopyLoader
with AutoCloseable {
private[this] val nativeLibs = new java.util.HashSet[File]().asScala
override protected val config = new NativeCopyConfig(
tempDir,
classpath,
IO.parseClasspath(System.getProperty("java.library.path", ""))
)
override def findLibrary(name: String): String = {
super.findLibrary(name) match {
case null => null
case l =>
nativeLibs += new File(l)
l
tempDir: File
) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
with NativeLoader {
setTempDir(tempDir)
}
/**
* This classloader doesn't load any classes. It is able to create a two layer bundled ClassLoader
* that is able to load the full project classpath. The top layer is able to directly load the
* project dependencies. The bottom layer can load the full classpath of the run or test task.
* If the top layer needs to load a class from the bottom layer via java reflection, we facilitate
* that with the `ReverseLookupClassLoader`.
*
*
* This holder caches the ReverseLookupClassLoader, which is the top loader in this hierarchy. The
* checkout method will get the RevereLookupClassLoader from the cache or make a new one if
* none is available. It will only cache at most one so if multiple concurrently tasks have the
* same dependency classpath, multiple instances of ReverseLookupClassLoader will be created for
* the classpath. If the ReverseLookupClassLoader makes a lookup in the BottomClassLoader, it
* invalidates itself and will not be cached when it is returned.
*
* The reason it is a ClassLoader even though it can't load any classes is so its
* lifecycle -- and therefore the lifecycle of its cache entry -- is managed by the
* ClassLoaderCache, allowing the cache entry to be evicted under memory pressure.
*
* @param classpath the dependency classpath of the managed loader
* @param parent the parent ClassLoader of the managed loader
*/
private[internal] final class ReverseLookupClassLoaderHolder(
val classpath: Seq[File],
val parent: ClassLoader
) extends URLClassLoader(Array.empty, null) {
private[this] val cached: AtomicReference[ReverseLookupClassLoader] = new AtomicReference
private[this] val closed = new AtomicBoolean(false)
/**
* Get a classloader. If there is a loader available in the cache, it will use that loader,
* otherwise it makes a new classloader.
*
* @return a ClassLoader
*/
def checkout(dependencyClasspath: Seq[File], tempDir: File): ClassLoader = {
if (closed.get()) {
val msg = "Tried to extract class loader from closed ReverseLookupClassLoaderHolder. " +
"Try running the `clearCaches` command and re-trying."
throw new IllegalStateException(msg)
}
val reverseLookupClassLoader = cached.getAndSet(null) match {
case null => new ReverseLookupClassLoader
case c => c
}
reverseLookupClassLoader.setTempDir(tempDir)
new BottomClassLoader(dependencyClasspath, reverseLookupClassLoader, tempDir)
}
private def checkin(reverseLookupClassLoader: ReverseLookupClassLoader): Unit = {
if (reverseLookupClassLoader.isDirty) reverseLookupClassLoader.close()
else {
if (closed.get()) reverseLookupClassLoader.close()
else
cached.getAndSet(reverseLookupClassLoader) match {
case null => if (closed.get) reverseLookupClassLoader.close()
case c => c.close()
}
}
}
override def close(): Unit = {
closed.set(true)
cached.get() match {
case null =>
case c => c.close()
}
}
private[this] val loaded = new ConcurrentHashMap[String, Class[_]]
private[this] val classLocks = new ConcurrentHashMap[String, AnyRef]()
/*
* Override findClass to both memoize its result and look down the class hierarchy to attempt to
* load a missing class from a descendant loader. If we didn't cache the loaded classes,
* then it would be possible for this class loader to load a different version of the class than
* the descendant, which would likely cause a crash. The search for the class in the descendants
* allows java reflection to work in cases where the class to load via reflection is not directly
* visible to the class that is attempting to load it.
/**
* A ClassLoader for the dependency layer of a run or test task. It is almost a normal
* URLClassLoader except that it has the ability to look one level down the classloading
* hierarchy to load a class via reflection that is not directly available to it. The ClassLoader
* that is below it in the hierarchy must be registered via setDescendant. If it ever loads a
* class from its descendant, then it cannot be used in a subsequent run because it will not be
* possible to reload that class.
*
* The descendant classloader checks it in and out via [[checkout]] and [[checkin]]. When it
* returns the loader via [[checkin]], if the loader is dirty, we close it. Otherwise we
* cache it if there is no existing cache entry.
*
* Performance degrades if loadClass is constantly looking back up to the provided
* BottomClassLoader so we provide an alternate loadClass definition that takes a reverseLookup
* boolean parameter. Because the [[BottomClassLoader]] knows what loader is calling into, when
* it delegates its search to the ReverseLookupClassLoader, it passes false for the reverseLookup
* flag. By default the flag is true. Most of the time, the default loadClass will only be
* invoked by java reflection calls. Even then, there's some chance that the class being loaded
* by java reflection is _not_ available on the bottom classpath so it is not guaranteed that
* performing the reverse lookup will invalidate this loader.
*
*/
override def findClass(name: String): Class[_] = loaded.get(name) match {
case null =>
val newLock = new AnyRef
val lock = classLocks.putIfAbsent(name, newLock) match {
case null => newLock
case l => l
}
lock.synchronized {
try {
val clazz = super.findClass(name)
loaded.putIfAbsent(name, clazz) match {
case null => clazz
case c => c
private class ReverseLookupClassLoader
extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
with NativeLoader {
private[this] val directDescendant: AtomicReference[BottomClassLoader] =
new AtomicReference
private[this] val dirty = new AtomicBoolean(false)
def isDirty: Boolean = dirty.get()
def setDescendant(classLoader: BottomClassLoader): Unit = directDescendant.set(classLoader)
def loadClass(name: String, resolve: Boolean, reverseLookup: Boolean): Class[_] = {
try super.loadClass(name, resolve)
catch {
case e: ClassNotFoundException if reverseLookup =>
directDescendant.get match {
case null => throw e
case cl =>
val res = cl.lookupClass(name)
dirty.set(true)
res
}
} catch {
case e: ClassNotFoundException =>
/*
* If new threads are spawned, they inherit the context class loader of the parent
* This means that if a test or run task spawns background threads to do work, then the
* same context class loader is available on all of the background threads. In the test
* and run tasks, we temporarily set the context class loader of the main sbt thread to
* be the classloader generated by ClassLoaders.getLayers. This creates an environment
* that looks somewhat like a forked jvm with the app classloader set to be the
* generated class loader. If the test or run main changes the thread context class
* loader, this search might fail even if it would have passed on the initial entry
* into the method. Applications generally only modify the context classloader if they
* are manually loading classes. It's likely that if an application generated
* ClassLoader needs access to the classes in the sbt classpath, then it would be using
* the original context class loader as the parent of the new context class loader
* anyway.
*
* If we wanted to make this change permanent so that the user could not
* override the global context classloader, we would possibly need to intercept the
* classloading of java.lang.Thread itself to return a custom Thread class that mirrors
* the java.lang.Thread api, but stores the context class loader in a custom field.
*
*/
var currentLoader: ClassLoader = Thread.currentThread.getContextClassLoader
val loaders = new ListBuffer[LayeredClassLoader]
do {
currentLoader match {
case cl: LayeredClassLoader if cl != this => loaders.prepend(cl)
case _ =>
}
currentLoader = currentLoader.getParent
} while (currentLoader != null && currentLoader != this)
if (currentLoader == this && loaders.nonEmpty) {
val resourceName = name.replace('.', '/').concat(".class")
loaders
.collectFirst {
case l if l.findResource(resourceName) != null => l.findClass(name)
}
.getOrElse(throw e)
} else throw e
}
}
case c => c
}
override def loadClass(name: String, resolve: Boolean): Class[_] =
loadClass(name, resolve, reverseLookup = true)
}
/**
* The bottom most layer in our layering hierarchy. This layer should never be cached. The
* dependency layer may need access to classes only available at this layer using java
* reflection. To make this work, we register this loader with the parent in its
* constructor. We also add the lookupClass method which gives ReverseLookupClassLoader
* a public interface to findClass.
*
* To improve performance, when loading classes from the parent, we call the loadClass
* method with the reverseLookup flag set to false. This prevents the ReverseLookupClassLoader
* from trying to call back into this loader when it can't find a particular class.
*
* @param dynamicClasspath the classpath for the run or test task excluding the dependency jars
* @param parent the ReverseLookupClassLoader with which this loader needs to register itself
* so that reverse lookups required by java reflection will work
* @param tempDir the temp directory to copy native libraries
*/
private class BottomClassLoader(
dynamicClasspath: Seq[File],
parent: ReverseLookupClassLoader,
tempDir: File
) extends URLClassLoader(dynamicClasspath.map(_.toURI.toURL).toArray, parent)
with NativeLoader {
parent.setDescendant(this)
setTempDir(tempDir)
final def lookupClass(name: String): Class[_] = findClass(name)
override def findClass(name: String): Class[_] = findLoadedClass(name) match {
case null => super.findClass(name)
case c => c
}
override def loadClass(name: String, resolve: Boolean): Class[_] = {
val clazz = findLoadedClass(name) match {
case null =>
val c = try parent.loadClass(name, resolve = false, reverseLookup = false)
catch { case _: ClassNotFoundException => findClass(name) }
if (resolve) resolveClass(c)
c
case c => c
}
if (resolve) resolveClass(clazz)
clazz
}
override def close(): Unit = {
checkin(parent)
super.close()
}
}
}
/**
* This is more or less copied from the NativeCopyLoader in zinc. It differs from the zinc
* NativeCopyLoader in that it doesn't allow any explicit mappings and it allows the tempDir
* to be dynamically reset. The explicit mappings feature isn't used by sbt. The dynamic
* temp directory use case is needed in some layered class loading scenarios.
*/
private trait NativeLoader extends ClassLoader with AutoCloseable {
private[this] val mapped = new ConcurrentHashMap[String, String]
private[this] val searchPaths =
sys.props.get("java.library.path").map(IO.parseClasspath).getOrElse(Nil)
private[this] val tempDir = new AtomicReference(new File("/dev/null"))
abstract override def close(): Unit = {
setTempDir(new File("/dev/null"))
super.close()
}
override protected def findLibrary(name: String): String = synchronized {
mapped.get(name) match {
case null =>
findLibrary0(name) match {
case null => null
case n =>
mapped.put(name, n)
NativeLibs.addNativeLib(n)
n
}
case n => n
}
}
private[internal] def setTempDir(file: File): Unit = {
deleteNativeLibs()
tempDir.set(file)
}
private[this] def deleteNativeLibs(): Unit = {
mapped.values().forEach(NativeLibs.delete)
mapped.clear()
}
private[this] def findLibrary0(name: String): String = {
val mappedName = System.mapLibraryName(name)
val search = searchPaths.toStream flatMap relativeLibrary(mappedName)
search.headOption.map(copy).orNull
}
private[this] def relativeLibrary(mappedName: String)(base: File): Seq[File] = {
val f = new File(base, mappedName)
if (f.isFile) f :: Nil else Nil
}
private[this] def copy(f: File): String = {
val target = new File(tempDir.get(), f.getName)
IO.copyFile(f, target)
target.getAbsolutePath
}
override def close(): Unit = nativeLibs.foreach(NativeLibs.delete)
override def toString: String = s"""LayeredClassLoader(
| classpath =
| ${classpath mkString "\n "}
| parent =
| ${parent.toString.linesIterator.mkString("\n ")}
|)""".stripMargin
}
private[internal] class ResourceLoaderImpl(
classpath: Seq[File],
parent: ClassLoader,
resources: Map[String, String]
) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
override val resources: Map[String, String]
) extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
with RawResources {
override def findClass(name: String): Class[_] = throw new ClassNotFoundException(name)
override def loadClass(name: String, resolve: Boolean): Class[_] = {
val clazz = parent.loadClass(name)
@ -143,7 +277,8 @@ private[internal] object NativeLibs {
nativeLibs.add(new File(lib))
()
}
def delete(file: File): Unit = {
def delete(lib: String): Unit = {
val file = new File(lib)
nativeLibs.remove(file)
file.delete()
()