luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4769 from eatkins/java-reflection 

Java reflection
This commit is contained in:
eugene yokota 2019-06-03 22:35:53 -04:00 committed by GitHub
parent dc20eb2d4a 1b8f0ed20f
commit ff94258116
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 421 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sbt
View File

@ -28,6 +28,8 @@ def buildLevelSettings: Seq[Setting[_]] =
bintrayPackage := "sbt",
bintrayReleaseOnPublish := false,
licenses := List("Apache-2.0" -> url("https://github.com/sbt/sbt/blob/0.13/LICENSE")),
javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
Compile / doc / javacOptions := Nil,
developers := List(
Developer("harrah", "Mark Harrah", "@harrah", url("https://github.com/harrah")),
Developer("eed3si9n", "Eugene Yokota", "@eed3si9n", url("https://github.com/eed3si9n")),

34
main-command/src/main/java/sbt/internal/classpath/WrappedLoader.java Normal file
View File

@ -0,0 +1,34 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal.classpath;
import java.net.URL;
import java.net.URLClassLoader;
public class WrappedLoader extends URLClassLoader {
static {
ClassLoader.registerAsParallelCapable();
}
WrappedLoader(final ClassLoader parent) {
super(new URL[] {}, parent);
}
@Override
public URL[] getURLs() {
final ClassLoader parent = getParent();
return (parent instanceof URLClassLoader)
? ((URLClassLoader) parent).getURLs()
: super.getURLs();
}
@Override
public String toString() {
return "WrappedClassLoader(" + getParent() + ")";
}
}

10
main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala
View File

@ -10,7 +10,7 @@ package sbt.internal.classpath
import java.io.File
import java.lang.management.ManagementFactory
import java.lang.ref.{ Reference, ReferenceQueue, SoftReference }
import java.net.{ URL, URLClassLoader }
import java.net.URLClassLoader
import java.util.concurrent.atomic.AtomicInteger
import sbt.internal.inc.classpath.{
@ -142,14 +142,6 @@ private[sbt] class ClassLoaderCache(
private[this] val cleanupThread = new CleanupThread(ClassLoaderCache.threadID.getAndIncrement())
private[this] val lock = new Object
private class WrappedLoader(parent: ClassLoader) extends URLClassLoader(Array.empty, parent) {
// This is to make dotty work which extracts the URLs from the loader
override def getURLs: Array[URL] = parent match {
case u: URLClassLoader => u.getURLs
case _ => Array.empty
}
override def toString: String = s"WrappedLoader($parent)"
}
private def close(classLoader: ClassLoader): Unit = classLoader match {
case a: AutoCloseable => a.close()
case _ =>

32
main/src/main/java/sbt/internal/FlatLoader.java Normal file
View File

@ -0,0 +1,32 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.net.URL;
import java.net.URLClassLoader;
class FlatLoader extends URLClassLoader {
static {
ClassLoader.registerAsParallelCapable();
}
FlatLoader(final URL[] urls, final ClassLoader parent) {
super(urls, parent);
}
@Override
public String toString() {
final StringBuilder jars = new StringBuilder();
for (final URL u : getURLs()) {
jars.append(" ");
jars.append(u);
jars.append("\n");
}
return "FlatLoader(\n parent = " + getParent() + "\n jars = " + jars.toString() + ")";
}
}

26
main/src/main/java/sbt/internal/LayeredClassLoader.java Normal file
View File

@ -0,0 +1,26 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.io.File;
import scala.collection.immutable.Map;
import scala.collection.Seq;
class LayeredClassLoader extends LayeredClassLoaderImpl {
LayeredClassLoader(
final Seq<File> classpath,
final ClassLoader parent,
final Map<String, String> resources,
final File tempDir) {
super(classpath, parent, resources, tempDir);
}
static {
ClassLoader.registerAsParallelCapable();
}
}

23
main/src/main/java/sbt/internal/ResourceLoader.java Normal file
View File

@ -0,0 +1,23 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.io.File;
import scala.collection.immutable.Map;
import scala.collection.Seq;
class ResourceLoader extends ResourceLoaderImpl {
ResourceLoader(
final Seq<File> classpath, final ClassLoader parent, final Map<String, String> resources) {
super(classpath, parent, resources);
}
static {
ClassLoader.registerAsParallelCapable();
}
}

29
main/src/main/java/sbt/internal/ScalaReflectClassLoader.java Normal file
View File

@ -0,0 +1,29 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.net.URL;
import java.net.URLClassLoader;
final class ScalaReflectClassLoader extends URLClassLoader {
static {
ClassLoader.registerAsParallelCapable();
}
private final URL jar;
ScalaReflectClassLoader(final URL jar, final ClassLoader parent) {
super(new URL[] {jar}, parent);
this.jar = jar;
}
@Override
public String toString() {
return "ScalaReflectClassLoader(" + jar + " parent = " + getParent() + ")";
}
}

12
main/src/main/scala/sbt/Defaults.scala
View File

@ -855,9 +855,12 @@ object Defaults extends BuildCommon {
}
val trl = (testResultLogger in (Test, test)).value
val taskName = Project.showContextKey(state.value).show(resolvedScoped.value)
val currentLoader = Thread.currentThread.getContextClassLoader
try {
Thread.currentThread.setContextClassLoader(testLoader.value)
trl.run(streams.value.log, executeTests.value, taskName)
} finally {
Thread.currentThread.setContextClassLoader(currentLoader)
close.foreach(_.apply())
}
},
@ -1022,8 +1025,13 @@ object Defaults extends BuildCommon {
)
val taskName = display.show(resolvedScoped.value)
val trl = testResultLogger.value
val processed = output.map(out => trl.run(s.log, out, taskName))
processed
val currentLoader = Thread.currentThread.getContextClassLoader
try {
Thread.currentThread.setContextClassLoader(testLoader.value)
output.map(out => trl.run(s.log, out, taskName))
} finally {
Thread.currentThread.setContextClassLoader(currentLoader)
}
}
}

43
main/src/main/scala/sbt/internal/ClassLoaders.scala
View File

@ -9,7 +9,6 @@ package sbt
package internal
import java.io.File
import java.net.URLClassLoader
import java.nio.file.Path
import sbt.ClassLoaderLayeringStrategy._
@ -76,16 +75,18 @@ private[sbt] object ClassLoaders {
)
s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
}
val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars
val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.libraryJars
val allDeps = dependencyJars(dependencyClasspath).value.filterNot(exclude)
val newLoader =
(classpath: Seq[File]) => {
val mappings = classpath.map(f => f.getName -> f).toMap
val transformedDependencies = allDeps.map(f => mappings.get(f.getName).getOrElse(f))
buildLayers(
strategy = classLoaderLayeringStrategy.value: @sbtUnchecked,
si = instance,
fullCP = classpath.map(f => f -> IO.getModifiedTimeOrZero(f)),
resourceCP = resourceCP,
allDependencies = allDeps,
allDependencies = transformedDependencies,
cache = extendedClassLoaderCache.value: @sbtUnchecked,
resources = ClasspathUtilities.createClasspathResources(classpath, instance),
tmp = taskTemporaryDirectory.value: @sbtUnchecked,
@ -125,7 +126,7 @@ private[sbt] object ClassLoaders {
scope: Scope
): ClassLoader = {
val cpFiles = fullCP.map(_._1)
val raw = strategy match {
strategy match {
case Flat => flatLoader(cpFiles, interfaceLoader)
case _ =>
val layerDependencies = strategy match {
@ -135,10 +136,17 @@ private[sbt] object ClassLoaders {
val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp)
val cpFiles = fullCP.map(_._1)
val scalaReflectJar = allDependencies.find(_.getName == "scala-reflect.jar")
val scalaReflectJar = allDependencies.collectFirst {
case f if f.getName == "scala-reflect.jar" =>
si.allJars.find(_.getName == "scala-reflect.jar")
}.flatten
val scalaReflectLayer = scalaReflectJar
.map { file =>
layer(file :: Nil, scalaLibraryLayer, cache, resources, tmp)
cache.apply(
file -> IO.getModifiedTimeOrZero(file) :: Nil,
scalaLibraryLayer,
() => new ScalaReflectClassLoader(file.toURI.toURL, scalaLibraryLayer)
)
}
.getOrElse(scalaLibraryLayer)
@ -153,14 +161,14 @@ private[sbt] object ClassLoaders {
if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp)
else resourceLayer
val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet
// layer 4
val filteredSet =
if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar
else Set(si.libraryJars ++ scalaReflectJar: _*)
val dynamicClasspath = cpFiles.filterNot(filteredSet)
val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName))
new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp)
}
ClasspathUtilities.filterByClasspath(cpFiles, raw)
}
private def dependencyJars(
@ -188,18 +196,6 @@ private[sbt] object ClassLoaders {
} else parent
}
private class ResourceLoader(
classpath: Seq[File],
parent: ClassLoader,
resources: Map[String, String]
) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
override def loadClass(name: String, resolve: Boolean): Class[_] = {
val clazz = parent.loadClass(name)
if (resolve) resolveClass(clazz)
clazz
}
override def toString: String = "ResourceLoader"
}
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
@ -221,14 +217,9 @@ private[sbt] object ClassLoaders {
} else parent
}
private[this] class FlatLoader(classpath: Seq[File], parent: ClassLoader)
extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) {
override def toString: String =
s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)"
}
// helper methods
private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader =
new FlatLoader(classpath, parent)
new FlatLoader(classpath.map(_.toURI.toURL).toArray, parent)
private[this] def modifiedTimes(stamps: Seq[(Path, FileStamp)]): Seq[(File, Long)] = stamps.map {
case (p, LastModified(lm)) => p.toFile -> lm
case (p, _) =>

66
main/src/main/scala/sbt/internal/LayeredClassLoader.scala
View File

@ -1,66 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.io.File
import java.net.URLClassLoader
import java.{ util => jutil }
import scala.collection.JavaConverters._
import sbt.internal.inc.classpath._
import sbt.io.IO
private[sbt] class LayeredClassLoader(
classpath: Seq[File],
parent: ClassLoader,
override protected val resources: Map[String, String],
tempDir: File,
) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent)
with RawResources
with NativeCopyLoader
with AutoCloseable {
private[this] val nativeLibs = new jutil.HashSet[File]().asScala
override protected val config = new NativeCopyConfig(
tempDir,
classpath,
IO.parseClasspath(System.getProperty("java.library.path", ""))
)
override def findLibrary(name: String): String = {
super.findLibrary(name) match {
case null => null
case l =>
nativeLibs += new File(l)
l
}
}
override def close(): Unit = nativeLibs.foreach(NativeLibs.delete)
override def toString: String = s"""LayeredClassLoader(
| classpath =
| ${classpath mkString "\n "}
| parent =
| ${parent.toString.linesIterator.mkString("\n ")}
|)""".stripMargin
}
private[internal] object NativeLibs {
private[this] val nativeLibs = new jutil.HashSet[File].asScala
ShutdownHooks.add(() => {
nativeLibs.foreach(IO.delete)
IO.deleteIfEmpty(nativeLibs.map(_.getParentFile).toSet)
nativeLibs.clear()
})
def addNativeLib(lib: String): Unit = {
nativeLibs.add(new File(lib))
()
}
def delete(file: File): Unit = {
nativeLibs.remove(file)
file.delete()
()
}
}

151
main/src/main/scala/sbt/internal/LayeredClassLoaders.scala Normal file
View File

@ -0,0 +1,151 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.io.File
import java.net.URLClassLoader
import java.util.concurrent.ConcurrentHashMap
import sbt.internal.inc.classpath._
import sbt.io.IO
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
private[sbt] class LayeredClassLoaderImpl(
classpath: Seq[File],
parent: ClassLoader,
override protected val resources: Map[String, String],
tempDir: File,
) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent)
with RawResources
with NativeCopyLoader
with AutoCloseable {
private[this] val nativeLibs = new java.util.HashSet[File]().asScala
override protected val config = new NativeCopyConfig(
tempDir,
classpath,
IO.parseClasspath(System.getProperty("java.library.path", ""))
)
override def findLibrary(name: String): String = {
super.findLibrary(name) match {
case null => null
case l =>
nativeLibs += new File(l)
l
}
}
private[this] val loaded = new ConcurrentHashMap[String, Class[_]]
private[this] val classLocks = new ConcurrentHashMap[String, AnyRef]()
/*
* Override findClass to both memoize its result and look down the class hierarchy to attempt to
* load a missing class from a descendant loader. If we didn't cache the loaded classes,
* then it would be possible for this class loader to load a different version of the class than
* the descendant, which would likely cause a crash. The search for the class in the descendants
* allows java reflection to work in cases where the class to load via reflection is not directly
* visible to the class that is attempting to load it.
*/
override def findClass(name: String): Class[_] = loaded.get(name) match {
case null =>
val newLock = new AnyRef
val lock = classLocks.putIfAbsent(name, newLock) match {
case null => newLock
case l => l
}
lock.synchronized {
try {
val clazz = super.findClass(name)
loaded.putIfAbsent(name, clazz) match {
case null => clazz
case c => c
}
} catch {
case e: ClassNotFoundException =>
/*
* If new threads are spawned, they inherit the context class loader of the parent
* This means that if a test or run task spawns background threads to do work, then the
* same context class loader is available on all of the background threads. In the test
* and run tasks, we temporarily set the context class loader of the main sbt thread to
* be the classloader generated by ClassLoaders.getLayers. This creates an environment
* that looks somewhat like a forked jvm with the app classloader set to be the
* generated class loader. If the test or run main changes the thread context class
* loader, this search might fail even if it would have passed on the initial entry
* into the method. Applications generally only modify the context classloader if they
* are manually loading classes. It's likely that if an application generated
* ClassLoader needs access to the classes in the sbt classpath, then it would be using
* the original context class loader as the parent of the new context class loader
* anyway.
*
* If we wanted to make this change permanent so that the user could not
* override the global context classloader, we would possibly need to intercept the
* classloading of java.lang.Thread itself to return a custom Thread class that mirrors
* the java.lang.Thread api, but stores the context class loader in a custom field.
*
*/
var currentLoader: ClassLoader = Thread.currentThread.getContextClassLoader
val loaders = new ListBuffer[LayeredClassLoader]
do {
currentLoader match {
case cl: LayeredClassLoader if cl != this => loaders.prepend(cl)
case _ =>
}
currentLoader = currentLoader.getParent
} while (currentLoader != null && currentLoader != this)
if (currentLoader == this && loaders.nonEmpty) {
val resourceName = name.replace('.', '/').concat(".class")
loaders
.collectFirst {
case l if l.findResource(resourceName) != null => l.findClass(name)
}
.getOrElse(throw e)
} else throw e
}
}
case c => c
}
override def close(): Unit = nativeLibs.foreach(NativeLibs.delete)
override def toString: String = s"""LayeredClassLoader(
| classpath =
| ${classpath mkString "\n "}
| parent =
| ${parent.toString.linesIterator.mkString("\n ")}
|)""".stripMargin
}
private[internal] class ResourceLoaderImpl(
classpath: Seq[File],
parent: ClassLoader,
resources: Map[String, String]
) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
override def findClass(name: String): Class[_] = throw new ClassNotFoundException(name)
override def loadClass(name: String, resolve: Boolean): Class[_] = {
val clazz = parent.loadClass(name)
if (resolve) resolveClass(clazz)
clazz
}
override def toString: String = "ResourceLoader"
}
private[internal] object NativeLibs {
private[this] val nativeLibs = new java.util.HashSet[File].asScala
ShutdownHooks.add(() => {
nativeLibs.foreach(IO.delete)
IO.deleteIfEmpty(nativeLibs.map(_.getParentFile).toSet)
nativeLibs.clear()
})
def addNativeLib(lib: String): Unit = {
nativeLibs.add(new File(lib))
()
}
def delete(file: File): Unit = {
nativeLibs.remove(file)
file.delete()
()
}
}

4
sbt/src/sbt-test/classloader-cache/java-serialization/build.sbt Normal file
View File

@ -0,0 +1,4 @@
val dependency = project.settings(exportJars := true)
val descendant = project.dependsOn(dependency).settings(
libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.5" % "test"
)

17
sbt/src/sbt-test/classloader-cache/java-serialization/dependency/src/main/scala/reflection/Reflection.scala Normal file
View File

@ -0,0 +1,17 @@
package reflection
import java.io._
import scala.util.control.NonFatal
object Reflection {
def roundTrip[A](a: A): A = {
val baos = new ByteArrayOutputStream()
val oos = new ObjectOutputStream(baos)
oos.writeObject(a)
oos.close()
val bais = new ByteArrayInputStream(baos.toByteArray())
val ois = new ObjectInputStream(bais)
try ois.readObject().asInstanceOf[A]
finally ois.close()
}
}

12
sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/Foo.scala Normal file
View File

@ -0,0 +1,12 @@
package test
class Foo extends Serializable {
private[this] var value: Int = 0
def getValue(): Int = value
def setValue(newValue: Int): Unit = value = newValue
override def equals(o: Any): Boolean = o match {
case that: Foo => this.getValue() == that.getValue()
case _ => false
}
override def hashCode: Int = value
}

12
sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/ReflectionTest.scala Normal file
View File

@ -0,0 +1,12 @@
package test
import org.scalatest._
class ReflectionTest extends FlatSpec {
val foo = new Foo
foo.setValue(3)
val newFoo = reflection.Reflection.roundTrip(foo)
assert(newFoo == foo)
assert(System.identityHashCode(newFoo) != System.identityHashCode(foo))
}

12
sbt/src/sbt-test/classloader-cache/java-serialization/test Normal file
View File

@ -0,0 +1,12 @@
> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
> test
> testOnly
> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary
> test
> testOnly

7
sbt/src/sbt-test/classloader-cache/spark/build.sbt Normal file
View File

@ -0,0 +1,7 @@
name := "Simple Project"
version := "1.0"
scalaVersion := "2.12.8"
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.3"

7
sbt/src/sbt-test/classloader-cache/spark/log.txt Normal file
View File

@ -0,0 +1,7 @@
a
b
c
d
e
f
g

14
sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala Normal file
View File

@ -0,0 +1,14 @@
import org.apache.spark.sql.SparkSession
object SimpleApp {
def main(args: Array[String]) {
val logFile = "log.txt"
val spark = SparkSession.builder.appName("Simple Application").config("spark.master", "local").getOrCreate()
try {
val logData = spark.read.textFile(logFile).cache()
val numAs = logData.filter(line => line.contains("a")).count()
val numBs = logData.filter(line => line.contains("b")).count()
println(s"Lines with a: $numAs, Lines with b: $numBs")
} finally spark.stop()
}
}

11
sbt/src/sbt-test/classloader-cache/spark/test Normal file
View File

@ -0,0 +1,11 @@
> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
> run
> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary
> run
> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
> run