luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use layered ClassLoaders in run and test tasks 

Using the data structures that I added in the previous commits, it is
now possible to rework the run and test task to use (configurable)
layered class loaders. The layering strategy is globally set to
LayeringStrategy.Default. The default strategy leads to what is
effectively a three layered ClassLoader for the both the test and run
tasks. The first layer contains the scala instance (and test framework
loader in the test task). The second layer contains all of the
dependencies for the configuration while the third layer contains the
project artifacts.

The layering strategy is very easily changed both at the Global or
Configuration level, e.g. adding
Test / layeringStrategy := LayeringStrategy.Flat
to the project build.sbt will make the test task not even use the scala
instance and instead a create a single layer containing the full
classpath of the test task.

I also tried to ensure that all of the ClassLoaders have good toString
overrides so that it's easy to see how the ClassLoader is constructed
with, e.g. `show testLoader`, in the sbt console.

In this commit, the ClassLoaderCache instances are settings. In the next
commit, I make them tasks so that we can easily clear out the caches
with a command.
This commit is contained in:
Ethan Atkins 2018-12-03 18:23:12 -08:00
parent 8cba83aebb
commit a06f5435c6
6 changed files with 287 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
main/src/main/scala/sbt/Defaults.scala
View File

@ -139,7 +139,9 @@ object Defaults extends BuildCommon {
)
private[sbt] lazy val globalCore: Seq[Setting[_]] = globalDefaults(
defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
excludeFilter :== HiddenFileFilter
excludeFilter :== HiddenFileFilter,
classLoaderCache := ClassLoaderCache(4),
layeringStrategy := LayeringStrategy.Default
) ++ globalIvyCore ++ globalJvmCore
) ++ globalSbtCore
@ -790,11 +792,7 @@ object Defaults extends BuildCommon {
: Seq[Setting[_]] = testTaskOptions(test) ++ testTaskOptions(testOnly) ++ testTaskOptions(
testQuick
) ++ testDefaults ++ Seq(
testLoader := TestFramework.createTestLoader(
data(fullClasspath.value),
scalaInstance.value,
IO.createUniqueDirectory(taskTemporaryDirectory.value)
),
testLoader := ClassLoaders.testTask.value,
loadedTestFrameworks := {
val loader = testLoader.value
val log = streams.value.log
@ -813,7 +811,8 @@ object Defaults extends BuildCommon {
(testExecution in test).value,
(fullClasspath in test).value,
testForkedParallel.value,
(javaOptions in test).value
(javaOptions in test).value,
(layeringStrategy).value
)
}
).value,
@ -979,7 +978,8 @@ object Defaults extends BuildCommon {
newConfig,
fullClasspath.value,
testForkedParallel.value,
javaOptions.value
javaOptions.value,
layeringStrategy.value
)
val taskName = display.show(resolvedScoped.value)
val trl = testResultLogger.value
@ -1022,7 +1022,8 @@ object Defaults extends BuildCommon {
config,
cp,
forkedParallelExecution = false,
javaOptions = Nil
javaOptions = Nil,
strategy = LayeringStrategy.Default
)
}
@ -1043,7 +1044,8 @@ object Defaults extends BuildCommon {
config,
cp,
forkedParallelExecution,
javaOptions = Nil
javaOptions = Nil,
strategy = LayeringStrategy.Default
)
}
@ -1055,7 +1057,8 @@ object Defaults extends BuildCommon {
config: Tests.Execution,
cp: Classpath,
forkedParallelExecution: Boolean,
javaOptions: Seq[String]
javaOptions: Seq[String],
strategy: LayeringStrategy,
): Initialize[Task[Tests.Output]] = {
val runners = createTestRunners(frameworks, loader, config)
val groupTasks = groups map {
@ -1083,6 +1086,23 @@ object Defaults extends BuildCommon {
}
val output = Tests.foldTasks(groupTasks, config.parallel)
val result = output map { out =>
out.events.foreach {
case (suite, e) =>
e.throwables
.collectFirst {
case t if t.isInstanceOf[NoClassDefFoundError] && strategy != LayeringStrategy.Flat =>
t
}
.foreach { t =>
s.log.error(
s"Test suite $suite failed with $t. This may be due to the LayeringStrategy"
+ s" ($strategy) used by your task. This issue may be resolved by changing the"
+ " LayeringStrategy in your configuration (generally Test or IntegrationTest),"
+ "e.g.:\nTest / layeringStrategy := LayeringStrategy.Flat\n"
+ "See LayeringStrategy.scala for the full list of options."
)
}
}
val summaries =
runners map {
case (tf, r) =>
@ -1386,35 +1406,7 @@ object Defaults extends BuildCommon {
def runnerTask: Setting[Task[ScalaRun]] = runner := runnerInit.value
def runnerInit: Initialize[Task[ScalaRun]] = Def.task {
val tmp = taskTemporaryDirectory.value
val resolvedScope = resolvedScoped.value.scope
val si = scalaInstance.value
val s = streams.value
val opts = forkOptions.value
val options = javaOptions.value
val trap = trapExit.value
if (fork.value) {
s.log.debug(s"javaOptions: $options")
new ForkRun(opts)
} else {
if (options.nonEmpty) {
val mask = ScopeMask(project = false)
val showJavaOptions = Scope.displayMasked(
(javaOptions in resolvedScope).scopedKey.scope,
(javaOptions in resolvedScope).key.label,
mask
)
val showFork = Scope.displayMasked(
(fork in resolvedScope).scopedKey.scope,
(fork in resolvedScope).key.label,
mask
)
s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
}
new Run(si, trap, tmp)
}
}
def runnerInit: Initialize[Task[ScalaRun]] = ClassLoaders.runner
private def foreachJobTask(
f: (BackgroundJobService, JobHandle) => Unit
@ -1777,11 +1769,18 @@ object Defaults extends BuildCommon {
(mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++
Classpaths.addUnmanagedLibrary
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks
// We need a cache of size two for the test dependency layers (regular and snapshot).
lazy val testSettings
: Seq[Setting[_]] = configSettings ++ testTasks :+ (classLoaderCache := ClassLoaderCache(2))
lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest)(testSettings)
lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++
inConfig(Test)(testSettings) ++ inConfig(Runtime)(Classpaths.configSettings)
inConfig(Test)(testSettings) ++ inConfig(Runtime)(
// We need a cache of size four so that the subset of the runtime dependencies that are used
// by the test task layers may be cached without evicting the runtime classloader layres. The
// cache size should be a multiple of two to support snapshot layers.
Classpaths.configSettings :+ (classLoaderCache := ClassLoaderCache(4))
)
// These are project level settings that MUST be on every project.
lazy val coreDefaultSettings: Seq[Setting[_]] =

2
main/src/main/scala/sbt/Keys.scala
View File

@ -264,6 +264,7 @@ object Keys {
val bgRunMain = inputKey[JobHandle]("Start a provided main class as a background job")
val fgRunMain = inputKey[Unit]("Start a provided main class as a foreground job")
val bgCopyClasspath = settingKey[Boolean]("Copies classpath on bgRun to prevent conflict.")
val layeringStrategy = settingKey[LayeringStrategy]("Creates the classloader layering strategy for the particular configuration.")
// Test Keys
val testLoader = taskKey[ClassLoader]("Provides the class loader used for testing.").withRank(DTask)
@ -460,6 +461,7 @@ object Keys {
val resolvedScoped = Def.resolvedScoped
val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask)
val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask)
val classLoaderCache = settingKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask)
// wrapper to work around SI-2915
private[sbt] final class TaskProgress(val progress: ExecuteProgress[Task])

212
main/src/main/scala/sbt/internal/ClassLoaders.scala Normal file
View File

@ -0,0 +1,212 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt
package internal
import java.io.File
import java.net.URLClassLoader
import sbt.Keys._
import sbt.SlashSyntax0._
import sbt.internal.inc.ScalaInstance
import sbt.internal.inc.classpath.{ ClasspathUtilities, DualLoader, NullLoader }
import sbt.internal.util.Attributed
import sbt.internal.util.Attributed.data
import sbt.io.IO
import sbt.librarymanagement.Configurations.Runtime
import PrettyPrint.indent
private[sbt] object ClassLoaders {
private[this] lazy val interfaceLoader =
combine(
classOf[sbt.testing.Framework].getClassLoader,
new NullLoader,
toString = "sbt.testing.Framework interface ClassLoader"
)
/*
* Get the class loader for a test task. The configuration could be IntegrationTest or Test.
*/
private[sbt] def testTask: Def.Initialize[Task[ClassLoader]] = Def.task {
val si = scalaInstance.value
val rawCP = data(fullClasspath.value)
val fullCP = if (si.isManagedVersion) rawCP else si.allJars.toSeq ++ rawCP
val strategy = layeringStrategy.value
val runtimeCache = (Runtime / classLoaderCache).value
val testCache = classLoaderCache.value
val tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value)
val resources = ClasspathUtilities.createClasspathResources(fullCP, si)
val raw = strategy match {
case LayeringStrategy.Flat => flatLoader(rawCP, interfaceLoader)
case s =>
/*
* Create a layered classloader. There are up to four layers:
* 1) the scala instance class loader
* 2) the runtime dependencies
* 3) the test dependencies
* 4) the rest of the classpath
* The first two layers may be optionally cached to reduce memory usage and improve
* start up latency. Because there may be mutually incompatible libraries in the runtime
* and test dependencies, it's important to be able to configure which layers are used.
*/
val (layerDependencies, layerTestDependencies) = s match {
case LayeringStrategy.Full => (true, true)
case LayeringStrategy.ScalaInstance => (false, false)
case LayeringStrategy.RuntimeDependencies => (true, false)
case _ => (false, true)
}
// Do not include exportedProducts in any cached layers because they may change between runs.
val exclude = dependencyJars(exportedProducts).value.toSet ++ si.allJars.toSeq
// The raw declarations are to avoid having to make a dynamic task. The
// allDependencies and allTestDependencies create a mutually exclusive list of jar
// dependencies for layers 2 and 3
val rawTestDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude)
val allTestDependencies = (if (layerTestDependencies) rawTestDependencies else Nil).toSet
val rawDependencies =
dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
val allDependencies = (if (layerDependencies) rawDependencies else Nil).toSet
// layer 2
val runtimeDependencies = allTestDependencies intersect allDependencies
val runtimeLayer =
layer(runtimeDependencies.toSeq, loader(si), runtimeCache, resources, tmp)
// layers 3 (optional if testDependencies are empty)
// The top layer needs to include the interface jar or else the test task cannot be created.
// It needs to be separated from the runtimeLayer or else the runtimeLayer cannot be
// shared between the runtime and test tasks.
val top = combine(interfaceLoader, runtimeLayer)
val testDependencies = allTestDependencies diff runtimeDependencies
val testLayer = layer(testDependencies.toSeq, top, testCache, resources, tmp)
// layer 4
val dynamicClasspath =
fullCP.filterNot(testDependencies ++ runtimeDependencies ++ si.allJars)
if (dynamicClasspath.nonEmpty)
new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp)
else testLayer
}
ClasspathUtilities.filterByClasspath(fullCP, raw)
}
private[sbt] def runner: Def.Initialize[Task[ScalaRun]] = Def.taskDyn {
val tmp = taskTemporaryDirectory.value
val resolvedScope = resolvedScoped.value.scope
val instance = scalaInstance.value
val s = streams.value
val opts = forkOptions.value
val options = javaOptions.value
val exclude = dependencyJars(Runtime / exportedProducts).value.toSet ++ instance.allJars
val dependencies = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
if (fork.value) {
s.log.debug(s"javaOptions: $options")
Def.task(new ForkRun(opts))
} else {
Def.task {
if (options.nonEmpty) {
val mask = ScopeMask(project = false)
val showJavaOptions = Scope.displayMasked(
(javaOptions in resolvedScope).scopedKey.scope,
(javaOptions in resolvedScope).key.label,
mask
)
val showFork = Scope.displayMasked(
(fork in resolvedScope).scopedKey.scope,
(fork in resolvedScope).key.label,
mask
)
s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
}
val cache = (Runtime / classLoaderCache).value
val newLoader =
(classpath: Seq[File]) => {
val resources = ClasspathUtilities.createClasspathResources(classpath, instance)
val classLoader = layeringStrategy.value match {
case LayeringStrategy.Flat =>
ClasspathUtilities
.toLoader(Nil, flatLoader(classpath, new NullLoader), resources, tmp)
case _ =>
val dependencyLoader = layer(dependencies, loader(instance), cache, resources, tmp)
val dynamicClasspath = (classpath.toSet -- dependencies).toSeq
new LayeredClassLoader(dynamicClasspath, dependencyLoader, resources, tmp)
}
ClasspathUtilities.filterByClasspath(classpath, classLoader)
}
new Run(newLoader, trapExit.value)
}
}
}
private def dependencyJars(
key: sbt.TaskKey[Seq[Attributed[File]]]
): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
// invalidating the regular jar layer. If the classpath is empty, it just returns the parent
// loader.
private def layer(
classpath: Seq[File],
parent: ClassLoader,
cache: ClassLoaderCache,
resources: Map[String, String],
tmp: File
): ClassLoader = {
val (snapshots, jars) = classpath.partition(_.toString.contains("-SNAPSHOT"))
val jarLoader = if (jars.isEmpty) parent else cache.get((jars, parent, resources, tmp))
if (snapshots.isEmpty) jarLoader else cache.get((snapshots, jarLoader, resources, tmp))
}
// Code related to combining two classloaders that primarily exists so the test loader correctly
// loads the testing framework using the same classloader as sbt itself.
private val interfaceFilter = (name: String) =>
name.startsWith("org.scalatools.testing.") || name.startsWith("sbt.testing.") || name
.startsWith("java.") || name.startsWith("sun.")
private val notInterfaceFilter = (name: String) => !interfaceFilter(name)
private class WrappedDualLoader(
val parent: ClassLoader,
val child: ClassLoader,
string: => String
) extends ClassLoader(
new DualLoader(parent, interfaceFilter, _ => false, child, notInterfaceFilter, _ => true)
) {
override def equals(o: Any): Boolean = o match {
case that: WrappedDualLoader => this.parent == that.parent && this.child == that.child
case _ => false
}
override def hashCode: Int = (parent.hashCode * 31) ^ child.hashCode
override lazy val toString: String = string
}
private def combine(parent: ClassLoader, child: ClassLoader, toString: String): ClassLoader =
new WrappedDualLoader(parent, child, toString)
private def combine(parent: ClassLoader, child: ClassLoader): ClassLoader =
new WrappedDualLoader(
parent,
child,
s"WrappedDualLoader(\n parent =\n${indent(parent, 4)}"
+ s"\n child =\n${indent(child, 4)}\n)"
)
// helper methods
private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader =
new URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent)
// This makes the toString method of the ScalaInstance classloader much more readable, but
// it is not strictly necessary.
private def loader(si: ScalaInstance): ClassLoader = new ClassLoader(si.loader) {
override lazy val toString: String =
"ScalaInstanceClassLoader(\n instance = " +
s"${indent(si.toString.split(",").mkString("\n ", ",\n ", "\n"), 4)}\n)"
// Delegate equals to that.equals in case that is itself some kind of wrapped classloader that
// needs to delegate its equals method to the delegated ClassLoader.
override def equals(that: Any): Boolean = if (that != null) that.equals(si.loader) else false
override def hashCode: Int = si.loader.hashCode
}
}

6
run/src/main/scala/sbt/Run.scala
View File

@ -58,7 +58,9 @@ class ForkRun(config: ForkOptions) extends ScalaRun {
private def classpathOption(classpath: Seq[File]) =
"-classpath" :: Path.makeString(classpath) :: Nil
}
class Run(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) extends ScalaRun {
class Run(newLoader: Seq[File] => ClassLoader, trapExit: Boolean) extends ScalaRun {
def this(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) =
this((cp: Seq[File]) => ClasspathUtilities.makeLoader(cp, instance, nativeTmp), trapExit)
/** Runs the class 'mainClass' using the given classpath and options using the scala runner.*/
def run(mainClass: String, classpath: Seq[File], options: Seq[String], log: Logger): Try[Unit] = {
@ -87,7 +89,7 @@ class Run(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) extends S
log: Logger
): Unit = {
log.debug(" Classpath:\n\t" + classpath.mkString("\n\t"))
val loader = ClasspathUtilities.makeLoader(classpath, instance, nativeTmp)
val loader = newLoader(classpath)
val main = getMainMethod(mainClassName, loader)
invokeMain(loader, main, options)
}

1
testing/src/main/scala/sbt/TestFramework.scala
View File

@ -258,6 +258,7 @@ object TestFramework {
Thread.currentThread.setContextClassLoader(loader)
try { eval } finally { Thread.currentThread.setContextClassLoader(oldLoader) }
}
@deprecated("1.3.0", "This has been replaced by the ClassLoaders.test task.")
def createTestLoader(
classpath: Seq[File],
scalaInstance: ScalaInstance,

30
testing/src/main/scala/sbt/TestReportListener.scala
View File

@ -50,8 +50,30 @@ final class SuiteResult(
val skippedCount: Int,
val ignoredCount: Int,
val canceledCount: Int,
val pendingCount: Int
val pendingCount: Int,
val throwables: Seq[Throwable]
) {
def this(
result: TestResult,
passedCount: Int,
failureCount: Int,
errorCount: Int,
skippedCount: Int,
ignoredCount: Int,
canceledCount: Int,
pendingCount: Int,
) =
this(
result,
passedCount,
failureCount,
errorCount,
skippedCount,
ignoredCount,
canceledCount,
pendingCount,
Nil
)
def +(other: SuiteResult): SuiteResult = {
val combinedTestResult =
(result, other.result) match {
@ -68,7 +90,8 @@ final class SuiteResult(
skippedCount + other.skippedCount,
ignoredCount + other.ignoredCount,
canceledCount + other.canceledCount,
pendingCount + other.pendingCount
pendingCount + other.pendingCount,
throwables ++ other.throwables
)
}
}
@ -86,7 +109,8 @@ object SuiteResult {
count(TStatus.Skipped),
count(TStatus.Ignored),
count(TStatus.Canceled),
count(TStatus.Pending)
count(TStatus.Pending),
events.collect { case e if e.throwable.isDefined => e.throwable.get }
)
}