Set classLoaderLayeringStrategy in relevant configs

Previously, the ClassLoaderLayeringStrategy was set globally. This
didn't really make sense because the Runtime and Test configs had
different strategies available (Test being a superset of Runtime).
Instead, we now set the layering strategy in the Runtime and Test
configurations directly. In doing this, we can eliminate the Default
ClassLoaderLayeringStrategy. Previously this had existed so that we
could set the layering strategy globally and have it do the right thing
in both test and runtime.

To implement this, I factored out the logic for generating the layered
classloader in the test task and shared it with the runtime task. I did
this because I realized that Test / run is a thing. Previously I had
been operating under the assumption that the runner would never include
the test dependencies. Once I realized this, it made sense to combine
the logic in both tasks.

As a bonus, I only allow the layering strategies that explicitly make
sense to be set in each configuration. If the user sets an invalid
strategy, an error will be thrown that specifies the valid strategies
for the task.

I also added ScalaInstance as an option for the runtime layer. It was an
oversight that this was left out.
This commit is contained in:
Ethan Atkins 2019-01-29 11:37:02 -08:00
parent 0fb60733cd
commit 0a4fbc9f5a
8 changed files with 228 additions and 107 deletions

View File

@ -70,11 +70,6 @@ sealed trait ClassLoaderLayeringStrategy
*/
object ClassLoaderLayeringStrategy {
/**
* Use the default ClassLoaderLayeringStrategy for this task.
*/
case object Default extends ClassLoaderLayeringStrategy
/**
* Include all of the dependencies in the loader. The base loader will be the Application
* ClassLoader. All classes apart from system classes will be reloaded with each run.

View File

@ -141,7 +141,6 @@ object Defaults extends BuildCommon {
defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
excludeFilter :== HiddenFileFilter,
classLoaderCache := ClassLoaderCache(4),
classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Default
) ++ TaskRepository
.proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore
) ++ globalSbtCore
@ -1024,7 +1023,7 @@ object Defaults extends BuildCommon {
cp,
forkedParallelExecution = false,
javaOptions = Nil,
strategy = ClassLoaderLayeringStrategy.Default
strategy = ClassLoaderLayeringStrategy.TestDependencies,
)
}
@ -1046,7 +1045,7 @@ object Defaults extends BuildCommon {
cp,
forkedParallelExecution,
javaOptions = Nil,
strategy = ClassLoaderLayeringStrategy.Default
strategy = ClassLoaderLayeringStrategy.TestDependencies,
)
}
@ -1407,8 +1406,10 @@ object Defaults extends BuildCommon {
}
}
@deprecated("This is no longer used internally by sbt.", "1.3.0")
def runnerTask: Setting[Task[ScalaRun]] = runner := runnerInit.value
@deprecated("This is no longer used internally by sbt.", "1.3.0")
def runnerInit: Initialize[Task[ScalaRun]] = ClassLoaders.runner
private def foreachJobTask(
@ -1759,7 +1760,11 @@ object Defaults extends BuildCommon {
// 1. runnerSettings is added unscoped via JvmPlugin.
// 2. In addition it's added scoped to run task.
lazy val runnerSettings: Seq[Setting[_]] = Seq(runnerTask, forkOptions := forkOptionsTask.value)
lazy val runnerSettings: Seq[Setting[_]] = {
val unscoped: Seq[Def.Setting[_]] =
Seq(runner := ClassLoaders.runner.value, forkOptions := forkOptionsTask.value)
inConfig(Compile)(unscoped) ++ inConfig(Test)(unscoped)
}
lazy val baseTasks: Seq[Setting[_]] = projectTasks ++ packageBase
@ -1767,25 +1772,32 @@ object Defaults extends BuildCommon {
Classpaths.configSettings ++ configTasks ++ configPaths ++ packageConfig ++
Classpaths.compilerPluginConfig ++ deprecationSettings
lazy val compileSettings: Seq[Setting[_]] =
configSettings ++
(mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++
Classpaths.addUnmanagedLibrary
// We need a cache of size two for the test dependency layers (regular and snapshot).
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++ TaskRepository.proxy(
private val runtimeLayeringSettings: Seq[Setting[_]] = TaskRepository.proxy(
classLoaderCache,
ClassLoaderCache(2)
)
lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest)(testSettings)
lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++
inConfig(Test)(testSettings) ++ inConfig(Runtime)(
// We need a cache of size four so that the subset of the runtime dependencies that are used
// by the test task layers may be cached without evicting the runtime classloader layers. The
// cache size should be a multiple of two to support snapshot layers.
Classpaths.configSettings ++ TaskRepository.proxy(classLoaderCache, ClassLoaderCache(4)),
)
ClassLoaderCache(4)
) :+ (classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies)
lazy val compileSettings: Seq[Setting[_]] =
configSettings ++
(mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++
Classpaths.addUnmanagedLibrary ++ runtimeLayeringSettings
private val testLayeringSettings: Seq[Setting[_]] = TaskRepository.proxy(
classLoaderCache,
// We need a cache of size two for the test dependency layers (regular and snapshot).
ClassLoaderCache(2)
) :+ (classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies)
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks
lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest) {
testSettings :+ (classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies)
}
lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++
inConfig(Test)(testSettings ++ testLayeringSettings) ++
inConfig(Runtime)(Classpaths.configSettings)
// These are project level settings that MUST be on every project.
lazy val coreDefaultSettings: Seq[Setting[_]] =
@ -3560,17 +3572,19 @@ trait BuildExtra extends BuildCommon with DefExtra {
InputTask.apply(Def.value((s: State) => Def.spaceDelimited()))(f)
Vector(
scoped := (inputTask { result =>
(initScoped(scoped.scopedKey, runnerInit)
.zipWith(Def.task { ((fullClasspath in config).value, streams.value, result.value) })) {
scoped := inputTask { result =>
initScoped(
scoped.scopedKey,
ClassLoaders.runner mapReferenced Project.mapScope(s => s.in(config))
).zipWith(Def.task { ((fullClasspath in config).value, streams.value, result.value) }) {
(rTask, t) =>
(t, rTask) map {
case ((cp, s, args), r) =>
r.run(mainClass, data(cp), baseArguments ++ args, s.log).get
}
}
}).evaluated
) ++ inTask(scoped)(forkOptions := forkOptionsTask.value)
}.evaluated
) ++ inTask(scoped)(forkOptions in config := forkOptionsTask.value)
}
// public API
@ -3582,16 +3596,18 @@ trait BuildExtra extends BuildCommon with DefExtra {
arguments: String*
): Vector[Setting[_]] =
Vector(
scoped := ((initScoped(scoped.scopedKey, runnerInit)
.zipWith(Def.task { ((fullClasspath in config).value, streams.value) })) {
scoped := initScoped(
scoped.scopedKey,
ClassLoaders.runner mapReferenced Project.mapScope(s => s.in(config))
).zipWith(Def.task { ((fullClasspath in config).value, streams.value) }) {
case (rTask, t) =>
(t, rTask) map {
case ((cp, s), r) =>
r.run(mainClass, data(cp), arguments, s.log).get
}
})
}
.value
) ++ inTask(scoped)(forkOptions := forkOptionsTask.value)
) ++ inTask(scoped)(forkOptions in config := forkOptionsTask.value)
def initScoped[T](sk: ScopedKey[_], i: Initialize[T]): Initialize[T] =
initScope(fillTaskAxis(sk.scope, sk.key), i)

View File

@ -21,6 +21,9 @@ import sbt.io.IO
import sbt.librarymanagement.Configurations.Runtime
import PrettyPrint.indent
import scala.annotation.tailrec
import ClassLoaderLayeringStrategy.{ ScalaInstance => ScalaInstanceLayer, _ }
private[sbt] object ClassLoaders {
private[this] lazy val interfaceLoader =
combine(
@ -28,6 +31,14 @@ private[sbt] object ClassLoaders {
new NullLoader,
toString = "sbt.testing.Framework interface ClassLoader"
)
private[this] lazy val baseLoader = {
@tailrec
def getBase(classLoader: ClassLoader): ClassLoader = classLoader.getParent match {
case null => classLoader
case loader => getBase(loader)
}
getBase(ClassLoaders.getClass.getClassLoader)
}
/*
* Get the class loader for a test task. The configuration could be IntegrationTest or Test.
*/
@ -35,76 +46,28 @@ private[sbt] object ClassLoaders {
val si = scalaInstance.value
val rawCP = data(fullClasspath.value)
val fullCP = if (si.isManagedVersion) rawCP else si.allJars.toSeq ++ rawCP
val strategy = classLoaderLayeringStrategy.value
val runtimeCache = (Runtime / classLoaderCache).value
val testCache = classLoaderCache.value
val tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value)
val resources = ClasspathUtilities.createClasspathResources(fullCP, si)
val raw = strategy match {
case ClassLoaderLayeringStrategy.Flat => flatLoader(rawCP, interfaceLoader)
case s =>
/*
* Create a layered classloader. There are up to four layers:
* 1) the scala instance class loader
* 2) the runtime dependencies
* 3) the test dependencies
* 4) the rest of the classpath
* The first two layers may be optionally cached to reduce memory usage and improve
* start up latency. Because there may be mutually incompatible libraries in the runtime
* and test dependencies, it's important to be able to configure which layers are used.
*/
val (layerDependencies, layerTestDependencies) = s match {
case ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies =>
(true, true)
case ClassLoaderLayeringStrategy.ScalaInstance => (false, false)
case ClassLoaderLayeringStrategy.RuntimeDependencies => (true, false)
case _ => (false, true)
}
// Do not include exportedProducts in any cached layers because they may change between runs.
val exclude = dependencyJars(exportedProducts).value.toSet ++ si.allJars.toSeq
// The raw declarations are to avoid having to make a dynamic task. The
// allDependencies and allTestDependencies create a mutually exclusive list of jar
// dependencies for layers 2 and 3
val rawTestDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude)
val allTestDependencies = (if (layerTestDependencies) rawTestDependencies else Nil).toSet
val rawDependencies =
dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
val allDependencies = (if (layerDependencies) rawDependencies else Nil).toSet
// layer 2
val runtimeDependencies = allTestDependencies intersect allDependencies
val runtimeLayer =
layer(runtimeDependencies.toSeq, loader(si), runtimeCache, resources, tmp)
// layers 3 (optional if testDependencies are empty)
// The top layer needs to include the interface jar or else the test task cannot be created.
// It needs to be separated from the runtimeLayer or else the runtimeLayer cannot be
// shared between the runtime and test tasks.
val top = combine(interfaceLoader, runtimeLayer)
val testDependencies = allTestDependencies diff runtimeDependencies
val testLayer = layer(testDependencies.toSeq, top, testCache, resources, tmp)
// layer 4
val dynamicClasspath =
fullCP.filterNot(testDependencies ++ runtimeDependencies ++ si.allJars)
if (dynamicClasspath.nonEmpty)
new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp)
else testLayer
}
ClasspathUtilities.filterByClasspath(fullCP, raw)
val exclude = dependencyJars(exportedProducts).value.toSet ++ si.allJars.toSeq
buildLayers(
classLoaderLayeringStrategy.value,
si,
fullCP,
dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude),
dependencyJars(dependencyClasspath).value.filterNot(exclude).toSet,
interfaceLoader,
(Runtime / classLoaderCache).value,
classLoaderCache.value,
ClasspathUtilities.createClasspathResources(fullCP, si),
IO.createUniqueDirectory(taskTemporaryDirectory.value),
resolvedScoped.value.scope
)
}
private[sbt] def runner: Def.Initialize[Task[ScalaRun]] = Def.taskDyn {
val tmp = taskTemporaryDirectory.value
val resolvedScope = resolvedScoped.value.scope
val instance = scalaInstance.value
val s = streams.value
val opts = forkOptions.value
val options = javaOptions.value
val exclude = dependencyJars(Runtime / exportedProducts).value.toSet ++ instance.allJars
val dependencies = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
if (fork.value) {
s.log.debug(s"javaOptions: $options")
Def.task(new ForkRun(opts))
@ -124,26 +87,99 @@ private[sbt] object ClassLoaders {
)
s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
}
val cache = (Runtime / classLoaderCache).value
val runtimeCache = (Runtime / classLoaderCache).value
val testCache = classLoaderCache.value
val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars
val newLoader =
(classpath: Seq[File]) => {
val resources = ClasspathUtilities.createClasspathResources(classpath, instance)
val classLoader = classLoaderLayeringStrategy.value match {
case ClassLoaderLayeringStrategy.Flat =>
ClasspathUtilities
.toLoader(Nil, flatLoader(classpath, new NullLoader), resources, tmp)
case _ =>
val dependencyLoader = layer(dependencies, loader(instance), cache, resources, tmp)
val dynamicClasspath = (classpath.toSet -- dependencies).toSeq
new LayeredClassLoader(dynamicClasspath, dependencyLoader, resources, tmp)
}
ClasspathUtilities.filterByClasspath(classpath, classLoader)
buildLayers(
classLoaderLayeringStrategy.value: @sbtUnchecked,
instance,
classpath,
(dependencyJars(Runtime / dependencyClasspath).value: @sbtUnchecked)
.filterNot(exclude),
(dependencyJars(dependencyClasspath).value: @sbtUnchecked).filterNot(exclude).toSet,
baseLoader,
runtimeCache,
testCache,
ClasspathUtilities.createClasspathResources(classpath, instance),
taskTemporaryDirectory.value: @sbtUnchecked,
resolvedScope
)
}
new Run(newLoader, trapExit.value)
}
}
}
/*
* Create a layered classloader. There are up to four layers:
* 1) the scala instance class loader
* 2) the runtime dependencies
* 3) the test dependencies
* 4) the rest of the classpath
* The first two layers may be optionally cached to reduce memory usage and improve
* start up latency. Because there may be mutually incompatible libraries in the runtime
* and test dependencies, it's important to be able to configure which layers are used.
*/
private def buildLayers(
strategy: ClassLoaderLayeringStrategy,
si: ScalaInstance,
fullCP: Seq[File],
rawRuntimeDependencies: Seq[File],
allDependencies: Set[File],
base: ClassLoader,
runtimeCache: ClassLoaderCache,
testCache: ClassLoaderCache,
resources: Map[String, String],
tmp: File,
scope: Scope
): ClassLoader = {
val isTest = scope.config.toOption.map(_.name) == Option("test")
val raw = strategy match {
case Flat => flatLoader(fullCP, base)
case _ =>
val (layerDependencies, layerTestDependencies) = strategy match {
case ShareRuntimeDependenciesLayerWithTestDependencies if isTest => (true, true)
case ScalaInstanceLayer => (false, false)
case RuntimeDependencies => (true, false)
case TestDependencies if isTest => (false, true)
case badStrategy =>
val msg = s"Layering strategy $badStrategy is not valid for the classloader in " +
s"$scope. Valid options are: ClassLoaderLayeringStrategy.{ " +
"Flat, ScalaInstance, RuntimeDependencies }"
throw new IllegalArgumentException(msg)
}
// The raw declarations are to avoid having to make a dynamic task. The
// allDependencies and allTestDependencies create a mutually exclusive list of jar
// dependencies for layers 2 and 3. Note that in the Runtime or Compile configs, it
// should always be the case that allTestDependencies == Nil
val allTestDependencies = if (layerTestDependencies) allDependencies else Set.empty[File]
val allRuntimeDependencies = (if (layerDependencies) rawRuntimeDependencies else Nil).toSet
// layer 2
val runtimeDependencies = allDependencies intersect allRuntimeDependencies
val runtimeLayer =
layer(runtimeDependencies.toSeq, loader(si), runtimeCache, resources, tmp)
// layer 3 (optional if testDependencies are empty)
// The top layer needs to include the interface jar or else the test task cannot be created.
// It needs to be separated from the runtimeLayer or else the runtimeLayer cannot be
// shared between the runtime and test tasks.
val top = combine(base, runtimeLayer)
val testDependencies = allTestDependencies diff runtimeDependencies
val testLayer = layer(testDependencies.toSeq, top, testCache, resources, tmp)
// layer 4
val dynamicClasspath =
fullCP.filterNot(testDependencies ++ runtimeDependencies ++ si.allJars)
if (dynamicClasspath.nonEmpty)
new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp)
else testLayer
}
ClasspathUtilities.filterByClasspath(fullCP, raw)
}
private def dependencyJars(
key: sbt.TaskKey[Seq[Attributed[File]]]
): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))

View File

@ -0,0 +1,6 @@
val layeringStrategyTest = (project in file(".")).settings(
name := "layering-strategy-test",
scalaVersion := "2.12.8",
organization := "sbt",
libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.5.16",
)

View File

@ -0,0 +1,15 @@
package sbt.scripted
import akka.actor.ActorSystem
import scala.concurrent.Await
import scala.concurrent.duration._
object AkkaTest {
def main(args: Array[String]): Unit = {
val now = System.nanoTime
val system = ActorSystem("akka")
Await.result(system.terminate(), 5.seconds)
val elapsed = System.nanoTime - now
println(s"Run took ${elapsed / 1.0e6} ms")
}
}

View File

@ -0,0 +1,15 @@
package sbt.scripted
import akka.actor.ActorSystem
import scala.concurrent.Await
import scala.concurrent.duration._
object TestAkkaTest {
def main(args: Array[String]): Unit = {
val now = System.nanoTime
val system = ActorSystem("akka")
Await.result(system.terminate(), 5.seconds)
val elapsed = System.nanoTime - now
println(s"Test run took ${elapsed / 1.0e6} ms")
}
}

View File

@ -0,0 +1,37 @@
> run
> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
> run
> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaInstance
> run
> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies
> run
> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
-> run
> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
> Test / runMain sbt.scripted.TestAkkaTest
> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaInstance
> Test / runMain sbt.scripted.TestAkkaTest
> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies
> Test / runMain sbt.scripted.TestAkkaTest
> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
> Test / runMain sbt.scripted.TestAkkaTest
> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
> Test / runMain sbt.scripted.TestAkkaTest

View File

@ -173,6 +173,7 @@ final class ScriptedTests(
case "classloader-cache/akka-actor-system" => LauncherBased // sbt/Package$
case "classloader-cache/jni" => LauncherBased // sbt/Package$
case "classloader-cache/library-mismatch" => LauncherBased // sbt/Package$
case "classloader-cache/runtime-layers" => LauncherBased // sbt/Package$
case "compiler-project/dotty-compiler-plugin" => LauncherBased // sbt/Package$
case "compiler-project/run-test" => LauncherBased // sbt/Package$
case "compiler-project/src-dep-plugin" => LauncherBased // sbt/Package$