diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 95dd8295f..d782aee3e 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -139,7 +139,9 @@ object Defaults extends BuildCommon { ) private[sbt] lazy val globalCore: Seq[Setting[_]] = globalDefaults( defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq( - excludeFilter :== HiddenFileFilter + excludeFilter :== HiddenFileFilter, + classLoaderCache := ClassLoaderCache(4), + layeringStrategy := LayeringStrategy.Default ) ++ globalIvyCore ++ globalJvmCore ) ++ globalSbtCore @@ -790,11 +792,7 @@ object Defaults extends BuildCommon { : Seq[Setting[_]] = testTaskOptions(test) ++ testTaskOptions(testOnly) ++ testTaskOptions( testQuick ) ++ testDefaults ++ Seq( - testLoader := TestFramework.createTestLoader( - data(fullClasspath.value), - scalaInstance.value, - IO.createUniqueDirectory(taskTemporaryDirectory.value) - ), + testLoader := ClassLoaders.testTask.value, loadedTestFrameworks := { val loader = testLoader.value val log = streams.value.log @@ -813,7 +811,8 @@ object Defaults extends BuildCommon { (testExecution in test).value, (fullClasspath in test).value, testForkedParallel.value, - (javaOptions in test).value + (javaOptions in test).value, + (layeringStrategy).value ) } ).value, @@ -979,7 +978,8 @@ object Defaults extends BuildCommon { newConfig, fullClasspath.value, testForkedParallel.value, - javaOptions.value + javaOptions.value, + layeringStrategy.value ) val taskName = display.show(resolvedScoped.value) val trl = testResultLogger.value @@ -1022,7 +1022,8 @@ object Defaults extends BuildCommon { config, cp, forkedParallelExecution = false, - javaOptions = Nil + javaOptions = Nil, + strategy = LayeringStrategy.Default ) } @@ -1043,7 +1044,8 @@ object Defaults extends BuildCommon { config, cp, forkedParallelExecution, - javaOptions = Nil + javaOptions = Nil, + strategy = LayeringStrategy.Default ) } @@ -1055,7 +1057,8 @@ object Defaults extends BuildCommon { config: Tests.Execution, cp: Classpath, forkedParallelExecution: Boolean, - javaOptions: Seq[String] + javaOptions: Seq[String], + strategy: LayeringStrategy, ): Initialize[Task[Tests.Output]] = { val runners = createTestRunners(frameworks, loader, config) val groupTasks = groups map { @@ -1083,6 +1086,23 @@ object Defaults extends BuildCommon { } val output = Tests.foldTasks(groupTasks, config.parallel) val result = output map { out => + out.events.foreach { + case (suite, e) => + e.throwables + .collectFirst { + case t if t.isInstanceOf[NoClassDefFoundError] && strategy != LayeringStrategy.Flat => + t + } + .foreach { t => + s.log.error( + s"Test suite $suite failed with $t. This may be due to the LayeringStrategy" + + s" ($strategy) used by your task. This issue may be resolved by changing the" + + " LayeringStrategy in your configuration (generally Test or IntegrationTest)," + + "e.g.:\nTest / layeringStrategy := LayeringStrategy.Flat\n" + + "See LayeringStrategy.scala for the full list of options." + ) + } + } val summaries = runners map { case (tf, r) => @@ -1386,35 +1406,7 @@ object Defaults extends BuildCommon { def runnerTask: Setting[Task[ScalaRun]] = runner := runnerInit.value - def runnerInit: Initialize[Task[ScalaRun]] = Def.task { - val tmp = taskTemporaryDirectory.value - val resolvedScope = resolvedScoped.value.scope - val si = scalaInstance.value - val s = streams.value - val opts = forkOptions.value - val options = javaOptions.value - val trap = trapExit.value - if (fork.value) { - s.log.debug(s"javaOptions: $options") - new ForkRun(opts) - } else { - if (options.nonEmpty) { - val mask = ScopeMask(project = false) - val showJavaOptions = Scope.displayMasked( - (javaOptions in resolvedScope).scopedKey.scope, - (javaOptions in resolvedScope).key.label, - mask - ) - val showFork = Scope.displayMasked( - (fork in resolvedScope).scopedKey.scope, - (fork in resolvedScope).key.label, - mask - ) - s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false") - } - new Run(si, trap, tmp) - } - } + def runnerInit: Initialize[Task[ScalaRun]] = ClassLoaders.runner private def foreachJobTask( f: (BackgroundJobService, JobHandle) => Unit @@ -1777,11 +1769,18 @@ object Defaults extends BuildCommon { (mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++ Classpaths.addUnmanagedLibrary - lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks + // We need a cache of size two for the test dependency layers (regular and snapshot). + lazy val testSettings + : Seq[Setting[_]] = configSettings ++ testTasks :+ (classLoaderCache := ClassLoaderCache(2)) lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest)(testSettings) lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++ - inConfig(Test)(testSettings) ++ inConfig(Runtime)(Classpaths.configSettings) + inConfig(Test)(testSettings) ++ inConfig(Runtime)( + // We need a cache of size four so that the subset of the runtime dependencies that are used + // by the test task layers may be cached without evicting the runtime classloader layres. The + // cache size should be a multiple of two to support snapshot layers. + Classpaths.configSettings :+ (classLoaderCache := ClassLoaderCache(4)) + ) // These are project level settings that MUST be on every project. lazy val coreDefaultSettings: Seq[Setting[_]] = diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index cf8cf4356..8f232d73b 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -264,6 +264,7 @@ object Keys { val bgRunMain = inputKey[JobHandle]("Start a provided main class as a background job") val fgRunMain = inputKey[Unit]("Start a provided main class as a foreground job") val bgCopyClasspath = settingKey[Boolean]("Copies classpath on bgRun to prevent conflict.") + val layeringStrategy = settingKey[LayeringStrategy]("Creates the classloader layering strategy for the particular configuration.") // Test Keys val testLoader = taskKey[ClassLoader]("Provides the class loader used for testing.").withRank(DTask) @@ -460,6 +461,7 @@ object Keys { val resolvedScoped = Def.resolvedScoped val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask) val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask) + val classLoaderCache = settingKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask) // wrapper to work around SI-2915 private[sbt] final class TaskProgress(val progress: ExecuteProgress[Task]) diff --git a/main/src/main/scala/sbt/internal/ClassLoaders.scala b/main/src/main/scala/sbt/internal/ClassLoaders.scala new file mode 100644 index 000000000..cba184ad0 --- /dev/null +++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala @@ -0,0 +1,212 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import java.io.File +import java.net.URLClassLoader + +import sbt.Keys._ +import sbt.SlashSyntax0._ +import sbt.internal.inc.ScalaInstance +import sbt.internal.inc.classpath.{ ClasspathUtilities, DualLoader, NullLoader } +import sbt.internal.util.Attributed +import sbt.internal.util.Attributed.data +import sbt.io.IO +import sbt.librarymanagement.Configurations.Runtime +import PrettyPrint.indent + +private[sbt] object ClassLoaders { + private[this] lazy val interfaceLoader = + combine( + classOf[sbt.testing.Framework].getClassLoader, + new NullLoader, + toString = "sbt.testing.Framework interface ClassLoader" + ) + /* + * Get the class loader for a test task. The configuration could be IntegrationTest or Test. + */ + private[sbt] def testTask: Def.Initialize[Task[ClassLoader]] = Def.task { + val si = scalaInstance.value + val rawCP = data(fullClasspath.value) + val fullCP = if (si.isManagedVersion) rawCP else si.allJars.toSeq ++ rawCP + val strategy = layeringStrategy.value + val runtimeCache = (Runtime / classLoaderCache).value + val testCache = classLoaderCache.value + val tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value) + val resources = ClasspathUtilities.createClasspathResources(fullCP, si) + + val raw = strategy match { + case LayeringStrategy.Flat => flatLoader(rawCP, interfaceLoader) + case s => + /* + * Create a layered classloader. There are up to four layers: + * 1) the scala instance class loader + * 2) the runtime dependencies + * 3) the test dependencies + * 4) the rest of the classpath + * The first two layers may be optionally cached to reduce memory usage and improve + * start up latency. Because there may be mutually incompatible libraries in the runtime + * and test dependencies, it's important to be able to configure which layers are used. + */ + val (layerDependencies, layerTestDependencies) = s match { + case LayeringStrategy.Full => (true, true) + case LayeringStrategy.ScalaInstance => (false, false) + case LayeringStrategy.RuntimeDependencies => (true, false) + case _ => (false, true) + } + // Do not include exportedProducts in any cached layers because they may change between runs. + val exclude = dependencyJars(exportedProducts).value.toSet ++ si.allJars.toSeq + // The raw declarations are to avoid having to make a dynamic task. The + // allDependencies and allTestDependencies create a mutually exclusive list of jar + // dependencies for layers 2 and 3 + val rawTestDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude) + val allTestDependencies = (if (layerTestDependencies) rawTestDependencies else Nil).toSet + val rawDependencies = + dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude) + val allDependencies = (if (layerDependencies) rawDependencies else Nil).toSet + + // layer 2 + val runtimeDependencies = allTestDependencies intersect allDependencies + val runtimeLayer = + layer(runtimeDependencies.toSeq, loader(si), runtimeCache, resources, tmp) + + // layers 3 (optional if testDependencies are empty) + + // The top layer needs to include the interface jar or else the test task cannot be created. + // It needs to be separated from the runtimeLayer or else the runtimeLayer cannot be + // shared between the runtime and test tasks. + val top = combine(interfaceLoader, runtimeLayer) + val testDependencies = allTestDependencies diff runtimeDependencies + val testLayer = layer(testDependencies.toSeq, top, testCache, resources, tmp) + + // layer 4 + val dynamicClasspath = + fullCP.filterNot(testDependencies ++ runtimeDependencies ++ si.allJars) + if (dynamicClasspath.nonEmpty) + new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp) + else testLayer + } + ClasspathUtilities.filterByClasspath(fullCP, raw) + } + + private[sbt] def runner: Def.Initialize[Task[ScalaRun]] = Def.taskDyn { + val tmp = taskTemporaryDirectory.value + val resolvedScope = resolvedScoped.value.scope + val instance = scalaInstance.value + val s = streams.value + val opts = forkOptions.value + val options = javaOptions.value + val exclude = dependencyJars(Runtime / exportedProducts).value.toSet ++ instance.allJars + val dependencies = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude) + if (fork.value) { + s.log.debug(s"javaOptions: $options") + Def.task(new ForkRun(opts)) + } else { + Def.task { + if (options.nonEmpty) { + val mask = ScopeMask(project = false) + val showJavaOptions = Scope.displayMasked( + (javaOptions in resolvedScope).scopedKey.scope, + (javaOptions in resolvedScope).key.label, + mask + ) + val showFork = Scope.displayMasked( + (fork in resolvedScope).scopedKey.scope, + (fork in resolvedScope).key.label, + mask + ) + s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false") + } + val cache = (Runtime / classLoaderCache).value + val newLoader = + (classpath: Seq[File]) => { + val resources = ClasspathUtilities.createClasspathResources(classpath, instance) + val classLoader = layeringStrategy.value match { + case LayeringStrategy.Flat => + ClasspathUtilities + .toLoader(Nil, flatLoader(classpath, new NullLoader), resources, tmp) + case _ => + val dependencyLoader = layer(dependencies, loader(instance), cache, resources, tmp) + val dynamicClasspath = (classpath.toSet -- dependencies).toSeq + new LayeredClassLoader(dynamicClasspath, dependencyLoader, resources, tmp) + } + ClasspathUtilities.filterByClasspath(classpath, classLoader) + } + new Run(newLoader, trapExit.value) + } + } + } + + private def dependencyJars( + key: sbt.TaskKey[Seq[Attributed[File]]] + ): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar"))) + + // Creates a one or two layered classloader for the provided classpaths depending on whether + // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer + // above the regular jar layer. This allows the snapshot layer to be invalidated without + // invalidating the regular jar layer. If the classpath is empty, it just returns the parent + // loader. + private def layer( + classpath: Seq[File], + parent: ClassLoader, + cache: ClassLoaderCache, + resources: Map[String, String], + tmp: File + ): ClassLoader = { + val (snapshots, jars) = classpath.partition(_.toString.contains("-SNAPSHOT")) + val jarLoader = if (jars.isEmpty) parent else cache.get((jars, parent, resources, tmp)) + if (snapshots.isEmpty) jarLoader else cache.get((snapshots, jarLoader, resources, tmp)) + } + + // Code related to combining two classloaders that primarily exists so the test loader correctly + // loads the testing framework using the same classloader as sbt itself. + private val interfaceFilter = (name: String) => + name.startsWith("org.scalatools.testing.") || name.startsWith("sbt.testing.") || name + .startsWith("java.") || name.startsWith("sun.") + private val notInterfaceFilter = (name: String) => !interfaceFilter(name) + private class WrappedDualLoader( + val parent: ClassLoader, + val child: ClassLoader, + string: => String + ) extends ClassLoader( + new DualLoader(parent, interfaceFilter, _ => false, child, notInterfaceFilter, _ => true) + ) { + override def equals(o: Any): Boolean = o match { + case that: WrappedDualLoader => this.parent == that.parent && this.child == that.child + case _ => false + } + override def hashCode: Int = (parent.hashCode * 31) ^ child.hashCode + override lazy val toString: String = string + } + private def combine(parent: ClassLoader, child: ClassLoader, toString: String): ClassLoader = + new WrappedDualLoader(parent, child, toString) + private def combine(parent: ClassLoader, child: ClassLoader): ClassLoader = + new WrappedDualLoader( + parent, + child, + s"WrappedDualLoader(\n parent =\n${indent(parent, 4)}" + + s"\n child =\n${indent(child, 4)}\n)" + ) + + // helper methods + private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader = + new URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) + + // This makes the toString method of the ScalaInstance classloader much more readable, but + // it is not strictly necessary. + private def loader(si: ScalaInstance): ClassLoader = new ClassLoader(si.loader) { + override lazy val toString: String = + "ScalaInstanceClassLoader(\n instance = " + + s"${indent(si.toString.split(",").mkString("\n ", ",\n ", "\n"), 4)}\n)" + // Delegate equals to that.equals in case that is itself some kind of wrapped classloader that + // needs to delegate its equals method to the delegated ClassLoader. + override def equals(that: Any): Boolean = if (that != null) that.equals(si.loader) else false + override def hashCode: Int = si.loader.hashCode + } +} diff --git a/run/src/main/scala/sbt/Run.scala b/run/src/main/scala/sbt/Run.scala index a09d3fc3f..754974e1e 100644 --- a/run/src/main/scala/sbt/Run.scala +++ b/run/src/main/scala/sbt/Run.scala @@ -58,7 +58,9 @@ class ForkRun(config: ForkOptions) extends ScalaRun { private def classpathOption(classpath: Seq[File]) = "-classpath" :: Path.makeString(classpath) :: Nil } -class Run(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) extends ScalaRun { +class Run(newLoader: Seq[File] => ClassLoader, trapExit: Boolean) extends ScalaRun { + def this(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) = + this((cp: Seq[File]) => ClasspathUtilities.makeLoader(cp, instance, nativeTmp), trapExit) /** Runs the class 'mainClass' using the given classpath and options using the scala runner.*/ def run(mainClass: String, classpath: Seq[File], options: Seq[String], log: Logger): Try[Unit] = { @@ -87,7 +89,7 @@ class Run(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) extends S log: Logger ): Unit = { log.debug(" Classpath:\n\t" + classpath.mkString("\n\t")) - val loader = ClasspathUtilities.makeLoader(classpath, instance, nativeTmp) + val loader = newLoader(classpath) val main = getMainMethod(mainClassName, loader) invokeMain(loader, main, options) } diff --git a/testing/src/main/scala/sbt/TestFramework.scala b/testing/src/main/scala/sbt/TestFramework.scala index d06f95c42..071576668 100644 --- a/testing/src/main/scala/sbt/TestFramework.scala +++ b/testing/src/main/scala/sbt/TestFramework.scala @@ -258,6 +258,7 @@ object TestFramework { Thread.currentThread.setContextClassLoader(loader) try { eval } finally { Thread.currentThread.setContextClassLoader(oldLoader) } } + @deprecated("1.3.0", "This has been replaced by the ClassLoaders.test task.") def createTestLoader( classpath: Seq[File], scalaInstance: ScalaInstance, diff --git a/testing/src/main/scala/sbt/TestReportListener.scala b/testing/src/main/scala/sbt/TestReportListener.scala index 6a02c35e0..e16f3c8ec 100644 --- a/testing/src/main/scala/sbt/TestReportListener.scala +++ b/testing/src/main/scala/sbt/TestReportListener.scala @@ -50,8 +50,30 @@ final class SuiteResult( val skippedCount: Int, val ignoredCount: Int, val canceledCount: Int, - val pendingCount: Int + val pendingCount: Int, + val throwables: Seq[Throwable] ) { + def this( + result: TestResult, + passedCount: Int, + failureCount: Int, + errorCount: Int, + skippedCount: Int, + ignoredCount: Int, + canceledCount: Int, + pendingCount: Int, + ) = + this( + result, + passedCount, + failureCount, + errorCount, + skippedCount, + ignoredCount, + canceledCount, + pendingCount, + Nil + ) def +(other: SuiteResult): SuiteResult = { val combinedTestResult = (result, other.result) match { @@ -68,7 +90,8 @@ final class SuiteResult( skippedCount + other.skippedCount, ignoredCount + other.ignoredCount, canceledCount + other.canceledCount, - pendingCount + other.pendingCount + pendingCount + other.pendingCount, + throwables ++ other.throwables ) } } @@ -86,7 +109,8 @@ object SuiteResult { count(TStatus.Skipped), count(TStatus.Ignored), count(TStatus.Canceled), - count(TStatus.Pending) + count(TStatus.Pending), + events.collect { case e if e.throwable.isDefined => e.throwable.get } ) }