Merge pull request #4729 from eatkins/metaspace

Classloader caching

.travis.yml

@@ -54,7 +54,7 @@ install:
 
 script:
   # It doesn't need that much memory because compile and run are forked
-  - sbt -Dsbt.version=1.2.6 -Dsbt.ci=true -J-XX:ReservedCodeCacheSize=128m -J-Xmx800M -J-Xms800M -J-server "$SBT_CMD"
+  - sbt -Dsbt.ci=true -J-XX:ReservedCodeCacheSize=128m -J-Xmx800M -J-Xms800M -J-server "$SBT_CMD"
 
 before_cache:
   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete

build.sbt

@@ -656,6 +656,7 @@ lazy val mainProj = (project in file("main"))
     sourceManaged in (Compile, generateContrabands) := baseDirectory.value / "src" / "main" / "contraband-scala",
     testOptions in Test += Tests
       .Argument(TestFrameworks.ScalaCheck, "-minSuccessfulTests", "1000"),
+    Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat, // Delete this after 1.3.0-RC2.
     mimaSettings,
     mimaBinaryIssueFilters ++= Vector(
       // New and changed methods on KeyIndex. internal.
@@ -895,6 +896,7 @@ def otherRootSettings =
     scripted := scriptedTask.evaluated,
     scriptedUnpublished := scriptedUnpublishedTask.evaluated,
     scriptedSource := (sourceDirectory in sbtProj).value / "sbt-test",
+    watchTriggers in scripted += scriptedSource.value.toGlob / **,
     scriptedLaunchOpts := List("-Xmx1500M", "-Xms512M", "-server"),
     publishAll := { val _ = (publishLocal).all(ScopeFilter(inAnyProject)).value },
     publishLocalBinAll := { val _ = (publishLocalBin).all(ScopeFilter(inAnyProject)).value },

main-command/src/main/scala/sbt/BasicKeys.scala

@@ -9,7 +9,8 @@ package sbt
 
 import java.io.File
 
-import sbt.internal.inc.classpath.ClassLoaderCache
+import sbt.internal.inc.classpath.{ ClassLoaderCache => IncClassLoaderCache }
+import sbt.internal.classpath.ClassLoaderCache
 import sbt.internal.server.ServerHandler
 import sbt.internal.util.AttributeKey
 import sbt.librarymanagement.ModuleID
@@ -82,11 +83,16 @@ object BasicKeys {
     "True if commands are currently being entered from an interactive environment.",
     10
   )
-  private[sbt] val classLoaderCache = AttributeKey[ClassLoaderCache](
+  private[sbt] val classLoaderCache = AttributeKey[IncClassLoaderCache](
     "class-loader-cache",
     "Caches class loaders based on the classpath entries and last modified times.",
     10
   )
+  private[sbt] val extendedClassLoaderCache = AttributeKey[ClassLoaderCache](
+    "extended-class-loader-cache",
+    "Caches class loaders based on the classpath entries and last modified times.",
+    10
+  )
   private[sbt] val OnFailureStack = AttributeKey[List[Option[Exec]]](
     "on-failure-stack",
     "Stack that remembers on-failure handlers.",

main-command/src/main/scala/sbt/State.scala

@@ -9,6 +9,9 @@ package sbt
 
 import java.io.File
 import java.util.concurrent.Callable
+
+import sbt.internal.classpath.ClassLoaderCache
+import sbt.internal.inc.classpath.{ ClassLoaderCache => IncClassLoaderCache }
 import sbt.util.Logger
 import sbt.internal.util.{
   AttributeKey,
@@ -19,7 +22,6 @@ import sbt.internal.util.{
   GlobalLogging
 }
 import sbt.internal.util.complete.{ HistoryCommands, Parser }
-import sbt.internal.inc.classpath.ClassLoaderCache
 
 /**
  * Data structure representing all command execution information.
@@ -193,7 +195,7 @@ trait StateOps extends Any {
   def setInteractive(flag: Boolean): State
 
   /** Get the class loader cache for the application.*/
-  def classLoaderCache: ClassLoaderCache
+  def classLoaderCache: IncClassLoaderCache
 
   /** Create and register a class loader cache.  This should be called once at the application entry-point.*/
   def initializeClassLoaderCache: State
@@ -221,6 +223,7 @@ object State {
 
   /**
    * Provides a list of recently executed commands.  The commands are stored as processed instead of as entered by the user.
+   *
    * @param executed the list of the most recently executed commands, with the most recent command first.
    * @param maxSize the maximum number of commands to keep, or 0 to keep an unlimited number.
    */
@@ -334,11 +337,18 @@ object State {
     def interactive = getBoolean(s, BasicKeys.interactive, false)
     def setInteractive(i: Boolean) = s.put(BasicKeys.interactive, i)
 
-    def classLoaderCache: ClassLoaderCache =
+    def classLoaderCache: IncClassLoaderCache =
-      s get BasicKeys.classLoaderCache getOrElse newClassLoaderCache
+      s get BasicKeys.classLoaderCache getOrElse (throw new IllegalStateException(
-    def initializeClassLoaderCache = s.put(BasicKeys.classLoaderCache, newClassLoaderCache)
+        "Tried to get classloader cache for uninitialized state."
+      ))
+    def initializeClassLoaderCache: State = {
+      s.get(BasicKeys.extendedClassLoaderCache).foreach(_.close())
+      val cache = newClassLoaderCache
+      s.put(BasicKeys.extendedClassLoaderCache, cache)
+        .put(BasicKeys.classLoaderCache, new IncClassLoaderCache(cache))
+    }
     private[this] def newClassLoaderCache =
-      new ClassLoaderCache(s.configuration.provider.scalaProvider.launcher.topLoader)
+      new ClassLoaderCache(s.configuration.provider.scalaProvider)
   }
 
   import ExceptionCategory._

main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala

@@ -0,0 +1,252 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.internal.classpath
+
+import java.io.File
+import java.lang.management.ManagementFactory
+import java.lang.ref.{ Reference, ReferenceQueue, SoftReference }
+import java.net.{ URL, URLClassLoader }
+import java.util.concurrent.atomic.AtomicInteger
+
+import sbt.internal.inc.classpath.{
+  AbstractClassLoaderCache,
+  ClassLoaderCache => IncClassLoaderCache
+}
+import sbt.internal.inc.{ AnalyzingCompiler, ZincUtil }
+import sbt.io.IO
+import xsbti.ScalaProvider
+import xsbti.compile.{ ClasspathOptions, ScalaInstance }
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+private object ClassLoaderCache {
+  private def threadID = new AtomicInteger(0)
+}
+private[sbt] class ClassLoaderCache(
+    override val commonParent: ClassLoader,
+    private val miniProvider: Option[(File, ClassLoader)]
+) extends AbstractClassLoaderCache {
+  def this(commonParent: ClassLoader) = this(commonParent, None)
+  def this(scalaProvider: ScalaProvider) =
+    this(scalaProvider.launcher.topLoader, {
+      scalaProvider.jars.find(_.getName == "scala-library.jar").flatMap { lib =>
+        val clazz = scalaProvider.getClass
+        try {
+          val loader = clazz.getDeclaredMethod("libraryLoaderOnly").invoke(scalaProvider)
+          Some(lib -> loader.asInstanceOf[ClassLoader])
+        } catch { case NonFatal(_) => None }
+      }
+    })
+  private val scalaProviderKey = miniProvider.map {
+    case (f, cl) =>
+      new Key((f -> IO.getModifiedTimeOrZero(f)) :: Nil, commonParent) {
+        override def toClassLoader: ClassLoader = cl
+      }
+  }
+  private class Key(val fileStamps: Seq[(File, Long)], val parent: ClassLoader) {
+    def this(files: List[File]) =
+      this(files.map(f => f -> IO.getModifiedTimeOrZero(f)), commonParent)
+    lazy val files: Seq[File] = fileStamps.map(_._1)
+    lazy val maxStamp: Long = fileStamps.maxBy(_._2)._2
+    class CachedClassLoader
+        extends URLClassLoader(fileStamps.map(_._1.toURI.toURL).toArray, parent) {
+      override def toString: String =
+        s"CachedClassloader {\n  parent: $parent\n  urls:\n" + getURLs.mkString("    ", "\n", "\n}")
+    }
+    def toClassLoader: ClassLoader = new CachedClassLoader
+    override def equals(o: Any): Boolean = o match {
+      case that: Key => this.fileStamps == that.fileStamps && this.parent == that.parent
+    }
+    override def hashCode(): Int = (fileStamps.hashCode * 31) ^ parent.hashCode
+    override def toString: String = s"Key(${fileStamps mkString ","}, $parent)"
+  }
+  private[this] val delegate =
+    new java.util.concurrent.ConcurrentHashMap[Key, Reference[ClassLoader]]()
+  private[this] val referenceQueue = new ReferenceQueue[ClassLoader]
+
+  private[this] def closeExpiredLoaders(): Unit = {
+    val toClose = lock.synchronized(delegate.asScala.groupBy(_._1.files.toSet).flatMap {
+      case (_, pairs) if pairs.size > 1 =>
+        val max = pairs.maxBy(_._1.maxStamp)._1
+        pairs.filterNot(_._1 == max).flatMap {
+          case (k, v) =>
+            delegate.remove(k)
+            Option(v.get)
+        }
+      case _ => Nil
+    })
+    toClose.foreach(close)
+  }
+  private[this] class CleanupThread(private[this] val id: Int)
+      extends Thread(s"classloader-cache-cleanup-$id") {
+    setDaemon(true)
+    start()
+    @tailrec
+    override final def run(): Unit = {
+      val stop = try {
+        referenceQueue.remove(1000) match {
+          case ClassLoaderReference(key, classLoader) =>
+            close(classLoader)
+            delegate.remove(key)
+          case _ =>
+        }
+        closeExpiredLoaders()
+        false
+      } catch {
+        case _: InterruptedException => true
+      }
+      if (!stop) run()
+    }
+  }
+
+  /*
+   * We need to manage the cache differently depending on whether or not sbt is started up with
+   * -XX:MaxMetaspaceSize=XXX. The reason is that when the metaspace limit is reached, the jvm
+   * will run a few Full GCs that will clear SoftReferences so that it can cleanup any classes
+   * that only softly reachable. If the GC during this phase is able to collect a classloader, it
+   * will free the metaspace (or at least some of it) previously occupied by the loader. This can
+   * prevent sbt from crashing with an OOM: Metaspace. The issue with this is that when a loader
+   * is collected in this way, it will leak handles to its url classpath. To prevent the resource
+   * leak, we can store a reference to a wrapper loader. That reference, in turn, holds a
+   * strong reference to the underlying loader. Under heap memory pressure, the jvm will clear the
+   * soft reference for the wrapped loader and add it to the reference queue. We add a thread
+   * that reads from the reference queue and closes the underlying URLClassLoader, preventing the
+   * resource leak. When the system is under heap memory pressure, this eviction approach works
+   * well. The problem is that we cannot prevent OOM: MetaSpace because the jvm doesn't give us
+   * a long enough window to clear the ClassLoader references. The wrapper class will get cleared
+   * during the Metaspace Full GC window, but, even though we quickly clear the strong reference
+   * to the underlying classloader and close it, the jvm gives up and crashes with an OOM.
+   *
+   * To avoid these crashes, if the user starts with a limit on metaspace size via
+   * -XX:MetaSpaceSize=XXX, we will just store direct soft references to the URLClassLoader and
+   * leak url classpath handles when loaders are evicted by garbage collection. This is consistent
+   * with the behavior of sbt versions < 1.3.0. In general, these leaks are probably not a big deal
+   * except on windows where they prevent any files for which the leaked class loader has an open
+   * handle from being modified. On linux and mac, we probably leak some file descriptors but it's
+   * fairly uncommon for sbt to run out of file descriptors.
+   *
+   */
+  private[this] val metaspaceIsLimited =
+    ManagementFactory.getMemoryPoolMXBeans.asScala
+      .exists(b => (b.getName == "Metaspace") && (b.getUsage.getMax > 0))
+  private[this] val mkReference: (Key, ClassLoader) => Reference[ClassLoader] =
+    if (metaspaceIsLimited)(_, cl) => new SoftReference(cl, referenceQueue)
+    else ClassLoaderReference.apply
+  private[this] val cleanupThread = new CleanupThread(ClassLoaderCache.threadID.getAndIncrement())
+  private[this] val lock = new Object
+
+  private class WrappedLoader(parent: ClassLoader) extends URLClassLoader(Array.empty, parent) {
+    // This is to make dotty work which extracts the URLs from the loader
+    override def getURLs: Array[URL] = parent match {
+      case u: URLClassLoader => u.getURLs
+      case _                 => Array.empty
+    }
+    override def toString: String = s"WrappedLoader($parent)"
+  }
+  private def close(classLoader: ClassLoader): Unit = classLoader match {
+    case a: AutoCloseable => a.close()
+    case _                =>
+  }
+  private case class ClassLoaderReference(key: Key, classLoader: ClassLoader)
+      extends SoftReference[ClassLoader](
+        new WrappedLoader(classLoader),
+        referenceQueue
+      )
+  def apply(
+      files: List[(File, Long)],
+      parent: ClassLoader,
+      mkLoader: () => ClassLoader
+  ): ClassLoader = {
+    val key = new Key(files, parent)
+    get(key, mkLoader)
+  }
+  override def apply(files: List[File]): ClassLoader = {
+    val key = new Key(files)
+    get(key, () => key.toClassLoader)
+  }
+  override def cachedCustomClassloader(
+      files: List[File],
+      mkLoader: () => ClassLoader
+  ): ClassLoader = {
+    val key = new Key(files)
+    get(key, mkLoader)
+  }
+  private[this] def get(key: Key, f: () => ClassLoader): ClassLoader = {
+    scalaProviderKey match {
+      case Some(k) if k == key => k.toClassLoader
+      case _ =>
+        def addLoader(): ClassLoader = {
+          val ref = mkReference(key, f())
+          val loader = ref.get
+          delegate.put(key, ref)
+          closeExpiredLoaders()
+          loader
+        }
+        lock.synchronized {
+          delegate.get(key) match {
+            case null => addLoader()
+            case ref =>
+              ref.get match {
+                case null => addLoader()
+                case l    => l
+              }
+          }
+        }
+    }
+  }
+  private def clear(lock: Object): Unit = {
+    delegate.forEach {
+      case (_, ClassLoaderReference(_, classLoader)) => close(classLoader)
+      case (_, r: Reference[ClassLoader]) =>
+        r.get match {
+          case null        =>
+          case classLoader => close(classLoader)
+        }
+      case (_, _) =>
+    }
+    delegate.clear()
+  }
+
+  /**
+   * Clears any ClassLoader instances from the internal cache and closes them. Calling this
+   * method will not stop the cleanup thread. Call [[close]] to fully clean up this cache.
+   */
+  def clear(): Unit = lock.synchronized(clear(lock))
+
+  /**
+   * Completely shuts down this cache. It stops the background thread for cleaning up classloaders
+   *
+   * Clears any ClassLoader instances from the internal cache and closes them. It also
+   * method will not stop the cleanup thread. Call [[close]] to fully clean up this cache.
+   */
+  override def close(): Unit = lock.synchronized {
+    cleanupThread.interrupt()
+    cleanupThread.join()
+    clear(lock)
+  }
+}
+
+private[sbt] object AlternativeZincUtil {
+  def scalaCompiler(
+      scalaInstance: ScalaInstance,
+      compilerBridgeJar: File,
+      classpathOptions: ClasspathOptions,
+      classLoaderCache: Option[IncClassLoaderCache]
+  ): AnalyzingCompiler = {
+    val bridgeProvider = ZincUtil.constantBridgeProvider(scalaInstance, compilerBridgeJar)
+    new AnalyzingCompiler(
+      scalaInstance,
+      bridgeProvider,
+      classpathOptions,
+      _ => (),
+      classLoaderCache
+    )
+  }
+}

main-command/src/test/scala/sbt/internal/ClassLoaderCacheTest.scala

@@ -11,56 +11,43 @@ import java.io.File
 import java.nio.file.Files
 
 import org.scalatest.{ FlatSpec, Matchers }
+import sbt.internal.classpath.ClassLoaderCache
 import sbt.io.IO
 
 object ClassLoaderCacheTest {
-  private val initLoader = this.getClass.getClassLoader
   implicit class CacheOps(val c: ClassLoaderCache) {
-    def get(classpath: Seq[File]): ClassLoader =
+    def get(classpath: Seq[File]): ClassLoader = c(classpath.toList)
-      c.get((classpath, initLoader, Map.empty, new File("/dev/null")))
   }
 }
 class ClassLoaderCacheTest extends FlatSpec with Matchers {
   import ClassLoaderCacheTest._
-  def withCache[R](size: Int)(f: CacheOps => R): R = {
+  private def withCache[R](f: ClassLoaderCache => R): R = {
-    val cache = ClassLoaderCache(size)
+    val cache = new ClassLoaderCache(ClassLoader.getSystemClassLoader)
-    try f(new CacheOps(cache))
+    try f(cache)
     finally cache.close()
   }
-  "ClassLoaderCache.get" should "make a new loader when full" in withCache(0) { cache =>
+  "ClassLoaderCache" should "make a new loader when full" in withCache { cache =>
     val classPath = Seq.empty[File]
     val firstLoader = cache.get(classPath)
+    cache.clear()
     val secondLoader = cache.get(classPath)
     assert(firstLoader != secondLoader)
   }
-  it should "not make a new loader when it already exists" in withCache(1) { cache =>
+  it should "not make a new loader when it already exists" in withCache { cache =>
     val classPath = Seq.empty[File]
     val firstLoader = cache.get(classPath)
     val secondLoader = cache.get(classPath)
     assert(firstLoader == secondLoader)
   }
-  it should "evict loaders" in withCache(2) { cache =>
-    val firstClassPath = Seq.empty[File]
-    val secondClassPath = new File("foo") :: Nil
-    val thirdClassPath = new File("foo") :: new File("bar") :: Nil
-    val firstLoader = cache.get(firstClassPath)
-    val secondLoader = cache.get(secondClassPath)
-    val thirdLoader = cache.get(thirdClassPath)
-    assert(cache.get(thirdClassPath) == thirdLoader)
-    assert(cache.get(secondClassPath) == secondLoader)
-    assert(cache.get(firstClassPath) != firstLoader)
-    assert(cache.get(thirdClassPath) != thirdLoader)
-  }
   "Snapshots" should "be invalidated" in IO.withTemporaryDirectory { dir =>
     val snapshotJar = Files.createFile(dir.toPath.resolve("foo-SNAPSHOT.jar")).toFile
     val regularJar = Files.createFile(dir.toPath.resolve("regular.jar")).toFile
-    withCache(1) { cache =>
+    withCache { cache =>
       val jarClassPath = snapshotJar :: regularJar :: Nil
       val initLoader = cache.get(jarClassPath)
       IO.setModifiedTimeOrFalse(snapshotJar, System.currentTimeMillis + 5000L)
       val secondLoader = cache.get(jarClassPath)
       assert(initLoader != secondLoader)
-      assert(initLoader.getParent == secondLoader.getParent)
       assert(cache.get(jarClassPath) == secondLoader)
       assert(cache.get(jarClassPath) != initLoader)
     }

main/src/main/scala/sbt/ClassLoaderLayeringStrategy.scala

@@ -89,36 +89,12 @@ object ClassLoaderLayeringStrategy {
   case object ScalaLibrary extends ScalaLibrary
 
   /**
-   * Add a layer on top of the ScalaLibrary layer for the runtime jar dependencies.
+   * Add a layer on top of the ScalaLibrary layer for all of the task jar dependencies.
    */
-  sealed trait RuntimeDependencies extends ScalaLibrary
+  sealed trait AllLibraryJars extends ScalaLibrary
 
   /**
-   * Add a layer on top of the ScalaLibrary layer for the runtime jar dependencies.
+   * Add a layer on top of the ScalaLibrary layer for all of the jar dependencies.
    */
-  case object RuntimeDependencies extends ScalaLibrary with RuntimeDependencies
+  object AllLibraryJars extends AllLibraryJars
-
-  /**
-   * Add a layer on top of the ScalaLibrary layer for the test jar dependencies.
-   */
-  sealed trait TestDependencies extends ScalaLibrary
-
-  /**
-   * Add a layer on top of the ScalaInstance layer for the test jar dependencies.
-   */
-  case object TestDependencies extends ScalaLibrary with TestDependencies
-
-  /**
-   * Add the TestDependencies layer on top of the RuntimeDependencies layer on top of the
-   * ScalaLibrary layer. This differs from TestDependencies in that it will not reload the
-   * runtime classpath. The drawback to using this is that if the test dependencies evict
-   * classes provided in the runtime layer, then tests can fail. In order for sharing the runtime
-   * layer to work, it is necessary to set [[Keys.bgCopyClasspath]] to false. Otherwise the
-   * runtime and test classpaths are completely different.
-   */
-  case object ShareRuntimeDependenciesLayerWithTestDependencies
-      extends ScalaLibrary
-      with RuntimeDependencies
-      with TestDependencies
-
 }

main/src/main/scala/sbt/Defaults.scala

@@ -28,6 +28,7 @@ import sbt.Project.{
 import sbt.Scope.{ GlobalScope, ThisScope, fillTaskAxis }
 import sbt.internal.CommandStrings.ExportStream
 import sbt.internal._
+import sbt.internal.classpath.AlternativeZincUtil
 import sbt.internal.inc.JavaInterfaceUtil._
 import sbt.internal.inc.classpath.ClasspathFilter
 import sbt.internal.inc.{ ZincLmUtil, ZincUtil }
@@ -148,7 +149,6 @@ object Defaults extends BuildCommon {
     defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
       excludeFilter :== HiddenFileFilter,
       pathToFileStamp :== sbt.nio.FileStamp.hash,
-      classLoaderCache := ClassLoaderCache(4),
       fileInputs :== Nil,
       inputFileStamper :== sbt.nio.FileStamper.Hash,
       outputFileStamper :== sbt.nio.FileStamper.LastModified,
@@ -162,15 +162,13 @@ object Defaults extends BuildCommon {
           .get(sbt.nio.Keys.persistentFileStampCache)
           .getOrElse(new sbt.nio.FileStamp.Cache)
       },
-    ) ++ TaskRepository
+    ) ++ globalIvyCore ++ globalJvmCore
-      .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore
   ) ++ globalSbtCore
 
   private[sbt] lazy val globalJvmCore: Seq[Setting[_]] =
     Seq(
       compilerCache := state.value get Keys.stateCompilerCache getOrElse CompilerCache.fresh,
-      classLoaderLayeringStrategy :== ClassLoaderLayeringStrategy.RuntimeDependencies,
+      classLoaderLayeringStrategy :== ClassLoaderLayeringStrategy.AllLibraryJars,
-      classLoaderLayeringStrategy in Test :== ClassLoaderLayeringStrategy.TestDependencies,
       sourcesInBase :== true,
       autoAPIMappings := false,
       apiMappings := Map.empty,
@@ -550,10 +548,11 @@ object Defaults extends BuildCommon {
       val scalac =
         scalaCompilerBridgeBinaryJar.value match {
           case Some(jar) =>
-            ZincUtil.scalaCompiler(
+            AlternativeZincUtil.scalaCompiler(
               scalaInstance = scalaInstance.value,
               classpathOptions = classpathOptions.value,
-              compilerBridgeJar = jar
+              compilerBridgeJar = jar,
+              classLoaderCache = st.get(BasicKeys.classLoaderCache)
             )
           case _ =>
             ZincLmUtil.scalaCompiler(
@@ -565,6 +564,7 @@ object Defaults extends BuildCommon {
               dependencyResolution = dr,
               compilerBridgeSource = scalaCompilerBridgeSource.value,
               scalaJarsTarget = zincDir,
+              classLoaderCache = st.get(BasicKeys.classLoaderCache),
               log = streams.value.log
             )
         }
@@ -594,7 +594,7 @@ object Defaults extends BuildCommon {
     compileInputsSettings
   ) ++ configGlobal ++ defaultCompileSettings ++ compileAnalysisSettings ++ Seq(
     clean := Clean.task(ThisScope, full = false).value,
-    fileOutputs := Seq(Glob(classDirectory.value, RecursiveGlob / "*.class")),
+    fileOutputs in compile := Seq(Glob(classDirectory.value, RecursiveGlob / "*.class")),
     compile := compileTask.value,
     internalDependencyConfigurations := InternalDependencies.configurations.value,
     manipulateBytecode := compileIncremental.value,
@@ -710,8 +710,16 @@ object Defaults extends BuildCommon {
         val scalaProvider = appConfiguration.value.provider.scalaProvider
         val version = scalaVersion.value
         if (version == scalaProvider.version) // use the same class loader as the Scala classes used by sbt
-          Def.task(ScalaInstance(version, scalaProvider))
+          Def.task {
-        else
+            val allJars = scalaProvider.jars
+            val libraryJars = allJars.filter(_.getName == "scala-library.jar")
+            allJars.filter(_.getName == "scala-compiler.jar") match {
+              case Array(compilerJar) if libraryJars.nonEmpty =>
+                val cache = state.value.classLoaderCache
+                mkScalaInstance(version, allJars, libraryJars, compilerJar, cache)
+              case _ => ScalaInstance(version, scalaProvider)
+            }
+          } else
           scalaInstanceFromUpdate
     }
   }
@@ -745,20 +753,51 @@ object Defaults extends BuildCommon {
     val allJars = toolReport.modules.flatMap(_.artifacts.map(_._2))
     val libraryJar = file(ScalaArtifacts.LibraryID)
     val compilerJar = file(ScalaArtifacts.CompilerID)
-    new ScalaInstance(
+    mkScalaInstance(
       scalaVersion.value,
-      makeClassLoader(state.value)(allJars.toList),
+      allJars,
-      makeClassLoader(state.value)(List(libraryJar)),
+      Array(libraryJar),
-      libraryJar,
+      compilerJar,
+      state.value.classLoaderCache
+    )
+  }
+  private[this] def mkScalaInstance(
+      version: String,
+      allJars: Seq[File],
+      libraryJars: Array[File],
+      compilerJar: File,
+      classLoaderCache: sbt.internal.inc.classpath.ClassLoaderCache
+  ): ScalaInstance = {
+    val libraryLoader = classLoaderCache(libraryJars.toList)
+    class ScalaLoader extends URLClassLoader(allJars.map(_.toURI.toURL).toArray, libraryLoader)
+    val fullLoader = classLoaderCache.cachedCustomClassloader(
+      allJars.toList,
+      () => new ScalaLoader
+    )
+    new ScalaInstance(
+      version,
+      fullLoader,
+      libraryLoader,
+      libraryJars,
       compilerJar,
       allJars.toArray,
-      None
+      Some(version)
     )
   }
   def scalaInstanceFromHome(dir: File): Initialize[Task[ScalaInstance]] = Def.task {
-    ScalaInstance(dir)(makeClassLoader(state.value))
+    val dummy = ScalaInstance(dir)(state.value.classLoaderCache.apply)
+    Seq(dummy.loader, dummy.loaderLibraryOnly).foreach {
+      case a: AutoCloseable => a.close()
+      case cl               =>
+    }
+    mkScalaInstance(
+      dummy.version,
+      dummy.allJars,
+      dummy.libraryJars,
+      dummy.compilerJar,
+      state.value.classLoaderCache
+    )
   }
-  private[this] def makeClassLoader(state: State) = state.classLoaderCache.apply _
 
   private[this] def testDefaults =
     Defaults.globalDefaults(
@@ -1019,7 +1058,7 @@ object Defaults extends BuildCommon {
       cp,
       forkedParallelExecution = false,
       javaOptions = Nil,
-      strategy = ClassLoaderLayeringStrategy.TestDependencies,
+      strategy = ClassLoaderLayeringStrategy.AllLibraryJars,
       projectId = "",
     )
   }
@@ -1042,7 +1081,7 @@ object Defaults extends BuildCommon {
       cp,
       forkedParallelExecution,
       javaOptions = Nil,
-      strategy = ClassLoaderLayeringStrategy.TestDependencies,
+      strategy = ClassLoaderLayeringStrategy.AllLibraryJars,
       projectId = "",
     )
   }
@@ -1823,34 +1862,9 @@ object Defaults extends BuildCommon {
       Classpaths.compilerPluginConfig ++ deprecationSettings
 
   lazy val compileSettings: Seq[Setting[_]] =
-    configSettings ++ (mainBgRunMainTask +: mainBgRunTask) ++
+    configSettings ++ (mainBgRunMainTask +: mainBgRunTask) ++ Classpaths.addUnmanagedLibrary
-      Classpaths.addUnmanagedLibrary ++
-      Vector(
-        TaskRepository.proxy(
-          Compile / classLoaderCache,
-          // We need a cache of size four so that the subset of the runtime dependencies that are used
-          // by the test task layers may be cached without evicting the runtime classloader layers. The
-          // cache size should be a multiple of two to support snapshot layers.
-          ClassLoaderCache(4)
-        ),
-        bgCopyClasspath in bgRun := {
-          val old = (bgCopyClasspath in bgRun).value
-          old && (Test / classLoaderLayeringStrategy).value != ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
-        },
-        bgCopyClasspath in bgRunMain := {
-          val old = (bgCopyClasspath in bgRunMain).value
-          old && (Test / classLoaderLayeringStrategy).value != ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
-        },
-      )
 
-  lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++
+  lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks
-    Vector(
-      TaskRepository.proxy(
-        Test / classLoaderCache,
-        // We need a cache of size two for the test dependency layers (regular and snapshot).
-        ClassLoaderCache(2)
-      )
-    )
 
   lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest) {
     testSettings
@@ -1959,7 +1973,8 @@ object Classpaths {
         includeFilter in unmanagedJars value,
         excludeFilter in unmanagedJars value
       )
-    ).map(exportClasspath)
+    ).map(exportClasspath) :+
+      (sbt.nio.Keys.classpathFiles := data(fullClasspath.value).map(_.toPath))
 
   private[this] def exportClasspath(s: Setting[Task[Classpath]]): Setting[Task[Classpath]] =
     s.mapInitialize(init => Def.task { exportClasspath(streams.value, init.value) })

main/src/main/scala/sbt/Keys.scala

@@ -485,8 +485,6 @@ object Keys {
   val resolvedScoped = Def.resolvedScoped
   val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask)
   val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask)
-  private[sbt] val classLoaderCache = taskKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask)
-  private[sbt] val taskRepository = AttributeKey[TaskRepository.Repr]("task-repository", "A repository that can be used to cache arbitrary values for a given task key that can be read or filled during task evaluation.", 10000)
   private[sbt] val taskCancelStrategy = settingKey[State => TaskCancellationStrategy]("Experimental task cancellation handler.").withRank(DTask)
 
   // Experimental in sbt 0.13.2 to enable grabbing semantic compile failures.

main/src/main/scala/sbt/Main.scala

@@ -8,7 +8,6 @@
 package sbt
 
 import java.io.{ File, IOException }
-import java.lang.reflect.InvocationTargetException
 import java.net.URI
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.{ Locale, Properties }
@@ -27,7 +26,6 @@ import sbt.io._
 import sbt.io.syntax._
 import sbt.util.{ Level, Logger, Show }
 import xsbti.compile.CompilerCache
-import xsbti.{ AppMain, AppProvider, ComponentProvider, Launcher, ScalaProvider }
 
 import scala.annotation.tailrec
 import scala.concurrent.ExecutionContext
@@ -35,67 +33,8 @@ import scala.util.control.NonFatal
 
 /** This class is the entry point for sbt. */
 final class xMain extends xsbti.AppMain {
-  def run(configuration: xsbti.AppConfiguration): xsbti.MainResult = {
+  def run(configuration: xsbti.AppConfiguration): xsbti.MainResult =
-    val modifiedConfiguration = new ModifiedConfiguration(configuration)
+    new XMainConfiguration().runXMain(configuration)
-    val loader = modifiedConfiguration.provider.loader
-    // No need to memoize the old class loader. It is reset by the launcher anyway.
-    Thread.currentThread.setContextClassLoader(loader)
-    val clazz = loader.loadClass("sbt.xMainImpl$")
-    val instance = clazz.getField("MODULE$").get(null)
-    val runMethod = clazz.getMethod("run", classOf[xsbti.AppConfiguration])
-    try {
-      new Thread("sbt-load-global-instance") {
-        setDaemon(true)
-        override def run(): Unit = {
-          // This preloads the scala.tools.nsc.Global as a performance optimization"
-          loader.loadClass("sbt.internal.parser.SbtParser$").getField("MODULE$").get(null)
-          ()
-        }
-      }.start()
-      runMethod.invoke(instance, modifiedConfiguration).asInstanceOf[xsbti.MainResult]
-    } catch {
-      case e: InvocationTargetException =>
-        // This propogates xsbti.FullReload to the launcher
-        throw e.getCause
-    } finally {
-      loader match {
-        case a: AutoCloseable => a.close()
-        case _                =>
-      }
-    }
-  }
-  /*
-   * Replaces the AppProvider.loader method with a new loader that puts the sbt test interface
-   * jar ahead of the rest of the sbt classpath in the classloading hierarchy.
-   */
-  private class ModifiedConfiguration(val configuration: xsbti.AppConfiguration)
-      extends xsbti.AppConfiguration {
-    private[this] val metaLoader: ClassLoader = SbtMetaBuildClassLoader(configuration.provider)
-
-    private class ModifiedAppProvider(val appProvider: AppProvider) extends AppProvider {
-      override def scalaProvider(): ScalaProvider = new ScalaProvider {
-        val delegate = configuration.provider.scalaProvider
-        override def launcher(): Launcher = delegate.launcher
-        override def version(): String = delegate.version
-        override def loader(): ClassLoader = metaLoader.getParent
-        override def jars(): Array[File] = delegate.jars
-        override def libraryJar(): File = delegate.libraryJar
-        override def compilerJar(): File = delegate.compilerJar
-        override def app(id: xsbti.ApplicationID): AppProvider = delegate.app(id)
-      }
-      override def id(): xsbti.ApplicationID = appProvider.id()
-      override def loader(): ClassLoader = metaLoader
-      @deprecated("Implements deprecated api", "1.3.0")
-      override def mainClass(): Class[_ <: AppMain] = appProvider.mainClass()
-      override def entryPoint(): Class[_] = appProvider.entryPoint()
-      override def newMain(): AppMain = appProvider.newMain()
-      override def mainClasspath(): Array[File] = appProvider.mainClasspath()
-      override def components(): ComponentProvider = appProvider.components()
-    }
-    override def arguments(): Array[String] = configuration.arguments
-    override def baseDirectory(): File = configuration.baseDirectory
-    override def provider(): AppProvider = new ModifiedAppProvider(configuration.provider)
-  }
 }
 private[sbt] object xMainImpl {
   private[sbt] def run(configuration: xsbti.AppConfiguration): xsbti.MainResult =
@@ -889,7 +828,8 @@ object BuiltinCommands {
 
     val session = Load.initialSession(structure, eval, s0)
     SessionSettings.checkSession(session, s)
-    registerGlobalCaches(Project.setProject(session, structure, s))
+    Project
+      .setProject(session, structure, s)
       .put(sbt.nio.Keys.hasCheckedMetaBuild, new AtomicBoolean(false))
   }
 
@@ -909,23 +849,11 @@ object BuiltinCommands {
       }
     s.put(Keys.stateCompilerCache, cache)
   }
-  private[sbt] def registerGlobalCaches(s: State): State =
-    try {
-      val cleanedUp = new AtomicBoolean(false)
-      def cleanup(): Unit = {
-        s.get(Keys.taskRepository).foreach(_.close())
-        ()
-      }
-      cleanup()
-      s.addExitHook(if (cleanedUp.compareAndSet(false, true)) cleanup())
-        .put(Keys.taskRepository, new TaskRepository.Repr)
-    } catch {
-      case NonFatal(_) => s
-    }
 
   def clearCaches: Command = {
     val help = Help.more(ClearCaches, ClearCachesDetailed)
-    Command.command(ClearCaches, help)(registerGlobalCaches _ andThen registerCompilerCache)
+    val f: State => State = registerCompilerCache _ andThen (_.initializeClassLoaderCache)
+    Command.command(ClearCaches, help)(f)
   }
 
   def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 =>

main/src/main/scala/sbt/MainLoop.scala

@@ -135,14 +135,44 @@ object MainLoop {
     }
 
   def next(state: State): State =
-    ErrorHandling.wideConvert { state.process(processCommand) } match {
+    try {
-      case Right(s)                  => s
+      ErrorHandling.wideConvert {
-      case Left(t: xsbti.FullReload) => throw t
+        state.process(processCommand)
-      case Left(t: RebootCurrent)    => throw t
+      } match {
-      case Left(Reload) =>
+        case Right(s)                  => s
-        val remaining = state.currentCommand.toList ::: state.remainingCommands
+        case Left(t: xsbti.FullReload) => throw t
-        state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
+        case Left(t: RebootCurrent)    => throw t
-      case Left(t) => state.handleError(t)
+        case Left(Reload) =>
+          val remaining = state.currentCommand.toList ::: state.remainingCommands
+          state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
+        case Left(t) => state.handleError(t)
+      }
+    } catch {
+      case oom: OutOfMemoryError if oom.getMessage.contains("Metaspace") =>
+        System.gc() // Since we're under memory pressure, see if more can be freed with a manual gc.
+        val isTestOrRun = state.remainingCommands.headOption.exists { exec =>
+          val cmd = exec.commandLine
+          cmd.contains("test") || cmd.contains("run")
+        }
+        val isConsole = state.remainingCommands.exists(_.commandLine == "shell") ||
+          (state.remainingCommands.last.commandLine == "iflast shell")
+        val testOrRunMessage =
+          if (!isTestOrRun) ""
+          else
+            " If this error occurred during a test or run evaluation, it can be caused by the " +
+              "choice of ClassLoaderLayeringStrategy. Of the available strategies, " +
+              "ClassLoaderLayeringStrategy.ScalaLibrary will typically use the least metaspace. " +
+              (if (isConsole)
+                 " To change the layering strategy for this session, run:\n\n" +
+                   "set ThisBuild / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy." +
+                   "ScalaLibrary"
+               else "")
+        val msg: String =
+          s"Caught $oom\nTo best utilize classloader caching and to prevent file handle leaks, we" +
+            s"recommend running sbt without a MaxMetaspaceSize limit. $testOrRunMessage"
+        state.log.error(msg)
+        state.log.error("\n")
+        state.handleError(oom)
     }
 
   /** This is the main function State transfer function of the sbt command processing. */

main/src/main/scala/sbt/internal/ClassLoaderCache.scala

@@ -1,93 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt.internal
-
-import java.io.File
-import java.nio.file.Files
-
-import sbt.internal.util.TypeFunctions.Id
-
-import scala.annotation.tailrec
-
-private[sbt] sealed trait ClassLoaderCache
-    extends Repository[Id, (Seq[File], ClassLoader, Map[String, String], File), ClassLoader]
-
-private[sbt] object ClassLoaderCache {
-  private type Resources = Map[String, String]
-  private sealed trait CachedClassLoader extends ClassLoader {
-    def close(): Unit
-  }
-  private sealed trait StableClassLoader extends CachedClassLoader
-  private sealed trait SnapshotClassLoader extends CachedClassLoader
-  def apply(maxSize: Int): ClassLoaderCache = {
-    new ClassLoaderCache {
-      private final def mktmp(tmp: File): File =
-        if (maxSize > 0) Files.createTempDirectory("sbt-jni").toFile else tmp
-      private[this] val lruCache =
-        LRUCache[(JarClassPath, ClassLoader), (JarClassPath, CachedClassLoader)](
-          maxSize = maxSize,
-          onExpire =
-            (_: (JarClassPath, ClassLoader), v: (JarClassPath, CachedClassLoader)) => close(v._2)
-        )
-      override def get(info: (Seq[File], ClassLoader, Resources, File)): ClassLoader =
-        synchronized {
-          val (paths, parent, resources, tmp) = info
-          val key @ (keyJCP, _) = (new JarClassPath(paths), parent)
-          def addLoader(base: Option[StableClassLoader] = None): CachedClassLoader = {
-            val baseLoader = base.getOrElse {
-              if (keyJCP.regularJars.isEmpty) new ClassLoader(parent) with StableClassLoader {
-                override def close(): Unit = parent match {
-                  case s: StableClassLoader => s.close()
-                  case _                    => ()
-                }
-                override def toString: String = parent.toString
-              } else
-                new LayeredClassLoader(keyJCP.regularJars, parent, resources, mktmp(tmp))
-                with StableClassLoader
-            }
-            val loader: CachedClassLoader =
-              if (keyJCP.snapshotJars.isEmpty) baseLoader
-              else
-                new LayeredClassLoader(keyJCP.snapshotJars, baseLoader, resources, mktmp(tmp))
-                with SnapshotClassLoader
-            lruCache.put(key, keyJCP -> loader)
-            loader
-          }
-          lruCache.get(key) match {
-            case Some((jcp, cl)) if keyJCP.strictEquals(jcp) => cl
-            case Some((_, cl: SnapshotClassLoader)) =>
-              cl.close()
-              cl.getParent match {
-                case p: StableClassLoader => addLoader(Some(p))
-                case _                    => addLoader()
-              }
-            case None => addLoader()
-          }
-        }
-      override def close(): Unit = synchronized(lruCache.close())
-      override def toString: String = {
-        import PrettyPrint.indent
-        val cacheElements = lruCache.entries.map {
-          case ((jcp, parent), (_, l)) =>
-            s"(\n${indent(jcp, 4)},\n${indent(parent, 4)}\n) =>\n  $l"
-        }
-        s"ClassLoaderCache(\n  size = $maxSize,\n  elements =\n${indent(cacheElements, 4)}\n)"
-      }
-
-      // Close the ClassLoader and all of it's closeable parents.
-      @tailrec
-      private def close(loader: CachedClassLoader): Unit = {
-        loader.close()
-        loader.getParent match {
-          case c: CachedClassLoader => close(c)
-          case _                    => ()
-        }
-      }
-    }
-  }
-}

main/src/main/scala/sbt/internal/ClassLoaders.scala

@@ -9,18 +9,20 @@ package sbt
 package internal
 
 import java.io.File
-import java.net.{ URL, URLClassLoader }
+import java.net.URLClassLoader
+import java.nio.file.Path
 
 import sbt.ClassLoaderLayeringStrategy._
 import sbt.Keys._
-import sbt.SlashSyntax0._
+import sbt.internal.classpath.ClassLoaderCache
 import sbt.internal.inc.ScalaInstance
 import sbt.internal.inc.classpath.ClasspathUtilities
 import sbt.internal.util.Attributed
 import sbt.internal.util.Attributed.data
 import sbt.io.IO
-import sbt.librarymanagement.Configurations.{ Runtime, Test }
+import sbt.nio.FileStamp
-import xsbti.AppProvider
+import sbt.nio.FileStamp.LastModified
+import sbt.nio.Keys._
 
 private[sbt] object ClassLoaders {
   private[this] val interfaceLoader = classOf[sbt.testing.Framework].getClassLoader
@@ -29,20 +31,20 @@ private[sbt] object ClassLoaders {
    */
   private[sbt] def testTask: Def.Initialize[Task[ClassLoader]] = Def.task {
     val si = scalaInstance.value
-    val rawCP = data(fullClasspath.value)
+    val rawCP = modifiedTimes((outputFileStamps in classpathFiles).value)
-    val fullCP = if (si.isManagedVersion) rawCP else List(si.libraryJar) ++ rawCP
+    val fullCP =
-    val exclude = dependencyJars(exportedProducts).value.toSet ++ Set(si.libraryJar)
+      if (si.isManagedVersion) rawCP
+      else si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toSeq ++ rawCP
+    val exclude = dependencyJars(exportedProducts).value.toSet ++ si.libraryJars
+    val resourceCP = modifiedTimes((outputFileStamps in resources).value)
     buildLayers(
       strategy = classLoaderLayeringStrategy.value,
       si = si,
       fullCP = fullCP,
-      rawRuntimeDependencies =
+      resourceCP = resourceCP,
-        dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude),
       allDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude),
-      globalCache = (Scope.GlobalScope / classLoaderCache).value,
+      cache = extendedClassLoaderCache.value,
-      runtimeCache = (Runtime / classLoaderCache).value,
+      resources = ClasspathUtilities.createClasspathResources(fullCP.map(_._1), si),
-      testCache = (Test / classLoaderCache).value,
-      resources = ClasspathUtilities.createClasspathResources(fullCP, si),
       tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value),
       scope = resolvedScoped.value.scope
     )
@@ -54,6 +56,7 @@ private[sbt] object ClassLoaders {
     val s = streams.value
     val opts = forkOptions.value
     val options = javaOptions.value
+    val resourceCP = modifiedTimes((outputFileStamps in resources).value)
     if (fork.value) {
       s.log.debug(s"javaOptions: $options")
       Def.task(new ForkRun(opts))
@@ -73,23 +76,17 @@ private[sbt] object ClassLoaders {
           )
           s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
         }
-        val globalCache = (Scope.GlobalScope / classLoaderCache).value
-        val runtimeCache = (Runtime / classLoaderCache).value
-        val testCache = (Test / classLoaderCache).value
         val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars
-        val runtimeDeps = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
         val allDeps = dependencyJars(dependencyClasspath).value.filterNot(exclude)
         val newLoader =
           (classpath: Seq[File]) => {
             buildLayers(
               strategy = classLoaderLayeringStrategy.value: @sbtUnchecked,
               si = instance,
-              fullCP = classpath,
+              fullCP = classpath.map(f => f -> IO.getModifiedTimeOrZero(f)),
-              rawRuntimeDependencies = runtimeDeps,
+              resourceCP = resourceCP,
               allDependencies = allDeps,
-              globalCache = globalCache,
+              cache = extendedClassLoaderCache.value: @sbtUnchecked,
-              runtimeCache = runtimeCache,
-              testCache = testCache,
               resources = ClasspathUtilities.createClasspathResources(classpath, instance),
               tmp = taskTemporaryDirectory.value: @sbtUnchecked,
               scope = resolvedScope
@@ -100,77 +97,63 @@ private[sbt] object ClassLoaders {
     }
   }
 
+  private[this] def extendedClassLoaderCache: Def.Initialize[Task[ClassLoaderCache]] = Def.task {
+    val errorMessage = "Tried to extract classloader cache for uninitialized state."
+    state.value
+      .get(BasicKeys.extendedClassLoaderCache)
+      .getOrElse(throw new IllegalStateException(errorMessage))
+  }
   /*
    * Create a layered classloader. There are up to four layers:
    * 1) the scala instance class loader
-   * 2) the runtime dependencies
+   * 2) the resource layer
-   * 3) the test dependencies
+   * 3) the dependency jars
    * 4) the rest of the classpath
-   * The first two layers may be optionally cached to reduce memory usage and improve
+   * The first three layers may be optionally cached to reduce memory usage and improve
    * start up latency. Because there may be mutually incompatible libraries in the runtime
    * and test dependencies, it's important to be able to configure which layers are used.
    */
   private def buildLayers(
       strategy: ClassLoaderLayeringStrategy,
       si: ScalaInstance,
-      fullCP: Seq[File],
+      fullCP: Seq[(File, Long)],
-      rawRuntimeDependencies: Seq[File],
+      resourceCP: Seq[(File, Long)],
       allDependencies: Seq[File],
-      globalCache: ClassLoaderCache,
+      cache: ClassLoaderCache,
-      runtimeCache: ClassLoaderCache,
-      testCache: ClassLoaderCache,
       resources: Map[String, String],
       tmp: File,
       scope: Scope
   ): ClassLoader = {
-    val isTest = scope.config.toOption.map(_.name) == Option("test")
+    val cpFiles = fullCP.map(_._1)
     val raw = strategy match {
-      case Flat => flatLoader(fullCP, interfaceLoader)
+      case Flat => flatLoader(cpFiles, interfaceLoader)
       case _ =>
-        val (layerDependencies, layerTestDependencies) = strategy match {
+        val layerDependencies = strategy match {
-          case ShareRuntimeDependenciesLayerWithTestDependencies if isTest => (true, true)
+          case _: AllLibraryJars => true
-          case ScalaLibrary                                                => (false, false)
+          case _                 => false
-          case RuntimeDependencies                                         => (true, false)
-          case TestDependencies if isTest                                  => (false, true)
-          case badStrategy =>
-            val msg = s"Layering strategy $badStrategy is not valid for the classloader in " +
-              s"$scope. Valid options are: ClassLoaderLayeringStrategy.{ " +
-              "Flat, ScalaInstance, RuntimeDependencies }"
-            throw new IllegalArgumentException(msg)
         }
         val allDependenciesSet = allDependencies.toSet
-        // The raw declarations are to avoid having to make a dynamic task. The
+        val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp)
-        // allDependencies and allTestDependencies create a mutually exclusive list of jar
+        val cpFiles = fullCP.map(_._1)
-        // dependencies for layers 2 and 3. Note that in the Runtime or Compile configs, it
-        // should always be the case that allTestDependencies == Nil
-        val allTestDependencies = if (layerTestDependencies) allDependenciesSet else Set.empty[File]
-        val allRuntimeDependencies = (if (layerDependencies) rawRuntimeDependencies else Nil).toSet
 
-        val scalaLibrarySet = Set(si.libraryJar)
+        // layer 2 (resources)
-        val scalaLibraryLayer =
+        val resourceLayer =
-          globalCache.get((scalaLibrarySet.toList, interfaceLoader, resources, tmp))
-        // layer 2
-        val runtimeDependencySet = allDependenciesSet intersect allRuntimeDependencies
-        val runtimeDependencies = rawRuntimeDependencies.filter(runtimeDependencySet)
-        lazy val runtimeLayer =
           if (layerDependencies)
-            layer(runtimeDependencies, scalaLibraryLayer, runtimeCache, resources, tmp)
+            getResourceLayer(cpFiles, resourceCP, scalaLibraryLayer, cache, resources)
           else scalaLibraryLayer
 
-        // layer 3 (optional if testDependencies are empty)
+        // layer 3 (optional if in the test config and the runtime layer is not shared)
-        val testDependencySet = allTestDependencies diff runtimeDependencySet
+        val dependencyLayer =
-        val testDependencies = allDependencies.filter(testDependencySet)
+          if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp)
-        val testLayer = layer(testDependencies, runtimeLayer, testCache, resources, tmp)
+          else resourceLayer
 
         // layer 4
-        val dynamicClasspath =
+        val dynamicClasspath = cpFiles.filterNot(allDependenciesSet ++ si.libraryJars)
-          fullCP.filterNot(testDependencySet ++ runtimeDependencies ++ scalaLibrarySet)
+        new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp)
-        if (dynamicClasspath.nonEmpty)
-          new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp)
-        else testLayer
     }
-    ClasspathUtilities.filterByClasspath(fullCP, raw)
+    ClasspathUtilities.filterByClasspath(cpFiles, raw)
   }
+
   private def dependencyJars(
       key: sbt.TaskKey[Seq[Attributed[File]]]
   ): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))
@@ -187,39 +170,60 @@ private[sbt] object ClassLoaders {
       resources: Map[String, String],
       tmp: File
   ): ClassLoader = {
-    val (snapshots, jars) = classpath.partition(_.toString.contains("-SNAPSHOT"))
+    if (classpath.nonEmpty) {
-    val jarLoader = if (jars.isEmpty) parent else cache.get((jars, parent, resources, tmp))
+      cache(
-    if (snapshots.isEmpty) jarLoader else cache.get((snapshots, jarLoader, resources, tmp))
+        classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
+        parent,
+        () => new LayeredClassLoader(classpath, parent, resources, tmp)
+      )
+    } else parent
   }
 
+  private class ResourceLoader(
+      classpath: Seq[File],
+      parent: ClassLoader,
+      resources: Map[String, String]
+  ) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
+    override def loadClass(name: String, resolve: Boolean): Class[_] = {
+      val clazz = parent.loadClass(name)
+      if (resolve) resolveClass(clazz)
+      clazz
+    }
+    override def toString: String = "ResourceLoader"
+  }
+  // Creates a one or two layered classloader for the provided classpaths depending on whether
+  // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
+  // above the regular jar layer. This allows the snapshot layer to be invalidated without
+  // invalidating the regular jar layer. If the classpath is empty, it just returns the parent
+  // loader.
+  private def getResourceLayer(
+      classpath: Seq[File],
+      resources: Seq[(File, Long)],
+      parent: ClassLoader,
+      cache: ClassLoaderCache,
+      resourceMap: Map[String, String]
+  ): ClassLoader = {
+    if (resources.nonEmpty) {
+      cache(
+        resources.toList,
+        parent,
+        () => new ResourceLoader(classpath, parent, resourceMap)
+      )
+    } else parent
+  }
+
+  private[this] class FlatLoader(classpath: Seq[File], parent: ClassLoader)
+      extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) {
+    override def toString: String =
+      s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)"
+  }
   // helper methods
   private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader =
-    new URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) {
+    new FlatLoader(classpath, parent)
-      override def toString: String =
+  private[this] def modifiedTimes(stamps: Seq[(Path, FileStamp)]): Seq[(File, Long)] = stamps.map {
-        s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)"
+    case (p, LastModified(lm)) => p.toFile -> lm
-    }
+    case (p, _) =>
-
+      val f = p.toFile
-}
+      f -> IO.getModifiedTimeOrZero(f)
-
-private[sbt] object SbtMetaBuildClassLoader {
-  def apply(appProvider: AppProvider): ClassLoader = {
-    val interfaceFilter: URL => Boolean = _.getFile.endsWith("test-interface-1.0.jar")
-    def urls(jars: Array[File]): Array[URL] = jars.map(_.toURI.toURL)
-    val (interfaceURL, rest) = urls(appProvider.mainClasspath).partition(interfaceFilter)
-    val scalaProvider = appProvider.scalaProvider
-    val interfaceLoader = new URLClassLoader(interfaceURL, scalaProvider.launcher.topLoader) {
-      override def toString: String = s"SbtTestInterfaceClassLoader(${getURLs.head})"
-    }
-    val updatedLibraryLoader = new URLClassLoader(urls(scalaProvider.jars), interfaceLoader) {
-      override def toString: String = s"ScalaClassLoader(jars = {${getURLs.mkString(", ")}}"
-    }
-    new URLClassLoader(rest, updatedLibraryLoader) {
-      override def toString: String = s"SbtMetaBuildClassLoader"
-      override def close(): Unit = {
-        super.close()
-        updatedLibraryLoader.close()
-        interfaceLoader.close()
-      }
-    }
   }
 }

main/src/main/scala/sbt/internal/ConsoleProject.scala

@@ -8,9 +8,10 @@
 package sbt
 package internal
 
-import sbt.util.Logger
+import sbt.internal.classpath.AlternativeZincUtil
+import sbt.internal.inc.{ ScalaInstance, ZincLmUtil }
 import sbt.internal.util.JLine
-import sbt.internal.inc.{ ScalaInstance, ZincLmUtil, ZincUtil }
+import sbt.util.Logger
 import xsbti.compile.ClasspathOptionsUtil
 
 object ConsoleProject {
@@ -35,10 +36,11 @@ object ConsoleProject {
     val launcher = app.provider.scalaProvider.launcher
     val compiler = scalaCompilerBridgeBinaryJar match {
       case Some(jar) =>
-        ZincUtil.scalaCompiler(
+        AlternativeZincUtil.scalaCompiler(
           scalaInstance = scalaInstance,
           classpathOptions = ClasspathOptionsUtil.repl,
-          compilerBridgeJar = jar
+          compilerBridgeJar = jar,
+          classLoaderCache = state1.get(BasicKeys.classLoaderCache)
         )
       case None =>
         ZincLmUtil.scalaCompiler(
@@ -50,6 +52,7 @@ object ConsoleProject {
           dependencyResolution = dependencyResolution,
           compilerBridgeSource = extracted.get(Keys.scalaCompilerBridgeSource),
           scalaJarsTarget = zincDir,
+          classLoaderCache = state1.get(BasicKeys.classLoaderCache),
           log = log
         )
     }

main/src/main/scala/sbt/internal/LRUCache.scala

@@ -1,113 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt.internal
-
-import java.util.concurrent.atomic.AtomicInteger
-
-import scala.annotation.tailrec
-
-private[sbt] sealed trait LRUCache[K, V] extends AutoCloseable {
-  def get(key: K): Option[V]
-  def entries: Seq[(K, V)]
-  def maxSize: Int
-  def put(key: K, value: V): Option[V]
-  def remove(key: K): Option[V]
-  def size: Int
-}
-
-private[sbt] object LRUCache {
-  private[this] class impl[K, V](override val maxSize: Int, onExpire: Option[((K, V)) => Unit])
-      extends LRUCache[K, V] {
-    private[this] val elementsSortedByAccess: Array[(K, V)] = new Array[(K, V)](maxSize)
-    private[this] val lastIndex: AtomicInteger = new AtomicInteger(-1)
-
-    override def close(): Unit = this.synchronized {
-      val f = onExpire.getOrElse((_: (K, V)) => Unit)
-      0 until maxSize foreach { i =>
-        elementsSortedByAccess(i) match {
-          case null =>
-          case el   => f(el)
-        }
-        elementsSortedByAccess(i) = null
-      }
-      lastIndex.set(-1)
-    }
-    override def entries: Seq[(K, V)] = this.synchronized {
-      (0 to lastIndex.get()).map(elementsSortedByAccess)
-    }
-    override def get(key: K): Option[V] = this.synchronized {
-      indexOf(key) match {
-        case -1 => None
-        case i  => replace(i, key, elementsSortedByAccess(i)._2)
-      }
-    }
-    override def put(key: K, value: V): Option[V] = this.synchronized {
-      indexOf(key) match {
-        case -1 =>
-          append(key, value)
-          None
-        case i => replace(i, key, value)
-      }
-    }
-    override def remove(key: K): Option[V] = this.synchronized {
-      indexOf(key) match {
-        case -1 => None
-        case i  => remove(i, lastIndex.get, expire = false)
-      }
-    }
-    override def size: Int = lastIndex.get + 1
-    override def toString: String = {
-      val values = 0 to lastIndex.get() map { i =>
-        val (key, value) = elementsSortedByAccess(i)
-        s"$key -> $value"
-      }
-      s"LRUCache(${values mkString ", "})"
-    }
-
-    private def indexOf(key: K): Int =
-      elementsSortedByAccess.view.take(lastIndex.get() + 1).indexWhere(_._1 == key)
-    private def replace(index: Int, key: K, value: V): Option[V] = {
-      val prev = remove(index, lastIndex.get(), expire = false)
-      append(key, value)
-      prev
-    }
-    private def append(key: K, value: V): Unit = {
-      while (lastIndex.get() >= maxSize - 1) {
-        remove(0, lastIndex.get(), expire = true)
-      }
-      val index = lastIndex.incrementAndGet()
-      elementsSortedByAccess(index) = (key, value)
-    }
-    private def remove(index: Int, endIndex: Int, expire: Boolean): Option[V] = {
-      @tailrec
-      def shift(i: Int): Unit = if (i < endIndex) {
-        elementsSortedByAccess(i) = elementsSortedByAccess(i + 1)
-        shift(i + 1)
-      }
-      val prev = elementsSortedByAccess(index)
-      shift(index)
-      lastIndex.set(endIndex - 1)
-      if (expire) onExpire.foreach(f => f(prev))
-      Some(prev._2)
-    }
-  }
-  private def emptyCache[K, V]: LRUCache[K, V] = new LRUCache[K, V] {
-    override def get(key: K): Option[V] = None
-    override def entries: Seq[(K, V)] = Nil
-    override def maxSize: Int = 0
-    override def put(key: K, value: V): Option[V] = None
-    override def remove(key: K): Option[V] = None
-    override def size: Int = 0
-    override def close(): Unit = {}
-    override def toString = "EmptyLRUCache"
-  }
-  def apply[K, V](maxSize: Int): LRUCache[K, V] =
-    if (maxSize > 0) new impl(maxSize, None) else emptyCache
-  def apply[K, V](maxSize: Int, onExpire: (K, V) => Unit): LRUCache[K, V] =
-    if (maxSize > 0) new impl(maxSize, Some(onExpire.tupled)) else emptyCache
-}

main/src/main/scala/sbt/internal/Load.scala

@@ -87,6 +87,7 @@ private[sbt] object Load {
       dependencyResolution = dependencyResolution,
       compilerBridgeSource = ZincLmUtil.getDefaultBridgeModule(scalaProvider.version),
       scalaJarsTarget = zincDir,
+      state.get(BasicKeys.classLoaderCache),
       log = log
     )
     val compilers = ZincUtil.compilers(
@@ -374,8 +375,9 @@ private[sbt] object Load {
           val projectSettings = build.defined flatMap {
             case (id, project) =>
               val ref = ProjectRef(uri, id)
-              val defineConfig: Seq[Setting[_]] = for (c <- project.configurations)
+              val defineConfig: Seq[Setting[_]] =
-                yield ((configuration in (ref, ConfigKey(c.name))) :== c)
+                for (c <- project.configurations)
+                  yield ((configuration in (ref, ConfigKey(c.name))) :== c)
               val builtin: Seq[Setting[_]] =
                 (thisProject :== project) +: (thisProjectRef :== ref) +: defineConfig
               val settings = builtin ++ project.settings ++ injectSettings.project

main/src/main/scala/sbt/internal/LogManager.scala

@@ -174,7 +174,7 @@ object LogManager {
       key: ScopedKey[_],
       state: State
   ): SuppressedTraceContext => Option[String] = {
-    lazy val display = Project.showContextKey(state)
+    val display = Project.showContextKey(state)
     def commandBase = "last " + display.show(unwrapStreamsKey(key))
     def command(useFormat: Boolean) =
       if (useFormat) BLUE + commandBase + RESET else s"'$commandBase'"

main/src/main/scala/sbt/internal/Repository.scala

@@ -1,59 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt.internal
-
-import java.util.concurrent.ConcurrentHashMap
-
-import scala.collection.JavaConverters._
-
-/**
- * Represents an abstract cache of values, accessible by a key. The interface is deliberately
- * minimal to give maximum flexibility to the implementation classes. For example, one can construct
- * a cache from a `sbt.io.FileTreeRepository` that automatically registers the paths with the
- * cache (but does not clear the cache on close):
- * {{{
- *  val repository = sbt.io.FileTreeRepository.default(_.getPath)
- *  val fileCache = new Repository[Seq, (Path, Boolean), TypedPath] {
- *    override def get(key: (Path, Boolean)): Seq[TypedPath] = {
- *      val (path, recursive) = key
- *      val depth = if (recursive) Int.MaxValue else 0
- *      repository.register(path, depth)
- *      repository.list(path, depth, AllPass)
- *    }
- *    override def close(): Unit = {}
- *  }
- * }}}
- *
- * @tparam M the container type of the cache. This will most commonly be `Option` or `Seq`.
- * @tparam K the key type
- * @tparam V the value type
- */
-private[sbt] trait Repository[M[_], K, V] extends AutoCloseable {
-  def get(key: K): M[V]
-  override def close(): Unit = {}
-}
-
-private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] {
-  private[this] val map = new ConcurrentHashMap[K, V].asScala
-  override def get(key: K): Option[V] = map.get(key)
-  def put(key: K, value: V): Unit = this.synchronized {
-    map.put(key, value)
-    ()
-  }
-  def remove(key: K): Unit = this.synchronized {
-    map.remove(key)
-    ()
-  }
-  override def close(): Unit = this.synchronized {
-    map.foreach {
-      case (_, v: AutoCloseable) => v.close()
-      case _                     =>
-    }
-    map.clear()
-  }
-}

main/src/main/scala/sbt/internal/TaskRepository.scala

@@ -1,38 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt.internal
-
-import sbt.Keys.state
-import sbt._
-
-private[sbt] object TaskRepository {
-  private[sbt] type Repr = MutableRepository[TaskKey[_], Any]
-  private[sbt] def proxy[T: Manifest](taskKey: TaskKey[T], task: => T): Def.Setting[Task[T]] =
-    proxy(taskKey, Def.task(task))
-  private[sbt] def proxy[T: Manifest](
-      taskKey: TaskKey[T],
-      task: Def.Initialize[Task[T]]
-  ): Def.Setting[Task[T]] =
-    taskKey := Def.taskDyn {
-      val taskRepository = state.value
-        .get(Keys.taskRepository)
-        .getOrElse {
-          val msg = "TaskRepository.proxy called before state was initialized"
-          throw new IllegalStateException(msg)
-        }
-      taskRepository.get(taskKey) match {
-        case Some(value: T) => Def.task(value)
-        case _ =>
-          Def.task {
-            val value = task.value
-            taskRepository.put(taskKey, value)
-            value
-          }
-      }
-    }.value
-}

main/src/main/scala/sbt/internal/XMainConfiguration.scala

@@ -0,0 +1,236 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.internal
+
+import java.io.File
+import java.lang.reflect.InvocationTargetException
+import java.net.{ URL, URLClassLoader }
+import java.util.regex.Pattern
+
+import xsbti._
+
+/**
+ * Generates a new app configuration and invokes xMainImpl.run. For AppConfigurations generated
+ * by recent launchers, it is unnecessary to modify the original configuration, but configurations
+ * generated by older launchers need to be modified to place the test interface jar higher in
+ * the class hierarchy. The methods this object are implemented without using the scala library
+ * so that we can avoid loading any classes from the old scala provider. Verified as of
+ * sbt 1.3.0 that there are no references to the scala standard library in any of the methods
+ * in this file.
+ */
+private[sbt] class XMainConfiguration {
+  private def close(classLoader: ClassLoader): Unit = classLoader match {
+    case a: AutoCloseable => a.close()
+    case _                =>
+  }
+  def runXMain(configuration: xsbti.AppConfiguration): xsbti.MainResult = {
+    val updatedConfiguration =
+      if (configuration.provider.scalaProvider.launcher.topLoader.getClass.getCanonicalName
+            .contains("TestInterfaceLoader")) {
+        configuration
+      } else {
+        makeConfiguration(configuration)
+      }
+    val loader = updatedConfiguration.provider.loader
+    Thread.currentThread.setContextClassLoader(loader)
+    val clazz = loader.loadClass("sbt.xMainImpl$")
+    val instance = clazz.getField("MODULE$").get(null)
+    val runMethod = clazz.getMethod("run", classOf[xsbti.AppConfiguration])
+    try {
+      loader.loadClass("sbt.internal.parser.SbtParserInit").getConstructor().newInstance()
+      runMethod.invoke(instance, updatedConfiguration).asInstanceOf[xsbti.MainResult]
+    } catch {
+      case e: InvocationTargetException =>
+        // This propogates xsbti.FullReload to the launcher
+        throw e.getCause
+    }
+  }
+
+  private def makeConfiguration(configuration: xsbti.AppConfiguration): xsbti.AppConfiguration = {
+    val baseLoader = classOf[XMainConfiguration].getClassLoader
+    val url = baseLoader.getResource("sbt/internal/XMainConfiguration.class")
+    val urlArray = new Array[URL](1)
+    urlArray(0) = new URL(url.getPath.replaceAll("[!][^!]*class", ""))
+    val topLoader = configuration.provider.scalaProvider.launcher.topLoader
+    // This loader doesn't have the scala library in it so it's critical that none of the code
+    // in this file use the scala library.
+    val modifiedLoader = new URLClassLoader(urlArray, topLoader) {
+      override def loadClass(name: String, resolve: Boolean): Class[_] = {
+        if (name.startsWith("sbt.internal.XMainConfiguration")) {
+          val clazz = findClass(name)
+          if (resolve) resolveClass(clazz)
+          clazz
+        } else {
+          super.loadClass(name, resolve)
+        }
+      }
+    }
+    val xMainConfigurationClass = modifiedLoader.loadClass("sbt.internal.XMainConfiguration")
+    val instance: AnyRef =
+      xMainConfigurationClass.getConstructor().newInstance().asInstanceOf[AnyRef]
+
+    val method = xMainConfigurationClass.getMethod("makeLoader", classOf[AppProvider])
+    val modifiedConfigurationClass =
+      modifiedLoader.loadClass("sbt.internal.XMainConfiguration$ModifiedConfiguration")
+
+    val loader = method.invoke(instance, configuration.provider).asInstanceOf[ClassLoader]
+
+    Thread.currentThread.setContextClassLoader(loader)
+    val cons = modifiedConfigurationClass.getConstructors()(0)
+    close(configuration.provider.loader)
+    val scalaProvider = configuration.provider.scalaProvider
+    val providerClass = scalaProvider.getClass
+    val _ = try {
+      val method = providerClass.getMethod("loaderLibraryOnly")
+      close(method.invoke(scalaProvider).asInstanceOf[ClassLoader])
+      1
+    } catch { case _: NoSuchMethodException => 1 }
+    close(scalaProvider.loader)
+    close(configuration.provider.loader)
+    cons.newInstance(instance, configuration, loader).asInstanceOf[xsbti.AppConfiguration]
+  }
+
+  /*
+   * Replaces the AppProvider.loader method with a new loader that puts the sbt test interface
+   * jar ahead of the rest of the sbt classpath in the classloading hierarchy.
+   */
+  private[sbt] class ModifiedConfiguration(
+      val configuration: xsbti.AppConfiguration,
+      val metaLoader: ClassLoader
+  ) extends xsbti.AppConfiguration {
+
+    private class ModifiedAppProvider(val appProvider: AppProvider) extends AppProvider {
+      private val delegate = configuration.provider.scalaProvider
+      object ModifiedScalaProvider extends ScalaProvider {
+        override def launcher(): Launcher = new Launcher {
+          private val delegateLauncher = delegate.launcher
+          private val interfaceLoader = metaLoader.loadClass("sbt.testing.Framework").getClassLoader
+          override def getScala(version: String): ScalaProvider = getScala(version, "")
+          override def getScala(version: String, reason: String): ScalaProvider =
+            getScala(version, reason, "org.scala-lang")
+          override def getScala(version: String, reason: String, scalaOrg: String): ScalaProvider =
+            delegateLauncher.getScala(version, reason, scalaOrg)
+          override def app(id: xsbti.ApplicationID, version: String): AppProvider =
+            delegateLauncher.app(id, version)
+          override def topLoader(): ClassLoader = interfaceLoader
+          override def globalLock(): GlobalLock = delegateLauncher.globalLock()
+          override def bootDirectory(): File = delegateLauncher.bootDirectory()
+          override def ivyRepositories(): Array[xsbti.Repository] =
+            delegateLauncher.ivyRepositories()
+          override def appRepositories(): Array[xsbti.Repository] =
+            delegateLauncher.appRepositories()
+          override def isOverrideRepositories: Boolean = delegateLauncher.isOverrideRepositories
+          override def ivyHome(): File = delegateLauncher.ivyHome()
+          override def checksums(): Array[String] = delegateLauncher.checksums()
+        }
+        override def version(): String = delegate.version
+        override def loader(): ClassLoader = metaLoader.getParent
+        override def jars(): Array[File] = delegate.jars
+        @deprecated("Implements deprecated api", "1.3.0")
+        override def libraryJar(): File = delegate.libraryJar
+        @deprecated("Implements deprecated api", "1.3.0")
+        override def compilerJar(): File = delegate.compilerJar
+        override def app(id: xsbti.ApplicationID): AppProvider = delegate.app(id)
+        def loaderLibraryOnly(): ClassLoader = metaLoader.getParent.getParent
+      }
+      override def scalaProvider(): ModifiedScalaProvider.type = ModifiedScalaProvider
+      override def id(): xsbti.ApplicationID = appProvider.id()
+      override def loader(): ClassLoader = metaLoader
+      @deprecated("Implements deprecated api", "1.3.0")
+      override def mainClass(): Class[_ <: AppMain] = appProvider.mainClass()
+      override def entryPoint(): Class[_] = appProvider.entryPoint()
+      override def newMain(): AppMain = appProvider.newMain()
+      override def mainClasspath(): Array[File] = appProvider.mainClasspath()
+      override def components(): ComponentProvider = appProvider.components()
+    }
+    override def arguments(): Array[String] = configuration.arguments
+    override def baseDirectory(): File = configuration.baseDirectory
+    override def provider(): AppProvider = new ModifiedAppProvider(configuration.provider)
+  }
+
+  /**
+   *  Rearrange the classloaders so that test-interface is above the scala library. Implemented
+   *  without using the scala standard library to minimize classloading.
+   * @param appProvider the appProvider that needs to be modified
+   * @return a ClassLoader with a URLClassLoader for the test-interface-1.0.jar above the
+   *         scala library.
+   */
+  private[sbt] def makeLoader(appProvider: AppProvider): ClassLoader = {
+    val pattern = Pattern.compile("test-interface-[0-9.]+\\.jar")
+    val cp = appProvider.mainClasspath
+    val interfaceURL = new Array[URL](1)
+    val rest = new Array[URL](cp.length - 1)
+
+    {
+      var i = 0
+      var j = 0 // index into rest
+      while (i < cp.length) {
+        val file = cp(i)
+        if (pattern.matcher(file.getName).find()) {
+          interfaceURL(0) = file.toURI.toURL
+        } else {
+          rest(j) = file.toURI.toURL
+          j += 1
+        }
+        i += 1
+      }
+    }
+    val scalaProvider = appProvider.scalaProvider
+    val topLoader = scalaProvider.launcher.topLoader
+    class InterfaceLoader extends URLClassLoader(interfaceURL, topLoader) {
+      override def toString: String = "SbtTestInterfaceClassLoader(" + interfaceURL(0) + ")"
+    }
+    val interfaceLoader = new InterfaceLoader
+    val siJars = scalaProvider.jars
+    val lib = new Array[URL](1)
+    val scalaRest = new Array[URL](siJars.length - 1)
+
+    {
+      var i = 0
+      var j = 0 // index into scalaRest
+      while (i < siJars.length) {
+        val file = siJars(i)
+        if (file.getName.equals("scala-library.jar")) {
+          lib(0) = file.toURI.toURL
+        } else {
+          scalaRest(j) = file.toURI.toURL
+          j += 1
+        }
+        i += 1
+      }
+    }
+    class LibraryLoader extends URLClassLoader(lib, interfaceLoader) {
+      override def toString: String = "ScalaLibraryLoader( " + lib(0) + ")"
+    }
+    val libraryLoader = new LibraryLoader
+    class FullLoader extends URLClassLoader(scalaRest, libraryLoader) {
+      private val jarString: String = {
+        val res = new java.lang.StringBuilder
+        var i = 0
+        while (i < scalaRest.length) {
+          res.append(scalaRest(i).getPath)
+          res.append(", ")
+          i += 1
+        }
+        res.toString
+      }
+      override def toString: String = "ScalaClassLoader(jars = " + jarString + ")"
+    }
+    val fullLoader = new FullLoader
+    class MetaBuildLoader extends URLClassLoader(rest, fullLoader) {
+      override def toString: String = "SbtMetaBuildClassLoader"
+      override def close(): Unit = {
+        super.close()
+        libraryLoader.close()
+        fullLoader.close()
+        interfaceLoader.close()
+      }
+    }
+    new MetaBuildLoader
+  }
+}

main/src/main/scala/sbt/internal/parser/SbtParser.scala

@@ -163,6 +163,16 @@ private[sbt] object SbtParser {
   }
 }
 
+private class SbtParserInit {
+  new Thread("sbt-parser-init-thread") {
+    setDaemon(true)
+    start()
+    override def run(): Unit = {
+      val _ = SbtParser.defaultGlobalForParser
+    }
+  }
+}
+
 /**
  * This method solely exists to add scaladoc to members in SbtParser which
  * are defined using pattern matching.

main/src/main/scala/sbt/nio/Keys.scala

@@ -148,6 +148,8 @@ object Keys {
   private[sbt] val pathToFileStamp = taskKey[Path => Option[FileStamp]](
     "A function that computes a file stamp for a path. It may have the side effect of updating a cache."
   ).withRank(Invisible)
+  private[sbt] val classpathFiles =
+    taskKey[Seq[Path]]("The classpath for a task.").withRank(Invisible)
 
   private[this] val hasCheckedMetaBuildMsg =
     "Indicates whether or not we have called the checkBuildSources task. This is to avoid warning " +

main/src/test/scala/sbt/internal/LRUCacheTest.scala

@@ -1,78 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt
-package internal
-
-import java.util.concurrent.atomic.AtomicInteger
-
-import org.scalatest.{ FlatSpec, Matchers }
-
-class LRUCacheTest extends FlatSpec with Matchers {
-  "LRUCache" should "flush entries when full" in {
-    val cache = LRUCache[Int, Int](2)
-    cache.put(1, 1)
-    cache.put(2, 2)
-    cache.put(3, 3)
-    assert(cache.get(1).isEmpty)
-    assert(cache.get(2).contains(2))
-    assert(cache.get(3).contains(3))
-
-    assert(cache.get(2).contains(2))
-    cache.put(1, 1)
-    assert(cache.get(3).isEmpty)
-    assert(cache.get(2).contains(2))
-    assert(cache.get(1).contains(1))
-  }
-  it should "remove entries" in {
-    val cache = LRUCache[Int, Int](2)
-    cache.put(1, 1)
-    cache.put(2, 2)
-    assert(cache.get(1).contains(1))
-    assert(cache.get(2).contains(2))
-
-    assert(cache.remove(1).getOrElse(-1) == 1)
-    assert(cache.get(1).isEmpty)
-    assert(cache.get(2).contains(2))
-  }
-  it should "clear entries on close" in {
-    val cache = LRUCache[Int, Int](2)
-    cache.put(1, 1)
-    assert(cache.get(1).contains(1))
-    cache.close()
-    assert(cache.get(1).isEmpty)
-  }
-  it should "call onExpire in close" in {
-    val count = new AtomicInteger(0)
-    val cache =
-      LRUCache[Int, Int](
-        maxSize = 3,
-        onExpire = (_: Int, _: Int) => { count.getAndIncrement(); () }
-      )
-    cache.put(1, 1)
-    cache.put(2, 2)
-    cache.put(3, 3)
-    cache.put(4, 4)
-    assert(count.get == 1)
-    cache.close()
-    assert(count.get == 4)
-  }
-  it should "apply on remove function" in {
-    val value = new AtomicInteger(0)
-    val cache = LRUCache[Int, Int](1, (k: Int, v: Int) => value.set(k + v))
-    cache.put(1, 3)
-    cache.put(2, 2)
-    assert(value.get() == 4)
-    assert(cache.get(2).contains(2))
-  }
-  it should "print sorted entries in toString" in {
-    val cache = LRUCache[Int, Int](2)
-    cache.put(2, 2)
-    cache.put(1, 1)
-    assert(cache.toString == s"LRUCache(2 -> 2, 1 -> 1)")
-  }
-}

project/Dependencies.scala

@@ -17,7 +17,7 @@ object Dependencies {
       case Some(version) => version
       case _             => nightlyVersion.getOrElse("1.3.0-M3")
     }
-  val zincVersion = nightlyVersion.getOrElse("1.3.0-M5")
+  val zincVersion = nightlyVersion.getOrElse("1.3.0-M6")
 
   private val sbtIO = "org.scala-sbt" %% "io" % ioVersion
 
@@ -32,7 +32,7 @@ object Dependencies {
   private val libraryManagementCore = "org.scala-sbt" %% "librarymanagement-core" % lmVersion
   private val libraryManagementIvy = "org.scala-sbt" %% "librarymanagement-ivy" % lmVersion
 
-  val launcherVersion = "1.0.4"
+  val launcherVersion = "1.1.0"
   val launcherInterface = "org.scala-sbt" % "launcher-interface" % launcherVersion
   val rawLauncher = "org.scala-sbt" % "launcher" % launcherVersion
   val testInterface = "org.scala-sbt" % "test-interface" % "1.0"

sbt/src/sbt-test/classloader-cache/akka-actor-system/test

@@ -1,4 +1,4 @@
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
+> run
 
 > run
 

sbt/src/sbt-test/classloader-cache/library-mismatch/test

@@ -1,15 +1,3 @@
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
-
-> run
-
-# This fails because the runtime layer includes an old version of the foo-lib library that doesn't
-# have the sbt.foo.Foo.y method defined.
-> test
-
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
-
 > run
 
 > test
-
-> test

sbt/src/sbt-test/classloader-cache/resources/build.sbt

@@ -0,0 +1,5 @@
+resolvers += "Local Maven" at (baseDirectory.value / "libraries" / "foo" / "ivy").toURI.toURL.toString
+
+libraryDependencies += "sbt" %% "foo-lib" % "0.1.0"
+
+libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.5" % "test"

sbt/src/sbt-test/classloader-cache/resources/changes/UpdatedResourceTest.scala

@@ -0,0 +1,9 @@
+package scripted
+
+import org.scalatest.FlatSpec
+
+class ResourceTest extends FlatSpec {
+  "test resources" should "load" in {
+    Main.main(Array("bar.txt", "updated-test"))
+  }
+}

sbt/src/sbt-test/classloader-cache/resources/changes/updated-main.txt

@@ -0,0 +1 @@
+updated-main

sbt/src/sbt-test/classloader-cache/resources/changes/updated-test.txt

@@ -0,0 +1 @@
+updated-test

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/build.sbt

@@ -0,0 +1,11 @@
+name := "foo-lib"
+
+organization := "sbt"
+
+publishTo := Some(Resolver.file("test-resolver", file("").getCanonicalFile / "ivy"))
+
+version := "0.1.0"
+
+classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
+
+Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0-javadoc.jar.md5

@@ -0,0 +1 @@
+9bab3dca3e061fdbbff2419ef0eee90d

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0-javadoc.jar.sha1

@@ -0,0 +1 @@
+11081018ca6824226893b153fe439d3a94e23286

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0-sources.jar.md5

@@ -0,0 +1 @@
+afba811024a110dc47734c6d9ec0eb5b

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0-sources.jar.sha1

@@ -0,0 +1 @@
+e4e1b37d77060a77ab20c508df90cdae6125f3bc

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0.jar.md5

@@ -0,0 +1 @@
+6de1cfe8548a007ac08763ab5f051e08

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0.jar.sha1

@@ -0,0 +1 @@
+f25267dfe760e734ea4f8a00b8dc6ace477589d6

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0.pom

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>sbt</groupId>
+    <artifactId>foo-lib_2.12</artifactId>
+    <packaging>jar</packaging>
+    <description>foo-lib</description>
+    <version>0.1.0</version>
+    <name>foo-lib</name>
+    <organization>
+        <name>sbt</name>
+    </organization>
+    <dependencies>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.12.8</version>
+        </dependency>
+    </dependencies>
+</project>

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0.pom.md5

@@ -0,0 +1 @@
+b83994a1d5ecfc3d6ff71021649e342d

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/ivy/sbt/foo-lib_2.12/0.1.0/foo-lib_2.12-0.1.0.pom.sha1

@@ -0,0 +1 @@
+6b2c0b4d69ed8e10e0221db841e0f1fdeef12780

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.3.0-RC1

sbt/src/sbt-test/classloader-cache/resources/libraries/foo/src/main/scala/resource/Resource.scala

@@ -0,0 +1,19 @@
+package resource
+
+import java.net.URL
+import java.nio.file._
+import java.util.Collections
+
+object Resource {
+  def readFile(url: URL): String = {
+      val uri = url.toURI
+      val fileSystem =
+        if (uri.getScheme != "jar") None
+        else Some(FileSystems.newFileSystem(uri, Collections.emptyMap[String, Object]))
+      try new String(Files.readAllBytes(Paths.get(uri)))
+      finally fileSystem.foreach(_.close())
+  }
+  def getStringResource(name: String): String = {
+    readFile(Resource.getClass.getClassLoader.getResource(name))
+  }
+}

sbt/src/sbt-test/classloader-cache/resources/src/main/resources/foo.txt

@@ -0,0 +1 @@
+main

sbt/src/sbt-test/classloader-cache/resources/src/main/scala/scripted/Main.scala

@@ -0,0 +1,10 @@
+package scripted
+
+import resource.Resource
+
+object Main {
+  def main(args: Array[String]): Unit = {
+    val Array(resource, expected) = args
+    assert(Resource.getStringResource(resource) == expected)
+  }
+}

sbt/src/sbt-test/classloader-cache/resources/src/test/resources/bar.txt

@@ -0,0 +1 @@
+test

sbt/src/sbt-test/classloader-cache/resources/src/test/scala/scripted/ResourceTest.scala

@@ -0,0 +1,9 @@
+package scripted
+
+import org.scalatest.FlatSpec
+
+class ResourceTest extends FlatSpec {
+  "test resources" should "load" in {
+    Main.main(Array("bar.txt", "test"))
+  }
+}

sbt/src/sbt-test/classloader-cache/resources/test

@@ -0,0 +1,15 @@
+> run foo.txt main
+
+$ copy-file changes/updated-main.txt src/main/resources/foo.txt
+
+> run foo.txt updated-main
+
+> test
+
+$ copy-file changes/updated-test.txt src/test/resources/bar.txt
+
+-> test
+
+$ copy-file changes/UpdatedResourceTest.scala src/test/scala/scripted/ResourceTest.scala
+
+> test

sbt/src/sbt-test/classloader-cache/runtime-layers/test

@@ -8,14 +8,10 @@
 
 > run
 
-> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies
+> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
 
 > run
 
-> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
-
--> run
-
 > set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
 
 > Test / runMain sbt.scripted.TestAkkaTest
@@ -24,14 +20,6 @@
 
 > Test / runMain sbt.scripted.TestAkkaTest
 
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies
+> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
-
-> Test / runMain sbt.scripted.TestAkkaTest
-
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
-
-> Test / runMain sbt.scripted.TestAkkaTest
-
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
 
 > Test / runMain sbt.scripted.TestAkkaTest

sbt/src/sbt-test/classloader-cache/scalatest/test

@@ -1,9 +1,5 @@
 > test
 
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
-
-> test
-
 > set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
 
 > test
@@ -12,7 +8,7 @@
 
 > test
 
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.TestDependencies
+> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
 
 $ copy-file changes/bad.scala src/test/scala/sbt/ScalatestTest.scala
 

tasks/src/main/scala/sbt/Execute.scala

@@ -7,6 +7,8 @@
 
 package sbt
 
+import java.util.concurrent.ExecutionException
+
 import sbt.internal.util.ErrorHandling.wideConvert
 import sbt.internal.util.{ DelegatingPMap, IDSet, PMap, RMap, ~> }
 import sbt.internal.util.Types._
@@ -109,7 +111,15 @@ private[sbt] final class Execute[F[_] <: AnyRef](
         }
       }
 
-      (strategy.take()).process()
+      try {
+        strategy.take().process()
+      } catch {
+        case e: ExecutionException =>
+          e.getCause match {
+            case oom: OutOfMemoryError => throw oom
+            case _                     => throw e
+          }
+      }
       if (reverse.nonEmpty) next()
     }
     next()

testing/src/main/scala/sbt/TestFramework.scala

@@ -52,7 +52,7 @@ final class TestFramework(val implClassNames: String*) extends Serializable {
           + " using a layered class loader that cannot reach the sbt.testing.Framework class."
           + " The most likely cause is that your project has a runtime dependency on your"
           + " test framework, e.g. scalatest. To fix this, you can try to set\n"
-          + "Test / classLoaderLayeringStrategy := new ClassLoaderLayeringStrategy.Test(false, true)\nor\n"
+          + "Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary\nor\n"
           + "Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"
       )
       None
@@ -138,8 +138,9 @@ final class TestRunner(
       val nestedTasks =
         try testTask.execute(handler, loggers.map(_.log).toArray)
         catch {
-          case NonFatal(e)           => errorEvents(e)
+          case e: NoClassDefFoundError => errorEvents(e)
-          case e: IllegalAccessError => errorEvents(e)
+          case NonFatal(e)             => errorEvents(e)
+          case e: IllegalAccessError   => errorEvents(e)
         } finally {
           loggers.foreach(_.flush())
         }

zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincLmUtil.scala

@@ -8,10 +8,9 @@
 package sbt.internal.inc
 
 import java.io.File
-import java.net.URLClassLoader
 
-import sbt.librarymanagement.{ DependencyResolution, ModuleID }
 import sbt.internal.inc.classpath.ClassLoaderCache
+import sbt.librarymanagement.{ DependencyResolution, ModuleID }
 import xsbti._
 import xsbti.compile._
 
@@ -32,6 +31,7 @@ object ZincLmUtil {
       dependencyResolution: DependencyResolution,
       compilerBridgeSource: ModuleID,
       scalaJarsTarget: File,
+      classLoaderCache: Option[ClassLoaderCache],
       log: Logger
   ): AnalyzingCompiler = {
     val compilerBridgeProvider = ZincComponentCompiler.interfaceProvider(
@@ -40,8 +40,13 @@ object ZincLmUtil {
       dependencyResolution,
       scalaJarsTarget,
     )
-    val loader = Some(new ClassLoaderCache(new URLClassLoader(new Array(0))))
+    new AnalyzingCompiler(
-    new AnalyzingCompiler(scalaInstance, compilerBridgeProvider, classpathOptions, _ => (), loader)
+      scalaInstance,
+      compilerBridgeProvider,
+      classpathOptions,
+      _ => (),
+      classLoaderCache
+    )
   }
 
   def getDefaultBridgeModule(scalaVersion: String): ModuleID =