SD-232 Recycle classloaders to be anti-hostile to JIT.

The compiler interface subclasses `scala.tools.nsc.Global`,
and loading this new subclass before each `compile` task forces
HotSpot JIT to deoptimize larges swathes of compiled code. It's
a bit like SBT has rigged the dice to always descend the longest
ladder in a game of Snakes and Ladders.

The slowdown seems to be larger with Scala 2.12. There are a number
of variables at play, but I think the main factor here is that
we now rely on JIT to devirtualize calls to final methods in traits
whereas we used to emit static calls. JIT does a good job at this,
so long as classloading doesn't undo that good work.

This commit extends the existing `ClassLoaderCache` to encompass
the classloader that includes the compiler interface JAR. I've
resorted to adding a var to `AnalyzingCompiler` to inject the
dependency to get the cache to the spot I need it without binary
incompatible changes to the intervening method signatures.

compile/src/main/scala/sbt/compiler/AnalyzingCompiler.scala

@@ -9,6 +9,8 @@ import xsbti.compile.{ CachedCompiler, CachedCompilerProvider, DependencyChanges
 import java.io.File
 import java.net.{ URL, URLClassLoader }
 
+import sbt.classpath.ClassLoaderCache
+
 /**
  * Interface to the Scala compiler that uses the dependency analysis plugin.  This class uses the Scala library and compiler
  * provided by scalaInstance.  This class requires a ComponentManager in order to obtain the interface code to scalac and
@@ -26,7 +28,12 @@ final class AnalyzingCompiler private (val scalaInstance: xsbti.compile.ScalaIns
   @deprecated("A Logger is no longer needed.", "0.13.0")
   def this(scalaInstance: xsbti.compile.ScalaInstance, provider: CompilerInterfaceProvider, cp: xsbti.compile.ClasspathOptions, log: Logger) = this(scalaInstance, provider, cp)
 
-  def onArgs(f: Seq[String] => Unit): AnalyzingCompiler = new AnalyzingCompiler(scalaInstance, provider, cp, f)
+  def onArgs(f: Seq[String] => Unit): AnalyzingCompiler =
+    {
+      val ac = new AnalyzingCompiler(scalaInstance, provider, cp, f)
+      ac.classLoaderCache = this.classLoaderCache
+      ac
+    }
 
   def apply(sources: Seq[File], changes: DependencyChanges, classpath: Seq[File], singleOutput: File, options: Seq[String], callback: AnalysisCallback, maximumErrors: Int, cache: GlobalsCache, log: Logger) {
     val arguments = (new CompilerArguments(scalaInstance, cp))(Nil, classpath, None, options)
@@ -110,17 +117,29 @@ final class AnalyzingCompiler private (val scalaInstance: xsbti.compile.ScalaIns
   private[this] def loader(log: Logger) =
     {
       val interfaceJar = provider(scalaInstance, log)
-      // this goes to scalaInstance.loader for scala classes and the loader of this class for xsbti classes
-      val dual = createDualLoader(scalaInstance.loader, getClass.getClassLoader)
-      new URLClassLoader(Array(interfaceJar.toURI.toURL), dual)
+      def createInterfaceLoader =
+        new URLClassLoader(Array(interfaceJar.toURI.toURL), createDualLoader(scalaInstance.loader(), getClass.getClassLoader))
+
+      classLoaderCache match {
+        case Some(cache) => cache.cachedCustomClassloader(interfaceJar :: scalaInstance.allJars().toList, () => createInterfaceLoader)
+        case None        => createInterfaceLoader
+      }
     }
+
   private[this] def getInterfaceClass(name: String, log: Logger) = Class.forName(name, true, loader(log))
+
   protected def createDualLoader(scalaLoader: ClassLoader, sbtLoader: ClassLoader): ClassLoader =
     {
       val xsbtiFilter = (name: String) => name.startsWith("xsbti.")
       val notXsbtiFilter = (name: String) => !xsbtiFilter(name)
       new classpath.DualLoader(scalaLoader, notXsbtiFilter, x => true, sbtLoader, xsbtiFilter, x => false)
     }
+
+  // TODO This should really be a constructor parameter, the var was used to avoid binary incompat changes
+  //      to signatures. Refactor for SBT 1.0.
+  def setClassLoaderCache(cache: ClassLoaderCache): Unit = classLoaderCache = Some(cache)
+  private var classLoaderCache: Option[ClassLoaderCache] = None
+
   override def toString = "Analyzing compiler (Scala " + scalaInstance.actualVersion + ")"
 }
 object AnalyzingCompiler {

main/src/main/scala/sbt/Defaults.scala

@@ -265,8 +265,13 @@ object Defaults extends BuildCommon {
       if (plugin) scalaBase / ("sbt-" + sbtv) else scalaBase
     }
 
-  def compilersSetting = compilers := Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value,
-    bootIvyConfiguration.value, scalaCompilerBridgeSource.value)(appConfiguration.value, streams.value.log)
+  def compilersSetting = compilers := {
+    val compilers = Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value,
+      bootIvyConfiguration.value, scalaCompilerBridgeSource.value)(appConfiguration.value, streams.value.log)
+    if (!java.lang.Boolean.getBoolean("sbt.disable.interface.classloader.cache"))
+      compilers.scalac.setClassLoaderCache(state.value.classLoaderCache)
+    compilers
+  }
 
   lazy val configTasks = docTaskSettings(doc) ++ inTask(compile)(compileInputsSettings) ++ configGlobal ++ compileAnalysisSettings ++ Seq(
     compile := compileTask.value,

util/classpath/src/main/scala/sbt/classpath/ClassLoaderCache.scala

@@ -15,25 +15,35 @@ private[sbt] final class ClassLoaderCache(val commonParent: ClassLoader) {
    * This method is thread-safe.
    */
   def apply(files: List[File]): ClassLoader = synchronized {
-    val tstamps = files.map(_.lastModified)
-    getFromReference(files, tstamps, delegate.get(files))
+    cachedCustomClassloader(files, () => new URLClassLoader(files.map(_.toURI.toURL).toArray, commonParent))
   }
 
-  private[this] def getFromReference(files: List[File], stamps: List[Long], existingRef: Reference[CachedClassLoader]) =
-    if (existingRef eq null)
-      newEntry(files, stamps)
-    else
-      get(files, stamps, existingRef.get)
+  /**
+   * Returns a ClassLoader, as created by `mkLoader`.
+   *
+   * The returned ClassLoader may be cached from a previous call if the last modified time of all `files` is unchanged.
+   * This method is thread-safe.
+   */
+  def cachedCustomClassloader(files: List[File], mkLoader: () => ClassLoader): ClassLoader = synchronized {
+    val tstamps = files.map(_.lastModified)
+    getFromReference(files, tstamps, delegate.get(files), mkLoader)
+  }
 
-  private[this] def get(files: List[File], stamps: List[Long], existing: CachedClassLoader): ClassLoader =
+  private[this] def getFromReference(files: List[File], stamps: List[Long], existingRef: Reference[CachedClassLoader], mkLoader: () => ClassLoader) =
+    if (existingRef eq null)
+      newEntry(files, stamps, mkLoader)
+    else
+      get(files, stamps, existingRef.get, mkLoader)
+
+  private[this] def get(files: List[File], stamps: List[Long], existing: CachedClassLoader, mkLoader: () => ClassLoader): ClassLoader =
     if (existing == null || stamps != existing.timestamps) {
-      newEntry(files, stamps)
+      newEntry(files, stamps, mkLoader)
     } else
       existing.loader
 
-  private[this] def newEntry(files: List[File], stamps: List[Long]): ClassLoader =
+  private[this] def newEntry(files: List[File], stamps: List[Long], mkLoader: () => ClassLoader): ClassLoader =
     {
-      val loader = new URLClassLoader(files.map(_.toURI.toURL).toArray, commonParent)
+      val loader = mkLoader()
       delegate.put(files, new SoftReference(new CachedClassLoader(loader, files, stamps)))
       loader
     }