From 5005abfef2a097c4b1af95d0b5585f006565aa2b Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Thu, 22 Sep 2016 17:24:30 +1000 Subject: [PATCH] SD-232 Recycle classloaders to be anti-hostile to JIT. The compiler interface subclasses `scala.tools.nsc.Global`, and loading this new subclass before each `compile` task forces HotSpot JIT to deoptimize larges swathes of compiled code. It's a bit like SBT has rigged the dice to always descend the longest ladder in a game of Snakes and Ladders. The slowdown seems to be larger with Scala 2.12. There are a number of variables at play, but I think the main factor here is that we now rely on JIT to devirtualize calls to final methods in traits whereas we used to emit static calls. JIT does a good job at this, so long as classloading doesn't undo that good work. This commit extends the existing `ClassLoaderCache` to encompass the classloader that includes the compiler interface JAR. I've resorted to adding a var to `AnalyzingCompiler` to inject the dependency to get the cache to the spot I need it without binary incompatible changes to the intervening method signatures. --- .../sbt/compiler/AnalyzingCompiler.scala | 27 +++++++++++++--- main/src/main/scala/sbt/Defaults.scala | 9 ++++-- .../sbt/classpath/ClassLoaderCache.scala | 32 ++++++++++++------- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/compile/src/main/scala/sbt/compiler/AnalyzingCompiler.scala b/compile/src/main/scala/sbt/compiler/AnalyzingCompiler.scala index 6f31452ff..1a74ffd8a 100644 --- a/compile/src/main/scala/sbt/compiler/AnalyzingCompiler.scala +++ b/compile/src/main/scala/sbt/compiler/AnalyzingCompiler.scala @@ -9,6 +9,8 @@ import xsbti.compile.{ CachedCompiler, CachedCompilerProvider, DependencyChanges import java.io.File import java.net.{ URL, URLClassLoader } +import sbt.classpath.ClassLoaderCache + /** * Interface to the Scala compiler that uses the dependency analysis plugin. This class uses the Scala library and compiler * provided by scalaInstance. This class requires a ComponentManager in order to obtain the interface code to scalac and @@ -26,7 +28,12 @@ final class AnalyzingCompiler private (val scalaInstance: xsbti.compile.ScalaIns @deprecated("A Logger is no longer needed.", "0.13.0") def this(scalaInstance: xsbti.compile.ScalaInstance, provider: CompilerInterfaceProvider, cp: xsbti.compile.ClasspathOptions, log: Logger) = this(scalaInstance, provider, cp) - def onArgs(f: Seq[String] => Unit): AnalyzingCompiler = new AnalyzingCompiler(scalaInstance, provider, cp, f) + def onArgs(f: Seq[String] => Unit): AnalyzingCompiler = + { + val ac = new AnalyzingCompiler(scalaInstance, provider, cp, f) + ac.classLoaderCache = this.classLoaderCache + ac + } def apply(sources: Seq[File], changes: DependencyChanges, classpath: Seq[File], singleOutput: File, options: Seq[String], callback: AnalysisCallback, maximumErrors: Int, cache: GlobalsCache, log: Logger) { val arguments = (new CompilerArguments(scalaInstance, cp))(Nil, classpath, None, options) @@ -110,17 +117,29 @@ final class AnalyzingCompiler private (val scalaInstance: xsbti.compile.ScalaIns private[this] def loader(log: Logger) = { val interfaceJar = provider(scalaInstance, log) - // this goes to scalaInstance.loader for scala classes and the loader of this class for xsbti classes - val dual = createDualLoader(scalaInstance.loader, getClass.getClassLoader) - new URLClassLoader(Array(interfaceJar.toURI.toURL), dual) + def createInterfaceLoader = + new URLClassLoader(Array(interfaceJar.toURI.toURL), createDualLoader(scalaInstance.loader(), getClass.getClassLoader)) + + classLoaderCache match { + case Some(cache) => cache.cachedCustomClassloader(interfaceJar :: scalaInstance.allJars().toList, () => createInterfaceLoader) + case None => createInterfaceLoader + } } + private[this] def getInterfaceClass(name: String, log: Logger) = Class.forName(name, true, loader(log)) + protected def createDualLoader(scalaLoader: ClassLoader, sbtLoader: ClassLoader): ClassLoader = { val xsbtiFilter = (name: String) => name.startsWith("xsbti.") val notXsbtiFilter = (name: String) => !xsbtiFilter(name) new classpath.DualLoader(scalaLoader, notXsbtiFilter, x => true, sbtLoader, xsbtiFilter, x => false) } + + // TODO This should really be a constructor parameter, the var was used to avoid binary incompat changes + // to signatures. Refactor for SBT 1.0. + def setClassLoaderCache(cache: ClassLoaderCache): Unit = classLoaderCache = Some(cache) + private var classLoaderCache: Option[ClassLoaderCache] = None + override def toString = "Analyzing compiler (Scala " + scalaInstance.actualVersion + ")" } object AnalyzingCompiler { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 9443430b4..64a95d3e0 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -265,8 +265,13 @@ object Defaults extends BuildCommon { if (plugin) scalaBase / ("sbt-" + sbtv) else scalaBase } - def compilersSetting = compilers := Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value, - bootIvyConfiguration.value, scalaCompilerBridgeSource.value)(appConfiguration.value, streams.value.log) + def compilersSetting = compilers := { + val compilers = Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value, + bootIvyConfiguration.value, scalaCompilerBridgeSource.value)(appConfiguration.value, streams.value.log) + if (!java.lang.Boolean.getBoolean("sbt.disable.interface.classloader.cache")) + compilers.scalac.setClassLoaderCache(state.value.classLoaderCache) + compilers + } lazy val configTasks = docTaskSettings(doc) ++ inTask(compile)(compileInputsSettings) ++ configGlobal ++ compileAnalysisSettings ++ Seq( compile := compileTask.value, diff --git a/util/classpath/src/main/scala/sbt/classpath/ClassLoaderCache.scala b/util/classpath/src/main/scala/sbt/classpath/ClassLoaderCache.scala index 2e489eb6f..7612ba265 100644 --- a/util/classpath/src/main/scala/sbt/classpath/ClassLoaderCache.scala +++ b/util/classpath/src/main/scala/sbt/classpath/ClassLoaderCache.scala @@ -15,25 +15,35 @@ private[sbt] final class ClassLoaderCache(val commonParent: ClassLoader) { * This method is thread-safe. */ def apply(files: List[File]): ClassLoader = synchronized { - val tstamps = files.map(_.lastModified) - getFromReference(files, tstamps, delegate.get(files)) + cachedCustomClassloader(files, () => new URLClassLoader(files.map(_.toURI.toURL).toArray, commonParent)) } - private[this] def getFromReference(files: List[File], stamps: List[Long], existingRef: Reference[CachedClassLoader]) = - if (existingRef eq null) - newEntry(files, stamps) - else - get(files, stamps, existingRef.get) + /** + * Returns a ClassLoader, as created by `mkLoader`. + * + * The returned ClassLoader may be cached from a previous call if the last modified time of all `files` is unchanged. + * This method is thread-safe. + */ + def cachedCustomClassloader(files: List[File], mkLoader: () => ClassLoader): ClassLoader = synchronized { + val tstamps = files.map(_.lastModified) + getFromReference(files, tstamps, delegate.get(files), mkLoader) + } - private[this] def get(files: List[File], stamps: List[Long], existing: CachedClassLoader): ClassLoader = + private[this] def getFromReference(files: List[File], stamps: List[Long], existingRef: Reference[CachedClassLoader], mkLoader: () => ClassLoader) = + if (existingRef eq null) + newEntry(files, stamps, mkLoader) + else + get(files, stamps, existingRef.get, mkLoader) + + private[this] def get(files: List[File], stamps: List[Long], existing: CachedClassLoader, mkLoader: () => ClassLoader): ClassLoader = if (existing == null || stamps != existing.timestamps) { - newEntry(files, stamps) + newEntry(files, stamps, mkLoader) } else existing.loader - private[this] def newEntry(files: List[File], stamps: List[Long]): ClassLoader = + private[this] def newEntry(files: List[File], stamps: List[Long], mkLoader: () => ClassLoader): ClassLoader = { - val loader = new URLClassLoader(files.map(_.toURI.toURL).toArray, commonParent) + val loader = mkLoader() delegate.put(files, new SoftReference(new CachedClassLoader(loader, files, stamps))) loader }