luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use new ClassLoaderCache for layered classloaders 

This commit removes the ClassLoaderCache that I'd added for the purpose
of caching layered classloaders. Instead, we will use the state's global
ClassLoaderCache. This is better both because it centralizes the
classloader caching and because the new ClassLoaderCache will evict
unused classloaders when the jvm is under memory pressure.

I also add a new layer for the resources that goes between the scala
library layer and the dependency layer. This should help in cases where
users depend on libraries that require access to resources, e.g.
logback.xml.
This commit is contained in:
Ethan Atkins 2019-05-26 14:07:49 -07:00
parent 2268d91b47
commit af9f665649
8 changed files with 75 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
main/src/main/scala/sbt/Defaults.scala
View File

@ -149,7 +149,6 @@ object Defaults extends BuildCommon {
defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
excludeFilter :== HiddenFileFilter,
pathToFileStamp :== sbt.nio.FileStamp.hash,
classLoaderCache := ClassLoaderCache(4),
fileInputs :== Nil,
inputFileStamper :== sbt.nio.FileStamper.Hash,
outputFileStamper :== sbt.nio.FileStamper.LastModified,
@ -163,8 +162,7 @@ object Defaults extends BuildCommon {
.get(sbt.nio.Keys.persistentFileStampCache)
.getOrElse(new sbt.nio.FileStamp.Cache)
},
) ++ TaskRepository
.proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore
) ++ globalIvyCore ++ globalJvmCore
) ++ globalSbtCore
private[sbt] lazy val globalJvmCore: Seq[Setting[_]] =
@ -1868,13 +1866,6 @@ object Defaults extends BuildCommon {
configSettings ++ (mainBgRunMainTask +: mainBgRunTask) ++
Classpaths.addUnmanagedLibrary ++
Vector(
TaskRepository.proxy(
Compile / classLoaderCache,
// We need a cache of size four so that the subset of the runtime dependencies that are used
// by the test task layers may be cached without evicting the runtime classloader layers. The
// cache size should be a multiple of two to support snapshot layers.
ClassLoaderCache(4)
),
bgCopyClasspath in bgRun := {
val old = (bgCopyClasspath in bgRun).value
old && (Test / classLoaderLayeringStrategy).value != ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies
@ -1885,14 +1876,7 @@ object Defaults extends BuildCommon {
},
)
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++
Vector(
TaskRepository.proxy(
Test / classLoaderCache,
// We need a cache of size two for the test dependency layers (regular and snapshot).
ClassLoaderCache(2)
)
)
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks
lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest) {
testSettings

2
main/src/main/scala/sbt/Keys.scala
View File

@ -485,8 +485,6 @@ object Keys {
val resolvedScoped = Def.resolvedScoped
val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask)
val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask)
private[sbt] val classLoaderCache = taskKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask)
private[sbt] val taskRepository = AttributeKey[TaskRepository.Repr]("task-repository", "A repository that can be used to cache arbitrary values for a given task key that can be read or filled during task evaluation.", 10000)
private[sbt] val taskCancelStrategy = settingKey[State => TaskCancellationStrategy]("Experimental task cancellation handler.").withRank(DTask)
// Experimental in sbt 0.13.2 to enable grabbing semantic compile failures.

21
main/src/main/scala/sbt/Main.scala
View File

@ -828,7 +828,8 @@ object BuiltinCommands {
val session = Load.initialSession(structure, eval, s0)
SessionSettings.checkSession(session, s)
registerGlobalCaches(Project.setProject(session, structure, s))
Project
.setProject(session, structure, s)
.put(sbt.nio.Keys.hasCheckedMetaBuild, new AtomicBoolean(false))
}
@ -848,25 +849,11 @@ object BuiltinCommands {
}
s.put(Keys.stateCompilerCache, cache)
}
private[sbt] def registerGlobalCaches(s: State): State =
try {
val cleanedUp = new AtomicBoolean(false)
def cleanup(): Unit = {
s.get(Keys.taskRepository).foreach(_.close())
()
}
cleanup()
s.addExitHook(if (cleanedUp.compareAndSet(false, true)) cleanup())
.put(Keys.taskRepository, new TaskRepository.Repr)
} catch {
case NonFatal(_) => s
}
def clearCaches: Command = {
val help = Help.more(ClearCaches, ClearCachesDetailed)
Command.command(ClearCaches, help)(
registerGlobalCaches _ andThen registerCompilerCache andThen (_.initializeClassLoaderCache)
)
val f: State => State = registerCompilerCache _ andThen (_.initializeClassLoaderCache)
Command.command(ClearCaches, help)(f)
}
def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 =>

93
main/src/main/scala/sbt/internal/ClassLoaderCache.scala
View File

@ -1,93 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.io.File
import java.nio.file.Files
import sbt.internal.util.TypeFunctions.Id
import scala.annotation.tailrec
private[sbt] sealed trait ClassLoaderCache
extends Repository[Id, (Seq[File], ClassLoader, Map[String, String], File), ClassLoader]
private[sbt] object ClassLoaderCache {
private type Resources = Map[String, String]
private sealed trait CachedClassLoader extends ClassLoader {
def close(): Unit
}
private sealed trait StableClassLoader extends CachedClassLoader
private sealed trait SnapshotClassLoader extends CachedClassLoader
def apply(maxSize: Int): ClassLoaderCache = {
new ClassLoaderCache {
private final def mktmp(tmp: File): File =
if (maxSize > 0) Files.createTempDirectory("sbt-jni").toFile else tmp
private[this] val lruCache =
LRUCache[(JarClassPath, ClassLoader), (JarClassPath, CachedClassLoader)](
maxSize = maxSize,
onExpire =
(_: (JarClassPath, ClassLoader), v: (JarClassPath, CachedClassLoader)) => close(v._2)
)
override def get(info: (Seq[File], ClassLoader, Resources, File)): ClassLoader =
synchronized {
val (paths, parent, resources, tmp) = info
val key @ (keyJCP, _) = (new JarClassPath(paths), parent)
def addLoader(base: Option[StableClassLoader] = None): CachedClassLoader = {
val baseLoader = base.getOrElse {
if (keyJCP.regularJars.isEmpty) new ClassLoader(parent) with StableClassLoader {
override def close(): Unit = parent match {
case s: StableClassLoader => s.close()
case _ => ()
}
override def toString: String = parent.toString
} else
new LayeredClassLoader(keyJCP.regularJars, parent, resources, mktmp(tmp))
with StableClassLoader
}
val loader: CachedClassLoader =
if (keyJCP.snapshotJars.isEmpty) baseLoader
else
new LayeredClassLoader(keyJCP.snapshotJars, baseLoader, resources, mktmp(tmp))
with SnapshotClassLoader
lruCache.put(key, keyJCP -> loader)
loader
}
lruCache.get(key) match {
case Some((jcp, cl)) if keyJCP.strictEquals(jcp) => cl
case Some((_, cl: SnapshotClassLoader)) =>
cl.close()
cl.getParent match {
case p: StableClassLoader => addLoader(Some(p))
case _ => addLoader()
}
case None => addLoader()
}
}
override def close(): Unit = synchronized(lruCache.close())
override def toString: String = {
import PrettyPrint.indent
val cacheElements = lruCache.entries.map {
case ((jcp, parent), (_, l)) =>
s"(\n${indent(jcp, 4)},\n${indent(parent, 4)}\n) =>\n $l"
}
s"ClassLoaderCache(\n size = $maxSize,\n elements =\n${indent(cacheElements, 4)}\n)"
}
// Close the ClassLoader and all of it's closeable parents.
@tailrec
private def close(loader: CachedClassLoader): Unit = {
loader.close()
loader.getParent match {
case c: CachedClassLoader => close(c)
case _ => ()
}
}
}
}
}

100
main/src/main/scala/sbt/internal/ClassLoaders.scala
View File

@ -14,12 +14,13 @@ import java.net.URLClassLoader
import sbt.ClassLoaderLayeringStrategy._
import sbt.Keys._
import sbt.SlashSyntax0._
import sbt.internal.classpath.ClassLoaderCache
import sbt.internal.inc.ScalaInstance
import sbt.internal.inc.classpath.ClasspathUtilities
import sbt.internal.util.Attributed
import sbt.internal.util.Attributed.data
import sbt.io.IO
import sbt.librarymanagement.Configurations.{ Runtime, Test }
import sbt.librarymanagement.Configurations.Runtime
private[sbt] object ClassLoaders {
private[this] val interfaceLoader = classOf[sbt.testing.Framework].getClassLoader
@ -38,9 +39,7 @@ private[sbt] object ClassLoaders {
rawRuntimeDependencies =
dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude),
allDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude),
globalCache = (Scope.GlobalScope / classLoaderCache).value,
runtimeCache = (Runtime / classLoaderCache).value,
testCache = (Test / classLoaderCache).value,
cache = extendedClassLoaderCache.value,
resources = ClasspathUtilities.createClasspathResources(fullCP, si),
tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value),
scope = resolvedScoped.value.scope
@ -72,9 +71,6 @@ private[sbt] object ClassLoaders {
)
s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false")
}
val globalCache = (Scope.GlobalScope / classLoaderCache).value
val runtimeCache = (Runtime / classLoaderCache).value
val testCache = (Test / classLoaderCache).value
val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars
val runtimeDeps = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude)
val allDeps = dependencyJars(dependencyClasspath).value.filterNot(exclude)
@ -86,9 +82,7 @@ private[sbt] object ClassLoaders {
fullCP = classpath,
rawRuntimeDependencies = runtimeDeps,
allDependencies = allDeps,
globalCache = globalCache,
runtimeCache = runtimeCache,
testCache = testCache,
cache = extendedClassLoaderCache.value: @sbtUnchecked,
resources = ClasspathUtilities.createClasspathResources(classpath, instance),
tmp = taskTemporaryDirectory.value: @sbtUnchecked,
scope = resolvedScope
@ -99,12 +93,19 @@ private[sbt] object ClassLoaders {
}
}
private[this] def extendedClassLoaderCache: Def.Initialize[Task[ClassLoaderCache]] = Def.task {
val errorMessage = "Tried to extract classloader cache for uninitialized state."
state.value
.get(BasicKeys.extendedClassLoaderCache)
.getOrElse(throw new IllegalStateException(errorMessage))
}
/*
* Create a layered classloader. There are up to four layers:
* Create a layered classloader. There are up to five layers:
* 1) the scala instance class loader
* 2) the runtime dependencies
* 3) the test dependencies
* 4) the rest of the classpath
* 2) the resource layer
* 3) the runtime dependencies
* 4) the test dependencies
* 5) the rest of the classpath
* The first two layers may be optionally cached to reduce memory usage and improve
* start up latency. Because there may be mutually incompatible libraries in the runtime
* and test dependencies, it's important to be able to configure which layers are used.
@ -115,9 +116,7 @@ private[sbt] object ClassLoaders {
fullCP: Seq[File],
rawRuntimeDependencies: Seq[File],
allDependencies: Seq[File],
globalCache: ClassLoaderCache,
runtimeCache: ClassLoaderCache,
testCache: ClassLoaderCache,
cache: ClassLoaderCache,
resources: Map[String, String],
tmp: File,
scope: Scope
@ -145,25 +144,29 @@ private[sbt] object ClassLoaders {
val allTestDependencies = if (layerTestDependencies) allDependenciesSet else Set.empty[File]
val allRuntimeDependencies = (if (layerDependencies) rawRuntimeDependencies else Nil).toSet
val scalaLibrarySet = Set(si.libraryJar)
val scalaLibraryLayer =
globalCache.get((scalaLibrarySet.toList, interfaceLoader, resources, tmp))
// layer 2
val scalaLibraryLayer = layer(si.libraryJar :: Nil, interfaceLoader, cache, resources, tmp)
// layer 2 (resources)
val resourceLayer =
if (layerDependencies) getResourceLayer(fullCP, scalaLibraryLayer, cache, resources)
else scalaLibraryLayer
// layer 3 (optional if in the test config and the runtime layer is not shared)
val runtimeDependencySet = allDependenciesSet intersect allRuntimeDependencies
val runtimeDependencies = rawRuntimeDependencies.filter(runtimeDependencySet)
lazy val runtimeLayer =
if (layerDependencies)
layer(runtimeDependencies, scalaLibraryLayer, runtimeCache, resources, tmp)
else scalaLibraryLayer
layer(runtimeDependencies, resourceLayer, cache, resources, tmp)
else resourceLayer
// layer 3 (optional if testDependencies are empty)
// layer 4 (optional if testDependencies are empty)
val testDependencySet = allTestDependencies diff runtimeDependencySet
val testDependencies = allDependencies.filter(testDependencySet)
val testLayer = layer(testDependencies, runtimeLayer, testCache, resources, tmp)
val testLayer = layer(testDependencies, runtimeLayer, cache, resources, tmp)
// layer 4
// layer 5
val dynamicClasspath =
fullCP.filterNot(testDependencySet ++ runtimeDependencies ++ scalaLibrarySet)
fullCP.filterNot(testDependencySet ++ runtimeDependencies + si.libraryJar)
if (dynamicClasspath.nonEmpty)
new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp)
else testLayer
@ -186,9 +189,45 @@ private[sbt] object ClassLoaders {
resources: Map[String, String],
tmp: File
): ClassLoader = {
val (snapshots, jars) = classpath.partition(_.toString.contains("-SNAPSHOT"))
val jarLoader = if (jars.isEmpty) parent else cache.get((jars, parent, resources, tmp))
if (snapshots.isEmpty) jarLoader else cache.get((snapshots, jarLoader, resources, tmp))
if (classpath.nonEmpty) {
cache(
classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
parent,
() => new LayeredClassLoader(classpath, parent, resources, tmp)
)
} else parent
}
private class ResourceLoader(
classpath: Seq[File],
parent: ClassLoader,
resources: Map[String, String]
) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) {
override def loadClass(name: String, resolve: Boolean): Class[_] = {
val clazz = parent.loadClass(name)
if (resolve) resolveClass(clazz)
clazz
}
override def toString: String = "ResourceLoader"
}
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
// invalidating the regular jar layer. If the classpath is empty, it just returns the parent
// loader.
private def getResourceLayer(
classpath: Seq[File],
parent: ClassLoader,
cache: ClassLoaderCache,
resources: Map[String, String]
): ClassLoader = {
if (classpath.nonEmpty) {
cache(
classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
parent,
() => new ResourceLoader(classpath, parent, resources)
)
} else parent
}
// helper methods
@ -197,5 +236,4 @@ private[sbt] object ClassLoaders {
override def toString: String =
s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)"
}
}

113
main/src/main/scala/sbt/internal/LRUCache.scala
View File

@ -1,113 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.util.concurrent.atomic.AtomicInteger
import scala.annotation.tailrec
private[sbt] sealed trait LRUCache[K, V] extends AutoCloseable {
def get(key: K): Option[V]
def entries: Seq[(K, V)]
def maxSize: Int
def put(key: K, value: V): Option[V]
def remove(key: K): Option[V]
def size: Int
}
private[sbt] object LRUCache {
private[this] class impl[K, V](override val maxSize: Int, onExpire: Option[((K, V)) => Unit])
extends LRUCache[K, V] {
private[this] val elementsSortedByAccess: Array[(K, V)] = new Array[(K, V)](maxSize)
private[this] val lastIndex: AtomicInteger = new AtomicInteger(-1)
override def close(): Unit = this.synchronized {
val f = onExpire.getOrElse((_: (K, V)) => Unit)
0 until maxSize foreach { i =>
elementsSortedByAccess(i) match {
case null =>
case el => f(el)
}
elementsSortedByAccess(i) = null
}
lastIndex.set(-1)
}
override def entries: Seq[(K, V)] = this.synchronized {
(0 to lastIndex.get()).map(elementsSortedByAccess)
}
override def get(key: K): Option[V] = this.synchronized {
indexOf(key) match {
case -1 => None
case i => replace(i, key, elementsSortedByAccess(i)._2)
}
}
override def put(key: K, value: V): Option[V] = this.synchronized {
indexOf(key) match {
case -1 =>
append(key, value)
None
case i => replace(i, key, value)
}
}
override def remove(key: K): Option[V] = this.synchronized {
indexOf(key) match {
case -1 => None
case i => remove(i, lastIndex.get, expire = false)
}
}
override def size: Int = lastIndex.get + 1
override def toString: String = {
val values = 0 to lastIndex.get() map { i =>
val (key, value) = elementsSortedByAccess(i)
s"$key -> $value"
}
s"LRUCache(${values mkString ", "})"
}
private def indexOf(key: K): Int =
elementsSortedByAccess.view.take(lastIndex.get() + 1).indexWhere(_._1 == key)
private def replace(index: Int, key: K, value: V): Option[V] = {
val prev = remove(index, lastIndex.get(), expire = false)
append(key, value)
prev
}
private def append(key: K, value: V): Unit = {
while (lastIndex.get() >= maxSize - 1) {
remove(0, lastIndex.get(), expire = true)
}
val index = lastIndex.incrementAndGet()
elementsSortedByAccess(index) = (key, value)
}
private def remove(index: Int, endIndex: Int, expire: Boolean): Option[V] = {
@tailrec
def shift(i: Int): Unit = if (i < endIndex) {
elementsSortedByAccess(i) = elementsSortedByAccess(i + 1)
shift(i + 1)
}
val prev = elementsSortedByAccess(index)
shift(index)
lastIndex.set(endIndex - 1)
if (expire) onExpire.foreach(f => f(prev))
Some(prev._2)
}
}
private def emptyCache[K, V]: LRUCache[K, V] = new LRUCache[K, V] {
override def get(key: K): Option[V] = None
override def entries: Seq[(K, V)] = Nil
override def maxSize: Int = 0
override def put(key: K, value: V): Option[V] = None
override def remove(key: K): Option[V] = None
override def size: Int = 0
override def close(): Unit = {}
override def toString = "EmptyLRUCache"
}
def apply[K, V](maxSize: Int): LRUCache[K, V] =
if (maxSize > 0) new impl(maxSize, None) else emptyCache
def apply[K, V](maxSize: Int, onExpire: (K, V) => Unit): LRUCache[K, V] =
if (maxSize > 0) new impl(maxSize, Some(onExpire.tupled)) else emptyCache
}

59
main/src/main/scala/sbt/internal/Repository.scala
View File

@ -1,59 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.util.concurrent.ConcurrentHashMap
import scala.collection.JavaConverters._
/**
* Represents an abstract cache of values, accessible by a key. The interface is deliberately
* minimal to give maximum flexibility to the implementation classes. For example, one can construct
* a cache from a `sbt.io.FileTreeRepository` that automatically registers the paths with the
* cache (but does not clear the cache on close):
* {{{
* val repository = sbt.io.FileTreeRepository.default(_.getPath)
* val fileCache = new Repository[Seq, (Path, Boolean), TypedPath] {
* override def get(key: (Path, Boolean)): Seq[TypedPath] = {
* val (path, recursive) = key
* val depth = if (recursive) Int.MaxValue else 0
* repository.register(path, depth)
* repository.list(path, depth, AllPass)
* }
* override def close(): Unit = {}
* }
* }}}
*
* @tparam M the container type of the cache. This will most commonly be `Option` or `Seq`.
* @tparam K the key type
* @tparam V the value type
*/
private[sbt] trait Repository[M[_], K, V] extends AutoCloseable {
def get(key: K): M[V]
override def close(): Unit = {}
}
private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] {
private[this] val map = new ConcurrentHashMap[K, V].asScala
override def get(key: K): Option[V] = map.get(key)
def put(key: K, value: V): Unit = this.synchronized {
map.put(key, value)
()
}
def remove(key: K): Unit = this.synchronized {
map.remove(key)
()
}
override def close(): Unit = this.synchronized {
map.foreach {
case (_, v: AutoCloseable) => v.close()
case _ =>
}
map.clear()
}
}

38
main/src/main/scala/sbt/internal/TaskRepository.scala
View File

@ -1,38 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import sbt.Keys.state
import sbt._
private[sbt] object TaskRepository {
private[sbt] type Repr = MutableRepository[TaskKey[_], Any]
private[sbt] def proxy[T: Manifest](taskKey: TaskKey[T], task: => T): Def.Setting[Task[T]] =
proxy(taskKey, Def.task(task))
private[sbt] def proxy[T: Manifest](
taskKey: TaskKey[T],
task: Def.Initialize[Task[T]]
): Def.Setting[Task[T]] =
taskKey := Def.taskDyn {
val taskRepository = state.value
.get(Keys.taskRepository)
.getOrElse {
val msg = "TaskRepository.proxy called before state was initialized"
throw new IllegalStateException(msg)
}
taskRepository.get(taskKey) match {
case Some(value: T) => Def.task(value)
case _ =>
Def.task {
val value = task.value
taskRepository.put(taskKey, value)
value
}
}
}.value
}