diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 9214867bb..f8774dff9 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -149,7 +149,6 @@ object Defaults extends BuildCommon { defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq( excludeFilter :== HiddenFileFilter, pathToFileStamp :== sbt.nio.FileStamp.hash, - classLoaderCache := ClassLoaderCache(4), fileInputs :== Nil, inputFileStamper :== sbt.nio.FileStamper.Hash, outputFileStamper :== sbt.nio.FileStamper.LastModified, @@ -163,8 +162,7 @@ object Defaults extends BuildCommon { .get(sbt.nio.Keys.persistentFileStampCache) .getOrElse(new sbt.nio.FileStamp.Cache) }, - ) ++ TaskRepository - .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore + ) ++ globalIvyCore ++ globalJvmCore ) ++ globalSbtCore private[sbt] lazy val globalJvmCore: Seq[Setting[_]] = @@ -1868,13 +1866,6 @@ object Defaults extends BuildCommon { configSettings ++ (mainBgRunMainTask +: mainBgRunTask) ++ Classpaths.addUnmanagedLibrary ++ Vector( - TaskRepository.proxy( - Compile / classLoaderCache, - // We need a cache of size four so that the subset of the runtime dependencies that are used - // by the test task layers may be cached without evicting the runtime classloader layers. The - // cache size should be a multiple of two to support snapshot layers. - ClassLoaderCache(4) - ), bgCopyClasspath in bgRun := { val old = (bgCopyClasspath in bgRun).value old && (Test / classLoaderLayeringStrategy).value != ClassLoaderLayeringStrategy.ShareRuntimeDependenciesLayerWithTestDependencies @@ -1885,14 +1876,7 @@ object Defaults extends BuildCommon { }, ) - lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++ - Vector( - TaskRepository.proxy( - Test / classLoaderCache, - // We need a cache of size two for the test dependency layers (regular and snapshot). - ClassLoaderCache(2) - ) - ) + lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest) { testSettings diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index fcf162d7a..467af30ad 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -485,8 +485,6 @@ object Keys { val resolvedScoped = Def.resolvedScoped val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask) val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask) - private[sbt] val classLoaderCache = taskKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask) - private[sbt] val taskRepository = AttributeKey[TaskRepository.Repr]("task-repository", "A repository that can be used to cache arbitrary values for a given task key that can be read or filled during task evaluation.", 10000) private[sbt] val taskCancelStrategy = settingKey[State => TaskCancellationStrategy]("Experimental task cancellation handler.").withRank(DTask) // Experimental in sbt 0.13.2 to enable grabbing semantic compile failures. diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index e0f712a62..54b8b5216 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -828,7 +828,8 @@ object BuiltinCommands { val session = Load.initialSession(structure, eval, s0) SessionSettings.checkSession(session, s) - registerGlobalCaches(Project.setProject(session, structure, s)) + Project + .setProject(session, structure, s) .put(sbt.nio.Keys.hasCheckedMetaBuild, new AtomicBoolean(false)) } @@ -848,25 +849,11 @@ object BuiltinCommands { } s.put(Keys.stateCompilerCache, cache) } - private[sbt] def registerGlobalCaches(s: State): State = - try { - val cleanedUp = new AtomicBoolean(false) - def cleanup(): Unit = { - s.get(Keys.taskRepository).foreach(_.close()) - () - } - cleanup() - s.addExitHook(if (cleanedUp.compareAndSet(false, true)) cleanup()) - .put(Keys.taskRepository, new TaskRepository.Repr) - } catch { - case NonFatal(_) => s - } def clearCaches: Command = { val help = Help.more(ClearCaches, ClearCachesDetailed) - Command.command(ClearCaches, help)( - registerGlobalCaches _ andThen registerCompilerCache andThen (_.initializeClassLoaderCache) - ) + val f: State => State = registerCompilerCache _ andThen (_.initializeClassLoaderCache) + Command.command(ClearCaches, help)(f) } def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 => diff --git a/main/src/main/scala/sbt/internal/ClassLoaderCache.scala b/main/src/main/scala/sbt/internal/ClassLoaderCache.scala deleted file mode 100644 index 90f4ea53b..000000000 --- a/main/src/main/scala/sbt/internal/ClassLoaderCache.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import java.io.File -import java.nio.file.Files - -import sbt.internal.util.TypeFunctions.Id - -import scala.annotation.tailrec - -private[sbt] sealed trait ClassLoaderCache - extends Repository[Id, (Seq[File], ClassLoader, Map[String, String], File), ClassLoader] - -private[sbt] object ClassLoaderCache { - private type Resources = Map[String, String] - private sealed trait CachedClassLoader extends ClassLoader { - def close(): Unit - } - private sealed trait StableClassLoader extends CachedClassLoader - private sealed trait SnapshotClassLoader extends CachedClassLoader - def apply(maxSize: Int): ClassLoaderCache = { - new ClassLoaderCache { - private final def mktmp(tmp: File): File = - if (maxSize > 0) Files.createTempDirectory("sbt-jni").toFile else tmp - private[this] val lruCache = - LRUCache[(JarClassPath, ClassLoader), (JarClassPath, CachedClassLoader)]( - maxSize = maxSize, - onExpire = - (_: (JarClassPath, ClassLoader), v: (JarClassPath, CachedClassLoader)) => close(v._2) - ) - override def get(info: (Seq[File], ClassLoader, Resources, File)): ClassLoader = - synchronized { - val (paths, parent, resources, tmp) = info - val key @ (keyJCP, _) = (new JarClassPath(paths), parent) - def addLoader(base: Option[StableClassLoader] = None): CachedClassLoader = { - val baseLoader = base.getOrElse { - if (keyJCP.regularJars.isEmpty) new ClassLoader(parent) with StableClassLoader { - override def close(): Unit = parent match { - case s: StableClassLoader => s.close() - case _ => () - } - override def toString: String = parent.toString - } else - new LayeredClassLoader(keyJCP.regularJars, parent, resources, mktmp(tmp)) - with StableClassLoader - } - val loader: CachedClassLoader = - if (keyJCP.snapshotJars.isEmpty) baseLoader - else - new LayeredClassLoader(keyJCP.snapshotJars, baseLoader, resources, mktmp(tmp)) - with SnapshotClassLoader - lruCache.put(key, keyJCP -> loader) - loader - } - lruCache.get(key) match { - case Some((jcp, cl)) if keyJCP.strictEquals(jcp) => cl - case Some((_, cl: SnapshotClassLoader)) => - cl.close() - cl.getParent match { - case p: StableClassLoader => addLoader(Some(p)) - case _ => addLoader() - } - case None => addLoader() - } - } - override def close(): Unit = synchronized(lruCache.close()) - override def toString: String = { - import PrettyPrint.indent - val cacheElements = lruCache.entries.map { - case ((jcp, parent), (_, l)) => - s"(\n${indent(jcp, 4)},\n${indent(parent, 4)}\n) =>\n $l" - } - s"ClassLoaderCache(\n size = $maxSize,\n elements =\n${indent(cacheElements, 4)}\n)" - } - - // Close the ClassLoader and all of it's closeable parents. - @tailrec - private def close(loader: CachedClassLoader): Unit = { - loader.close() - loader.getParent match { - case c: CachedClassLoader => close(c) - case _ => () - } - } - } - } -} diff --git a/main/src/main/scala/sbt/internal/ClassLoaders.scala b/main/src/main/scala/sbt/internal/ClassLoaders.scala index 51d19204e..396d2da31 100644 --- a/main/src/main/scala/sbt/internal/ClassLoaders.scala +++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala @@ -14,12 +14,13 @@ import java.net.URLClassLoader import sbt.ClassLoaderLayeringStrategy._ import sbt.Keys._ import sbt.SlashSyntax0._ +import sbt.internal.classpath.ClassLoaderCache import sbt.internal.inc.ScalaInstance import sbt.internal.inc.classpath.ClasspathUtilities import sbt.internal.util.Attributed import sbt.internal.util.Attributed.data import sbt.io.IO -import sbt.librarymanagement.Configurations.{ Runtime, Test } +import sbt.librarymanagement.Configurations.Runtime private[sbt] object ClassLoaders { private[this] val interfaceLoader = classOf[sbt.testing.Framework].getClassLoader @@ -38,9 +39,7 @@ private[sbt] object ClassLoaders { rawRuntimeDependencies = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude), allDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude), - globalCache = (Scope.GlobalScope / classLoaderCache).value, - runtimeCache = (Runtime / classLoaderCache).value, - testCache = (Test / classLoaderCache).value, + cache = extendedClassLoaderCache.value, resources = ClasspathUtilities.createClasspathResources(fullCP, si), tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value), scope = resolvedScoped.value.scope @@ -72,9 +71,6 @@ private[sbt] object ClassLoaders { ) s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false") } - val globalCache = (Scope.GlobalScope / classLoaderCache).value - val runtimeCache = (Runtime / classLoaderCache).value - val testCache = (Test / classLoaderCache).value val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars val runtimeDeps = dependencyJars(Runtime / dependencyClasspath).value.filterNot(exclude) val allDeps = dependencyJars(dependencyClasspath).value.filterNot(exclude) @@ -86,9 +82,7 @@ private[sbt] object ClassLoaders { fullCP = classpath, rawRuntimeDependencies = runtimeDeps, allDependencies = allDeps, - globalCache = globalCache, - runtimeCache = runtimeCache, - testCache = testCache, + cache = extendedClassLoaderCache.value: @sbtUnchecked, resources = ClasspathUtilities.createClasspathResources(classpath, instance), tmp = taskTemporaryDirectory.value: @sbtUnchecked, scope = resolvedScope @@ -99,12 +93,19 @@ private[sbt] object ClassLoaders { } } + private[this] def extendedClassLoaderCache: Def.Initialize[Task[ClassLoaderCache]] = Def.task { + val errorMessage = "Tried to extract classloader cache for uninitialized state." + state.value + .get(BasicKeys.extendedClassLoaderCache) + .getOrElse(throw new IllegalStateException(errorMessage)) + } /* - * Create a layered classloader. There are up to four layers: + * Create a layered classloader. There are up to five layers: * 1) the scala instance class loader - * 2) the runtime dependencies - * 3) the test dependencies - * 4) the rest of the classpath + * 2) the resource layer + * 3) the runtime dependencies + * 4) the test dependencies + * 5) the rest of the classpath * The first two layers may be optionally cached to reduce memory usage and improve * start up latency. Because there may be mutually incompatible libraries in the runtime * and test dependencies, it's important to be able to configure which layers are used. @@ -115,9 +116,7 @@ private[sbt] object ClassLoaders { fullCP: Seq[File], rawRuntimeDependencies: Seq[File], allDependencies: Seq[File], - globalCache: ClassLoaderCache, - runtimeCache: ClassLoaderCache, - testCache: ClassLoaderCache, + cache: ClassLoaderCache, resources: Map[String, String], tmp: File, scope: Scope @@ -145,25 +144,29 @@ private[sbt] object ClassLoaders { val allTestDependencies = if (layerTestDependencies) allDependenciesSet else Set.empty[File] val allRuntimeDependencies = (if (layerDependencies) rawRuntimeDependencies else Nil).toSet - val scalaLibrarySet = Set(si.libraryJar) - val scalaLibraryLayer = - globalCache.get((scalaLibrarySet.toList, interfaceLoader, resources, tmp)) - // layer 2 + val scalaLibraryLayer = layer(si.libraryJar :: Nil, interfaceLoader, cache, resources, tmp) + + // layer 2 (resources) + val resourceLayer = + if (layerDependencies) getResourceLayer(fullCP, scalaLibraryLayer, cache, resources) + else scalaLibraryLayer + + // layer 3 (optional if in the test config and the runtime layer is not shared) val runtimeDependencySet = allDependenciesSet intersect allRuntimeDependencies val runtimeDependencies = rawRuntimeDependencies.filter(runtimeDependencySet) lazy val runtimeLayer = if (layerDependencies) - layer(runtimeDependencies, scalaLibraryLayer, runtimeCache, resources, tmp) - else scalaLibraryLayer + layer(runtimeDependencies, resourceLayer, cache, resources, tmp) + else resourceLayer - // layer 3 (optional if testDependencies are empty) + // layer 4 (optional if testDependencies are empty) val testDependencySet = allTestDependencies diff runtimeDependencySet val testDependencies = allDependencies.filter(testDependencySet) - val testLayer = layer(testDependencies, runtimeLayer, testCache, resources, tmp) + val testLayer = layer(testDependencies, runtimeLayer, cache, resources, tmp) - // layer 4 + // layer 5 val dynamicClasspath = - fullCP.filterNot(testDependencySet ++ runtimeDependencies ++ scalaLibrarySet) + fullCP.filterNot(testDependencySet ++ runtimeDependencies + si.libraryJar) if (dynamicClasspath.nonEmpty) new LayeredClassLoader(dynamicClasspath, testLayer, resources, tmp) else testLayer @@ -186,9 +189,45 @@ private[sbt] object ClassLoaders { resources: Map[String, String], tmp: File ): ClassLoader = { - val (snapshots, jars) = classpath.partition(_.toString.contains("-SNAPSHOT")) - val jarLoader = if (jars.isEmpty) parent else cache.get((jars, parent, resources, tmp)) - if (snapshots.isEmpty) jarLoader else cache.get((snapshots, jarLoader, resources, tmp)) + if (classpath.nonEmpty) { + cache( + classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)), + parent, + () => new LayeredClassLoader(classpath, parent, resources, tmp) + ) + } else parent + } + + private class ResourceLoader( + classpath: Seq[File], + parent: ClassLoader, + resources: Map[String, String] + ) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) { + override def loadClass(name: String, resolve: Boolean): Class[_] = { + val clazz = parent.loadClass(name) + if (resolve) resolveClass(clazz) + clazz + } + override def toString: String = "ResourceLoader" + } + // Creates a one or two layered classloader for the provided classpaths depending on whether + // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer + // above the regular jar layer. This allows the snapshot layer to be invalidated without + // invalidating the regular jar layer. If the classpath is empty, it just returns the parent + // loader. + private def getResourceLayer( + classpath: Seq[File], + parent: ClassLoader, + cache: ClassLoaderCache, + resources: Map[String, String] + ): ClassLoader = { + if (classpath.nonEmpty) { + cache( + classpath.toList.map(f => f -> IO.getModifiedTimeOrZero(f)), + parent, + () => new ResourceLoader(classpath, parent, resources) + ) + } else parent } // helper methods @@ -197,5 +236,4 @@ private[sbt] object ClassLoaders { override def toString: String = s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)" } - } diff --git a/main/src/main/scala/sbt/internal/LRUCache.scala b/main/src/main/scala/sbt/internal/LRUCache.scala deleted file mode 100644 index a80a2837a..000000000 --- a/main/src/main/scala/sbt/internal/LRUCache.scala +++ /dev/null @@ -1,113 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import java.util.concurrent.atomic.AtomicInteger - -import scala.annotation.tailrec - -private[sbt] sealed trait LRUCache[K, V] extends AutoCloseable { - def get(key: K): Option[V] - def entries: Seq[(K, V)] - def maxSize: Int - def put(key: K, value: V): Option[V] - def remove(key: K): Option[V] - def size: Int -} - -private[sbt] object LRUCache { - private[this] class impl[K, V](override val maxSize: Int, onExpire: Option[((K, V)) => Unit]) - extends LRUCache[K, V] { - private[this] val elementsSortedByAccess: Array[(K, V)] = new Array[(K, V)](maxSize) - private[this] val lastIndex: AtomicInteger = new AtomicInteger(-1) - - override def close(): Unit = this.synchronized { - val f = onExpire.getOrElse((_: (K, V)) => Unit) - 0 until maxSize foreach { i => - elementsSortedByAccess(i) match { - case null => - case el => f(el) - } - elementsSortedByAccess(i) = null - } - lastIndex.set(-1) - } - override def entries: Seq[(K, V)] = this.synchronized { - (0 to lastIndex.get()).map(elementsSortedByAccess) - } - override def get(key: K): Option[V] = this.synchronized { - indexOf(key) match { - case -1 => None - case i => replace(i, key, elementsSortedByAccess(i)._2) - } - } - override def put(key: K, value: V): Option[V] = this.synchronized { - indexOf(key) match { - case -1 => - append(key, value) - None - case i => replace(i, key, value) - } - } - override def remove(key: K): Option[V] = this.synchronized { - indexOf(key) match { - case -1 => None - case i => remove(i, lastIndex.get, expire = false) - } - } - override def size: Int = lastIndex.get + 1 - override def toString: String = { - val values = 0 to lastIndex.get() map { i => - val (key, value) = elementsSortedByAccess(i) - s"$key -> $value" - } - s"LRUCache(${values mkString ", "})" - } - - private def indexOf(key: K): Int = - elementsSortedByAccess.view.take(lastIndex.get() + 1).indexWhere(_._1 == key) - private def replace(index: Int, key: K, value: V): Option[V] = { - val prev = remove(index, lastIndex.get(), expire = false) - append(key, value) - prev - } - private def append(key: K, value: V): Unit = { - while (lastIndex.get() >= maxSize - 1) { - remove(0, lastIndex.get(), expire = true) - } - val index = lastIndex.incrementAndGet() - elementsSortedByAccess(index) = (key, value) - } - private def remove(index: Int, endIndex: Int, expire: Boolean): Option[V] = { - @tailrec - def shift(i: Int): Unit = if (i < endIndex) { - elementsSortedByAccess(i) = elementsSortedByAccess(i + 1) - shift(i + 1) - } - val prev = elementsSortedByAccess(index) - shift(index) - lastIndex.set(endIndex - 1) - if (expire) onExpire.foreach(f => f(prev)) - Some(prev._2) - } - } - private def emptyCache[K, V]: LRUCache[K, V] = new LRUCache[K, V] { - override def get(key: K): Option[V] = None - override def entries: Seq[(K, V)] = Nil - override def maxSize: Int = 0 - override def put(key: K, value: V): Option[V] = None - override def remove(key: K): Option[V] = None - override def size: Int = 0 - override def close(): Unit = {} - override def toString = "EmptyLRUCache" - } - def apply[K, V](maxSize: Int): LRUCache[K, V] = - if (maxSize > 0) new impl(maxSize, None) else emptyCache - def apply[K, V](maxSize: Int, onExpire: (K, V) => Unit): LRUCache[K, V] = - if (maxSize > 0) new impl(maxSize, Some(onExpire.tupled)) else emptyCache -} diff --git a/main/src/main/scala/sbt/internal/Repository.scala b/main/src/main/scala/sbt/internal/Repository.scala deleted file mode 100644 index 7d0b0f82f..000000000 --- a/main/src/main/scala/sbt/internal/Repository.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import java.util.concurrent.ConcurrentHashMap - -import scala.collection.JavaConverters._ - -/** - * Represents an abstract cache of values, accessible by a key. The interface is deliberately - * minimal to give maximum flexibility to the implementation classes. For example, one can construct - * a cache from a `sbt.io.FileTreeRepository` that automatically registers the paths with the - * cache (but does not clear the cache on close): - * {{{ - * val repository = sbt.io.FileTreeRepository.default(_.getPath) - * val fileCache = new Repository[Seq, (Path, Boolean), TypedPath] { - * override def get(key: (Path, Boolean)): Seq[TypedPath] = { - * val (path, recursive) = key - * val depth = if (recursive) Int.MaxValue else 0 - * repository.register(path, depth) - * repository.list(path, depth, AllPass) - * } - * override def close(): Unit = {} - * } - * }}} - * - * @tparam M the container type of the cache. This will most commonly be `Option` or `Seq`. - * @tparam K the key type - * @tparam V the value type - */ -private[sbt] trait Repository[M[_], K, V] extends AutoCloseable { - def get(key: K): M[V] - override def close(): Unit = {} -} - -private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] { - private[this] val map = new ConcurrentHashMap[K, V].asScala - override def get(key: K): Option[V] = map.get(key) - def put(key: K, value: V): Unit = this.synchronized { - map.put(key, value) - () - } - def remove(key: K): Unit = this.synchronized { - map.remove(key) - () - } - override def close(): Unit = this.synchronized { - map.foreach { - case (_, v: AutoCloseable) => v.close() - case _ => - } - map.clear() - } -} diff --git a/main/src/main/scala/sbt/internal/TaskRepository.scala b/main/src/main/scala/sbt/internal/TaskRepository.scala deleted file mode 100644 index ada63172d..000000000 --- a/main/src/main/scala/sbt/internal/TaskRepository.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import sbt.Keys.state -import sbt._ - -private[sbt] object TaskRepository { - private[sbt] type Repr = MutableRepository[TaskKey[_], Any] - private[sbt] def proxy[T: Manifest](taskKey: TaskKey[T], task: => T): Def.Setting[Task[T]] = - proxy(taskKey, Def.task(task)) - private[sbt] def proxy[T: Manifest]( - taskKey: TaskKey[T], - task: Def.Initialize[Task[T]] - ): Def.Setting[Task[T]] = - taskKey := Def.taskDyn { - val taskRepository = state.value - .get(Keys.taskRepository) - .getOrElse { - val msg = "TaskRepository.proxy called before state was initialized" - throw new IllegalStateException(msg) - } - taskRepository.get(taskKey) match { - case Some(value: T) => Def.task(value) - case _ => - Def.task { - val value = task.value - taskRepository.put(taskKey, value) - value - } - } - }.value -}