From 5fc584673705ea939b6d749124309093e45c6a35 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Mon, 26 Nov 2018 08:55:59 -0800 Subject: [PATCH] Add TaskRepository to manage ClassLoaderCache We want the user to be able to invalidate the classloader cache in the event that it somehow gets in a bad state. The cache is, however, defined in multiple configurations, so there are in fact many ClassLoaderCache instances that are managed by sbt. To make this sane, I add a global cache that is keyed by a TaskKey[_] and can return arbitrary data back. Invalidating all of the ClassLoaderCache instances is then as straightforward as just replacing the TaskRepository instance. I also went ahead and unified the management of the global file tree repository. Instead of having to specifically clear the file tree repository or the classloader cache, the user can now invalidate both with the new clearCaches command. --- .../main/scala/sbt/BasicCommandStrings.scala | 6 +-- .../src/main/scala/sbt/BasicKeys.scala | 1 + main/src/main/scala/sbt/Defaults.scala | 13 ++++--- main/src/main/scala/sbt/Keys.scala | 3 +- main/src/main/scala/sbt/Main.scala | 27 +++++++------ .../main/scala/sbt/internal/Repository.scala | 24 ++++++++++++ .../scala/sbt/internal/TaskRepository.scala | 38 +++++++++++++++++++ 7 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/TaskRepository.scala diff --git a/main-command/src/main/scala/sbt/BasicCommandStrings.scala b/main-command/src/main/scala/sbt/BasicCommandStrings.scala index cd760a313..5240a060f 100644 --- a/main-command/src/main/scala/sbt/BasicCommandStrings.scala +++ b/main-command/src/main/scala/sbt/BasicCommandStrings.scala @@ -233,8 +233,6 @@ $AliasCommand name= def continuousDetail: String = "Executes the specified command whenever source files change." def continuousBriefHelp: (String, String) = (ContinuousExecutePrefix + " ", continuousDetail) - def FlushFileTreeRepository: String = "flushFileTreeRepository" - def FlushDetailed: String = - "Resets the global file repository in the event that the repository has become inconsistent " + - "with the file system." + def ClearCaches: String = "clearCaches" + def ClearCachesDetailed: String = "Clears all of sbt's internal caches." } diff --git a/main-command/src/main/scala/sbt/BasicKeys.scala b/main-command/src/main/scala/sbt/BasicKeys.scala index 66de24ce6..0f71774f8 100644 --- a/main-command/src/main/scala/sbt/BasicKeys.scala +++ b/main-command/src/main/scala/sbt/BasicKeys.scala @@ -8,6 +8,7 @@ package sbt import java.io.File + import sbt.internal.util.AttributeKey import sbt.internal.inc.classpath.ClassLoaderCache import sbt.internal.server.ServerHandler diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index d782aee3e..54755ebee 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -142,7 +142,8 @@ object Defaults extends BuildCommon { excludeFilter :== HiddenFileFilter, classLoaderCache := ClassLoaderCache(4), layeringStrategy := LayeringStrategy.Default - ) ++ globalIvyCore ++ globalJvmCore + ) ++ TaskRepository + .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore ) ++ globalSbtCore private[sbt] lazy val globalJvmCore: Seq[Setting[_]] = @@ -1770,16 +1771,18 @@ object Defaults extends BuildCommon { Classpaths.addUnmanagedLibrary // We need a cache of size two for the test dependency layers (regular and snapshot). - lazy val testSettings - : Seq[Setting[_]] = configSettings ++ testTasks :+ (classLoaderCache := ClassLoaderCache(2)) + lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++ TaskRepository.proxy( + classLoaderCache, + ClassLoaderCache(2) + ) lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest)(testSettings) lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++ inConfig(Test)(testSettings) ++ inConfig(Runtime)( // We need a cache of size four so that the subset of the runtime dependencies that are used - // by the test task layers may be cached without evicting the runtime classloader layres. The + // by the test task layers may be cached without evicting the runtime classloader layers. The // cache size should be a multiple of two to support snapshot layers. - Classpaths.configSettings :+ (classLoaderCache := ClassLoaderCache(4)) + Classpaths.configSettings ++ TaskRepository.proxy(classLoaderCache, ClassLoaderCache(4)), ) // These are project level settings that MUST be on every project. diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 8f232d73b..65e619504 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -461,7 +461,8 @@ object Keys { val resolvedScoped = Def.resolvedScoped val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask) val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask) - val classLoaderCache = settingKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask) + val classLoaderCache = taskKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask) + private[sbt] val taskRepository = AttributeKey[TaskRepository.Repr]("task-repository", "A repository that can be used to cache arbitrary values for a given task key that can be read or filled during task evaluation.", 10000) // wrapper to work around SI-2915 private[sbt] final class TaskProgress(val progress: ExecuteProgress[Task]) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 2dfb6da25..0ae325509 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -214,7 +214,7 @@ object BuiltinCommands { BasicCommands.multi, act, continuous, - flushFileTreeRepository + clearCaches ) ++ allBasicCommands def DefaultBootCommands: Seq[String] = @@ -830,7 +830,7 @@ object BuiltinCommands { val session = Load.initialSession(structure, eval, s0) SessionSettings.checkSession(session, s) - registerGlobalFileRepository(Project.setProject(session, structure, s)) + registerGlobalCaches(Project.setProject(session, structure, s)) } def registerCompilerCache(s: State): State = { @@ -848,26 +848,31 @@ object BuiltinCommands { } s.put(Keys.stateCompilerCache, cache) } - def registerGlobalFileRepository(s: State): State = { + private[sbt] def registerGlobalCaches(s: State): State = { val extracted = Project.extract(s) try { - val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s) - val view: FileTreeDataView[StampedFile] = config.newDataView() - val newState = s.addExitHook { + def cleanup(): Unit = { s.get(BasicKeys.globalFileTreeView).foreach(_.close()) s.attributes.remove(BasicKeys.globalFileTreeView) + s.get(Keys.taskRepository).foreach(_.close()) + s.attributes.remove(Keys.taskRepository) () } - newState.get(BasicKeys.globalFileTreeView).foreach(_.close()) - newState.put(BasicKeys.globalFileTreeView, view) + val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s) + val view: FileTreeDataView[StampedFile] = config.newDataView() + val newState = s.addExitHook(cleanup()) + cleanup() + newState + .put(BasicKeys.globalFileTreeView, view) + .put(Keys.taskRepository, new TaskRepository.Repr) } catch { case NonFatal(_) => s } } - def flushFileTreeRepository: Command = { - val help = Help.more(FlushFileTreeRepository, FlushDetailed) - Command.command(FlushFileTreeRepository, help)(registerGlobalFileRepository) + def clearCaches: Command = { + val help = Help.more(ClearCaches, ClearCachesDetailed) + Command.command(ClearCaches, help)(registerGlobalCaches) } def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 => diff --git a/main/src/main/scala/sbt/internal/Repository.scala b/main/src/main/scala/sbt/internal/Repository.scala index 239dd02ef..63f9f230e 100644 --- a/main/src/main/scala/sbt/internal/Repository.scala +++ b/main/src/main/scala/sbt/internal/Repository.scala @@ -7,6 +7,10 @@ package sbt.internal +import java.util.concurrent.ConcurrentHashMap + +import scala.collection.JavaConverters._ + /** * Represents an abstract cache of values, accessible by a key. The interface is deliberately * minimal to give maximum flexibility to the implementation classes. For example, one can construct @@ -32,3 +36,23 @@ package sbt.internal trait Repository[M[_], K, V] extends AutoCloseable { def get(key: K): M[V] } + +private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] { + private[this] val map = new ConcurrentHashMap[K, V].asScala + override def get(key: K): Option[V] = map.get(key) + def put(key: K, value: V): Unit = this.synchronized { + map.put(key, value) + () + } + def remove(key: K): Unit = this.synchronized { + map.remove(key) + () + } + override def close(): Unit = this.synchronized { + map.foreach { + case (_, v: AutoCloseable) => v.close() + case _ => + } + map.clear() + } +} diff --git a/main/src/main/scala/sbt/internal/TaskRepository.scala b/main/src/main/scala/sbt/internal/TaskRepository.scala new file mode 100644 index 000000000..ada63172d --- /dev/null +++ b/main/src/main/scala/sbt/internal/TaskRepository.scala @@ -0,0 +1,38 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import sbt.Keys.state +import sbt._ + +private[sbt] object TaskRepository { + private[sbt] type Repr = MutableRepository[TaskKey[_], Any] + private[sbt] def proxy[T: Manifest](taskKey: TaskKey[T], task: => T): Def.Setting[Task[T]] = + proxy(taskKey, Def.task(task)) + private[sbt] def proxy[T: Manifest]( + taskKey: TaskKey[T], + task: Def.Initialize[Task[T]] + ): Def.Setting[Task[T]] = + taskKey := Def.taskDyn { + val taskRepository = state.value + .get(Keys.taskRepository) + .getOrElse { + val msg = "TaskRepository.proxy called before state was initialized" + throw new IllegalStateException(msg) + } + taskRepository.get(taskKey) match { + case Some(value: T) => Def.task(value) + case _ => + Def.task { + val value = task.value + taskRepository.put(taskKey, value) + value + } + } + }.value +}