Add TaskRepository to manage ClassLoaderCache

We want the user to be able to invalidate the classloader cache in the
event that it somehow gets in a bad state. The cache is, however,
defined in multiple configurations, so there are in fact many
ClassLoaderCache instances that are managed by sbt. To make this sane, I
add a global cache that is keyed by a TaskKey[_] and can return
arbitrary data back. Invalidating all of the ClassLoaderCache instances
is then as straightforward as just replacing the TaskRepository
instance.

I also went ahead and unified the management of the global file tree
repository. Instead of having to specifically clear the file tree
repository or the classloader cache, the user can now invalidate both
with the new clearCaches command.

main-command/src/main/scala/sbt/BasicCommandStrings.scala

@@ -233,8 +233,6 @@ $AliasCommand name=
   def continuousDetail: String = "Executes the specified command whenever source files change."
   def continuousBriefHelp: (String, String) =
     (ContinuousExecutePrefix + " <command>", continuousDetail)
-  def FlushFileTreeRepository: String = "flushFileTreeRepository"
-  def FlushDetailed: String =
-    "Resets the global file repository in the event that the repository has become inconsistent " +
-      "with the file system."
+  def ClearCaches: String = "clearCaches"
+  def ClearCachesDetailed: String = "Clears all of sbt's internal caches."
 }

main-command/src/main/scala/sbt/BasicKeys.scala

@@ -8,6 +8,7 @@
 package sbt
 
 import java.io.File
+
 import sbt.internal.util.AttributeKey
 import sbt.internal.inc.classpath.ClassLoaderCache
 import sbt.internal.server.ServerHandler

main/src/main/scala/sbt/Defaults.scala

@@ -142,7 +142,8 @@ object Defaults extends BuildCommon {
       excludeFilter :== HiddenFileFilter,
       classLoaderCache := ClassLoaderCache(4),
       layeringStrategy := LayeringStrategy.Default
-    ) ++ globalIvyCore ++ globalJvmCore
+    ) ++ TaskRepository
+      .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore
   ) ++ globalSbtCore
 
   private[sbt] lazy val globalJvmCore: Seq[Setting[_]] =
@@ -1770,16 +1771,18 @@ object Defaults extends BuildCommon {
       Classpaths.addUnmanagedLibrary
 
   // We need a cache of size two for the test dependency layers (regular and snapshot).
-  lazy val testSettings
-    : Seq[Setting[_]] = configSettings ++ testTasks :+ (classLoaderCache := ClassLoaderCache(2))
+  lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks ++ TaskRepository.proxy(
+    classLoaderCache,
+    ClassLoaderCache(2)
+  )
 
   lazy val itSettings: Seq[Setting[_]] = inConfig(IntegrationTest)(testSettings)
   lazy val defaultConfigs: Seq[Setting[_]] = inConfig(Compile)(compileSettings) ++
     inConfig(Test)(testSettings) ++ inConfig(Runtime)(
     // We need a cache of size four so that the subset of the runtime dependencies that are used
-    // by the test task layers may be cached without evicting the runtime classloader layres. The
+    // by the test task layers may be cached without evicting the runtime classloader layers. The
     // cache size should be a multiple of two to support snapshot layers.
-    Classpaths.configSettings :+ (classLoaderCache := ClassLoaderCache(4))
+    Classpaths.configSettings ++ TaskRepository.proxy(classLoaderCache, ClassLoaderCache(4)),
   )
 
   // These are project level settings that MUST be on every project.

main/src/main/scala/sbt/Keys.scala

@@ -461,7 +461,8 @@ object Keys {
   val resolvedScoped = Def.resolvedScoped
   val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask)
   val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask)
-  val classLoaderCache = settingKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask)
+  val classLoaderCache = taskKey[internal.ClassLoaderCache]("The cache of ClassLoaders to be used for layering in tasks that invoke other java code").withRank(DTask)
+  private[sbt] val taskRepository = AttributeKey[TaskRepository.Repr]("task-repository", "A repository that can be used to cache arbitrary values for a given task key that can be read or filled during task evaluation.", 10000)
 
   // wrapper to work around SI-2915
   private[sbt] final class TaskProgress(val progress: ExecuteProgress[Task])

main/src/main/scala/sbt/Main.scala

@@ -214,7 +214,7 @@ object BuiltinCommands {
       BasicCommands.multi,
       act,
       continuous,
-      flushFileTreeRepository
+      clearCaches
     ) ++ allBasicCommands
 
   def DefaultBootCommands: Seq[String] =
@@ -830,7 +830,7 @@ object BuiltinCommands {
 
     val session = Load.initialSession(structure, eval, s0)
     SessionSettings.checkSession(session, s)
-    registerGlobalFileRepository(Project.setProject(session, structure, s))
+    registerGlobalCaches(Project.setProject(session, structure, s))
   }
 
   def registerCompilerCache(s: State): State = {
@@ -848,26 +848,31 @@ object BuiltinCommands {
       }
     s.put(Keys.stateCompilerCache, cache)
   }
-  def registerGlobalFileRepository(s: State): State = {
+  private[sbt] def registerGlobalCaches(s: State): State = {
     val extracted = Project.extract(s)
     try {
-      val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s)
-      val view: FileTreeDataView[StampedFile] = config.newDataView()
-      val newState = s.addExitHook {
+      def cleanup(): Unit = {
         s.get(BasicKeys.globalFileTreeView).foreach(_.close())
         s.attributes.remove(BasicKeys.globalFileTreeView)
+        s.get(Keys.taskRepository).foreach(_.close())
+        s.attributes.remove(Keys.taskRepository)
         ()
       }
-      newState.get(BasicKeys.globalFileTreeView).foreach(_.close())
-      newState.put(BasicKeys.globalFileTreeView, view)
+      val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s)
+      val view: FileTreeDataView[StampedFile] = config.newDataView()
+      val newState = s.addExitHook(cleanup())
+      cleanup()
+      newState
+        .put(BasicKeys.globalFileTreeView, view)
+        .put(Keys.taskRepository, new TaskRepository.Repr)
     } catch {
       case NonFatal(_) => s
     }
   }
 
-  def flushFileTreeRepository: Command = {
-    val help = Help.more(FlushFileTreeRepository, FlushDetailed)
-    Command.command(FlushFileTreeRepository, help)(registerGlobalFileRepository)
+  def clearCaches: Command = {
+    val help = Help.more(ClearCaches, ClearCachesDetailed)
+    Command.command(ClearCaches, help)(registerGlobalCaches)
   }
 
   def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 =>

main/src/main/scala/sbt/internal/Repository.scala

@@ -7,6 +7,10 @@
 
 package sbt.internal
 
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.collection.JavaConverters._
+
 /**
  * Represents an abstract cache of values, accessible by a key. The interface is deliberately
  * minimal to give maximum flexibility to the implementation classes. For example, one can construct
@@ -32,3 +36,23 @@ package sbt.internal
 trait Repository[M[_], K, V] extends AutoCloseable {
   def get(key: K): M[V]
 }
+
+private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] {
+  private[this] val map = new ConcurrentHashMap[K, V].asScala
+  override def get(key: K): Option[V] = map.get(key)
+  def put(key: K, value: V): Unit = this.synchronized {
+    map.put(key, value)
+    ()
+  }
+  def remove(key: K): Unit = this.synchronized {
+    map.remove(key)
+    ()
+  }
+  override def close(): Unit = this.synchronized {
+    map.foreach {
+      case (_, v: AutoCloseable) => v.close()
+      case _                     =>
+    }
+    map.clear()
+  }
+}

main/src/main/scala/sbt/internal/TaskRepository.scala

@@ -0,0 +1,38 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.internal
+
+import sbt.Keys.state
+import sbt._
+
+private[sbt] object TaskRepository {
+  private[sbt] type Repr = MutableRepository[TaskKey[_], Any]
+  private[sbt] def proxy[T: Manifest](taskKey: TaskKey[T], task: => T): Def.Setting[Task[T]] =
+    proxy(taskKey, Def.task(task))
+  private[sbt] def proxy[T: Manifest](
+      taskKey: TaskKey[T],
+      task: Def.Initialize[Task[T]]
+  ): Def.Setting[Task[T]] =
+    taskKey := Def.taskDyn {
+      val taskRepository = state.value
+        .get(Keys.taskRepository)
+        .getOrElse {
+          val msg = "TaskRepository.proxy called before state was initialized"
+          throw new IllegalStateException(msg)
+        }
+      taskRepository.get(taskKey) match {
+        case Some(value: T) => Def.task(value)
+        case _ =>
+          Def.task {
+            val value = task.value
+            taskRepository.put(taskKey, value)
+            value
+          }
+      }
+    }.value
+}