Cache compiled map during build load

The continuous command recompiles the setting graph into a CompiledMap
data structure so that it can determine which files it needs to
transitively monitor during watch. Generating the CompiledMap can be
very slow for large projects (5 seconds or so on my computer in the sbt
project) and this startup cost is paid every time the user enters a
watch with `~`. To avoid this, we can cache the compile map that is
generated during the initial settings evaluation.

The only real drawback I can see is that the compiled map is guaranteed
to remain in memory so long as the BuildStructure instance that holds it
is alive. Given the performance benefit, this seems like a worthwhile
tradeoff.

internal/util-collection/src/main/scala/sbt/internal/util/Settings.scala

@@ -198,17 +198,24 @@ trait Init[ScopeType] {
     compile(dMap)
   }
 
+  @deprecated("Use makeWithCompiledMap", "1.4.0")
   def make(init: Seq[Setting[_]])(
       implicit delegates: ScopeType => Seq[ScopeType],
       scopeLocal: ScopeLocal,
       display: Show[ScopedKey[_]]
-  ): Settings[ScopeType] = {
+  ): Settings[ScopeType] = makeWithCompiledMap(init)._2
+
+  def makeWithCompiledMap(init: Seq[Setting[_]])(
+      implicit delegates: ScopeType => Seq[ScopeType],
+      scopeLocal: ScopeLocal,
+      display: Show[ScopedKey[_]]
+  ): (CompiledMap, Settings[ScopeType]) = {
     val cMap = compiled(init)(delegates, scopeLocal, display)
     // order the initializations.  cyclic references are detected here.
     val ordered: Seq[Compiled[_]] = sort(cMap)
     // evaluation: apply the initializations.
     try {
-      applyInits(ordered)
+      (cMap, applyInits(ordered))
     } catch {
       case rru: RuntimeUndefined =>
         throw Uninitialized(cMap.keys.toSeq, delegates, rru.undefined, true)

internal/util-collection/src/test/scala/SettingsExample.scala

@@ -64,7 +64,8 @@ case class SettingsUsage(val settingsExample: SettingsExample) {
   // "compiles" and applies the settings.
   //  This can be split into multiple steps to access intermediate results if desired.
   //  The 'inspect' command operates on the output of 'compile', for example.
-  val applied: Settings[Scope] = make(mySettings)(delegates, scopeLocal, showFullKey)
+  val applied: Settings[Scope] =
+    makeWithCompiledMap(mySettings)(delegates, scopeLocal, showFullKey)._2
 
   // Show results.
   /*	for(i <- 0 to 5; k <- Seq(a, b)) {

internal/util-collection/src/test/scala/SettingsTest.scala

@@ -200,7 +200,7 @@ object SettingsTest extends Properties("settings") {
 
   def evaluate(settings: Seq[Setting[_]]): Settings[Scope] =
     try {
-      make(settings)(delegates, scopeLocal, showFullKey)
+      makeWithCompiledMap(settings)(delegates, scopeLocal, showFullKey)._2
     } catch {
       case e: Throwable => e.printStackTrace(); throw e
     }

main/src/main/scala/sbt/internal/BuildStructure.scala

@@ -32,7 +32,19 @@ final class BuildStructure(
     val streams: State => Streams,
     val delegates: Scope => Seq[Scope],
     val scopeLocal: ScopeLocal,
+    private[sbt] val compiledMap: Map[ScopedKey[_], Def.Compiled[_]],
 ) {
+  @deprecated("Used the variant that takes a compiledMap", "1.4.0")
+  def this(
+      units: Map[URI, LoadedBuildUnit],
+      root: URI,
+      settings: Seq[Setting[_]],
+      data: Settings[Scope],
+      index: StructureIndex,
+      streams: State => Streams,
+      delegates: Scope => Seq[Scope],
+      scopeLocal: ScopeLocal,
+  ) = this(units, root, settings, data, index, streams, delegates, scopeLocal, Map.empty)
 
   val extra: BuildUtil[ResolvedProject] = BuildUtil(root, units, index.keyIndex, data)
 

main/src/main/scala/sbt/internal/Load.scala

@@ -251,12 +251,16 @@ private[sbt] object Load {
     val delegates = timed("Load.apply: config.delegates", log) {
       config.delegates(loaded)
     }
-    val data = timed("Load.apply: Def.make(settings)...", log) {
+    val (cMap, data) = timed("Load.apply: Def.make(settings)...", log) {
       // When settings.size is 100000, Def.make takes around 10s.
       if (settings.size > 10000) {
         log.info(s"resolving key references (${settings.size} settings) ...")
       }
-      Def.make(settings)(delegates, config.scopeLocal, Project.showLoadingKey(loaded))
+      Def.makeWithCompiledMap(settings)(
+        delegates,
+        config.scopeLocal,
+        Project.showLoadingKey(loaded)
+      )
     }
     Project.checkTargets(data) foreach sys.error
     val index = timed("Load.apply: structureIndex", log) {
@@ -271,7 +275,8 @@ private[sbt] object Load {
       index,
       streams,
       delegates,
-      config.scopeLocal
+      config.scopeLocal,
+      cMap
     )
     (rootEval, bs)
   }
@@ -338,7 +343,8 @@ private[sbt] object Load {
       implicit display: Show[ScopedKey[_]]
   ): BuildStructure = {
     val transformed = finalTransforms(newSettings)
-    val newData = Def.make(transformed)(structure.delegates, structure.scopeLocal, display)
+    val (cMap, newData) =
+      Def.makeWithCompiledMap(transformed)(structure.delegates, structure.scopeLocal, display)
     def extra(index: KeyIndex) = BuildUtil(structure.root, structure.units, index, newData)
     val newIndex = structureIndex(newData, transformed, extra, structure.units)
     val newStreams = mkStreams(structure.units, structure.root, newData)
@@ -350,7 +356,8 @@ private[sbt] object Load {
       index = newIndex,
       streams = newStreams,
       delegates = structure.delegates,
-      scopeLocal = structure.scopeLocal
+      scopeLocal = structure.scopeLocal,
+      compiledMap = cMap,
     )
   }
 

main/src/main/scala/sbt/internal/SettingsGraph.scala

@@ -41,8 +41,7 @@ private[sbt] object SettingsGraph {
     f(extracted, compile(extracted.structure))
   }
 
-  private[sbt] def compile(structure: BuildStructure): CompiledMap =
-    compiled(structure.settings)(structure.delegates, structure.scopeLocal, (_: ScopedKey[_]) => "")
+  private[sbt] def compile(structure: BuildStructure): CompiledMap = structure.compiledMap
   private[sbt] final class Arguments(
       val scopedKey: ScopedKey[_],
       val extracted: Extracted,

main/src/test/scala/PluginCommandTest.scala

@@ -102,7 +102,8 @@ object FakeState {
     val delegates: (Scope) => Seq[Scope] = _ => Nil
     val scopeLocal: Def.ScopeLocal = _ => Nil
 
-    val data: Settings[Scope] = Def.make(settings)(delegates, scopeLocal, Def.showFullKey)
+    val (cMap, data: Settings[Scope]) =
+      Def.makeWithCompiledMap(settings)(delegates, scopeLocal, Def.showFullKey)
     val extra: KeyIndex => BuildUtil[_] = (keyIndex) =>
       BuildUtil(base.toURI, Map.empty, keyIndex, data)
     val structureIndex: StructureIndex =
@@ -140,7 +141,8 @@ object FakeState {
       structureIndex,
       streams,
       delegates,
-      scopeLocal
+      scopeLocal,
+      cMap,
     )
 
     val attributes = AttributeMap.empty ++ AttributeMap(

main/src/test/scala/sbt/internal/TestBuild.scala

@@ -241,7 +241,7 @@ abstract class TestBuild {
           throw e
       }
     }
-    val data = Def.make(settings)(env.delegates, const(Nil), display)
+    val data = Def.makeWithCompiledMap(settings)(env.delegates, const(Nil), display)._2
     val keys = data.allKeys((s, key) => ScopedKey(s, key))
     val keyMap = keys.map(k => (k.key.label, k.key)).toMap[String, AttributeKey[_]]
     val projectsMap = env.builds.map(b => (b.uri, b.projects.map(_.id).toSet)).toMap

main/src/test/scala/sbt/internal/server/SettingQueryTest.scala

@@ -18,7 +18,6 @@ import scala.collection.mutable
 
 import xsbti.{ Logger => _, _ }
 import sbt.io.IO
-import sbt.internal.util._
 import sbt.internal.BuildStreams.{ Streams => _, _ }
 import sbt.internal.Load._
 import sbt.util._
@@ -171,14 +170,24 @@ object SettingQueryTest extends org.specs2.mutable.Specification {
     val scopeLocal: ScopeLocal = EvaluateTask.injectStreams
     val display: Show[ScopedKey[_]] = Project showLoadingKey loadedBuild
 
-    val data: Settings[Scope] = Def.make(settings)(delegates, scopeLocal, display)
+    val (cMap, data) = Def.makeWithCompiledMap(settings)(delegates, scopeLocal, display)
     val extra: KeyIndex => BuildUtil[_] = index => BuildUtil(baseUri, units, index, data)
 
     val index: StructureIndex = structureIndex(data, settings, extra, units)
     val streams: State => Streams = mkStreams(units, baseUri, data)
 
     val structure: BuildStructure =
-      new BuildStructure(units, baseUri, settings, data, index, streams, delegates, scopeLocal)
+      new BuildStructure(
+        units,
+        baseUri,
+        settings,
+        data,
+        index,
+        streams,
+        delegates,
+        scopeLocal,
+        cMap
+      )
 
     structure
   }