Merge pull request #7866 from adpi2/2.x-reduce-heap

[2.x] Reduce number of long-living instances to speed up loading

internal/util-core/src/main/scala/sbt/internal/util/Util.scala

@@ -10,6 +10,8 @@ package sbt.internal.util
 
 import java.util.Locale
 
+import scala.collection.concurrent.TrieMap
+
 object Util {
   def makeList[T](size: Int, value: T): List[T] = List.fill(size)(value)
 
@@ -71,4 +73,9 @@ object Util {
   extension [A](value: A) {
     def some: Option[A] = (Some(value): Option[A])
   }
+
+  private[sbt] def withCaching[A1, A2](f: A1 => A2): A1 => A2 = {
+    val cache = TrieMap.empty[A1, A2]
+    x => cache.getOrElseUpdate(x, f(x))
+  }
 }

main-settings/src/main/scala/sbt/Project.scala

@@ -15,6 +15,8 @@ import sbt.internal.util.Dag
 import sbt.internal.util.complete.Parser
 import sbt.internal.util.complete.DefaultParsers
 import Scope.ThisScope
+import sbt.Scope.ThisBuildScope
+import sbt.internal.util.Util
 
 sealed trait ProjectDefinition[PR <: ProjectReference] {
 
@@ -335,20 +337,16 @@ object Project:
     [a] => (k: ScopedKey[a]) => ScopedKey(f(k.scope), k.key)
 
   def transform(g: Scope => Scope, ss: Seq[Def.Setting[?]]): Seq[Def.Setting[?]] =
-    val f = mapScope(g)
+    // We use caching to avoid creating new Scope instances too many times
-    ss.map { setting =>
+    // Creating a new Scope is CPU expensive because of the uniqueness cache
-      setting.mapKey(f).mapReferenced(f)
+    val f = mapScope(Util.withCaching(g))
-    }
+    ss.map(_.mapKey(f).mapReferenced(f))
-
-  def transformRef(g: Scope => Scope, ss: Seq[Def.Setting[?]]): Seq[Def.Setting[?]] =
-    val f = mapScope(g)
-    ss.map(_.mapReferenced(f))
 
   def inThisBuild(ss: Seq[Setting[?]]): Seq[Setting[?]] =
-    inScope(ThisScope.copy(project = Select(ThisBuild)))(ss)
+    inScope(ThisBuildScope)(ss)
 
   private[sbt] def inThisBuild[T](i: Initialize[T]): Initialize[T] =
-    inScope(ThisScope.copy(project = Select(ThisBuild)), i)
+    inScope(ThisBuildScope, i)
 
   private[sbt] def inConfig[T](conf: Configuration, i: Initialize[T]): Initialize[T] =
     inScope(ThisScope.copy(config = Select(conf)), i)

main-settings/src/main/scala/sbt/Scope.scala

@@ -14,17 +14,30 @@ import sbt.internal.util.{ AttributeKey, AttributeMap, Dag }
 import sbt.internal.util.Util._
 
 import sbt.io.IO
+import scala.collection.concurrent.TrieMap
+import scala.runtime.ScalaRunTime
 
-final case class Scope(
+final case class Scope private (
     project: ScopeAxis[Reference],
     config: ScopeAxis[ConfigKey],
     task: ScopeAxis[AttributeKey[?]],
     extra: ScopeAxis[AttributeMap]
 ):
+  // Since we use a uniqueness cache we can pre-compute the hashCode for free
+  // It is always going to be used at least once
+  override val hashCode = ScalaRunTime._hashCode(this)
+
   def rescope(project: Reference): Scope = copy(project = Select(project))
   def rescope(config: ConfigKey): Scope = copy(config = Select(config))
   def rescope(task: AttributeKey[?]): Scope = copy(task = Select(task))
 
+  def copy(
+      project: ScopeAxis[Reference] = this.project,
+      config: ScopeAxis[ConfigKey] = this.config,
+      task: ScopeAxis[AttributeKey[?]] = this.task,
+      extra: ScopeAxis[AttributeMap] = this.extra
+  ): Scope = Scope(project, config, task, extra)
+
   override def toString: String = this match
     case Scope(Zero, Zero, Zero, Zero) => "Global"
     case Scope(_, _, _, This)          => s"$project / $config / $task"
@@ -32,8 +45,23 @@ final case class Scope(
 end Scope
 
 object Scope:
-  val ThisScope: Scope = Scope(This, This, This, This)
+  // We use a global uniqueness cache to avoid duplicating Scope.
-  val Global: Scope = Scope(Zero, Zero, Zero, Zero)
+  // At the time of writing, it divides the number of long-living instances by 15
+  // reducing the pressure on the heap, and speed up the loading.
+  private val uniquenessCache = TrieMap.empty[Scope, Scope]
+
+  def apply(
+      project: ScopeAxis[Reference],
+      config: ScopeAxis[ConfigKey],
+      task: ScopeAxis[AttributeKey[?]],
+      extra: ScopeAxis[AttributeMap]
+  ): Scope =
+    val scope = new Scope(project, config, task, extra)
+    uniquenessCache.getOrElseUpdate(scope, scope)
+
+  val ThisScope: Scope = new Scope(This, This, This, This)
+  val Global: Scope = new Scope(Zero, Zero, Zero, Zero)
+  val ThisBuildScope: Scope = Scope(Select(ThisBuild), This, This, This)
   val GlobalScope: Scope = Global
 
   private[sbt] final val inIsDeprecated =
@@ -250,33 +278,6 @@ object Scope:
 
   def showProject012Style = (ref: Reference) => Reference.display(ref) + "/"
 
-  @deprecated("No longer used", "1.1.3")
-  def transformTaskName(s: String) = {
-    val parts = s.split("-+")
-    (parts.take(1) ++ parts.drop(1).map(_.capitalize)).mkString
-  }
-
-  @deprecated("Use variant without extraInherit", "1.1.1")
-  def delegates[Proj](
-      refs: Seq[(ProjectRef, Proj)],
-      configurations: Proj => Seq[ConfigKey],
-      resolve: Reference => ResolvedReference,
-      rootProject: URI => String,
-      projectInherit: ProjectRef => Seq[ProjectRef],
-      configInherit: (ResolvedReference, ConfigKey) => Seq[ConfigKey],
-      taskInherit: AttributeKey[?] => Seq[AttributeKey[?]],
-      extraInherit: (ResolvedReference, AttributeMap) => Seq[AttributeMap]
-  ): Scope => Seq[Scope] =
-    delegates(
-      refs,
-      configurations,
-      resolve,
-      rootProject,
-      projectInherit,
-      configInherit,
-      taskInherit,
-    )
-
   // *Inherit functions should be immediate delegates and not include argument itself.  Transitivity will be provided by this method
   def delegates[Proj](
       refs: Seq[(ProjectRef, Proj)],
@@ -291,17 +292,7 @@ object Scope:
     scope => indexedDelegates(resolve, index, rootProject, taskInherit)(scope)
   }
 
-  @deprecated("Use variant without extraInherit", "1.1.1")
+  private def indexedDelegates(
-  def indexedDelegates(
-      resolve: Reference => ResolvedReference,
-      index: DelegateIndex,
-      rootProject: URI => String,
-      taskInherit: AttributeKey[?] => Seq[AttributeKey[?]],
-      extraInherit: (ResolvedReference, AttributeMap) => Seq[AttributeMap]
-  )(rawScope: Scope): Seq[Scope] =
-    indexedDelegates(resolve, index, rootProject, taskInherit)(rawScope)
-
-  def indexedDelegates(
       resolve: Reference => ResolvedReference,
       index: DelegateIndex,
       rootProject: URI => String,
@@ -367,19 +358,20 @@ object Scope:
   }
 
   private val zeroL = List(Zero)
+  private val globalL = List(GlobalScope)
+
   def withZeroAxis[T](base: ScopeAxis[T]): Seq[ScopeAxis[T]] =
-    if (base.isSelect) List(base, Zero: ScopeAxis[T])
+    if (base.isSelect) base :: zeroL else zeroL
-    else zeroL
 
   def withGlobalScope(base: Scope): Seq[Scope] =
-    if (base == GlobalScope) GlobalScope :: Nil else base :: GlobalScope :: Nil
+    if (base == GlobalScope) globalL else base :: globalL
+
   def withRawBuilds(ps: Seq[ScopeAxis[ProjectRef]]): Seq[ScopeAxis[ResolvedReference]] =
-    (ps: Seq[ScopeAxis[ResolvedReference]]) ++
+    ps ++ ps.flatMap(rawBuild).distinct :+ Zero
-      ((ps flatMap rawBuild).distinct: Seq[ScopeAxis[ResolvedReference]]) :+
-      (Zero: ScopeAxis[ResolvedReference])
 
   def rawBuild(ps: ScopeAxis[ProjectRef]): Seq[ScopeAxis[BuildRef]] = ps match {
-    case Select(ref) => Select(BuildRef(ref.build)) :: Nil; case _ => Nil
+    case Select(ref) => Select(BuildRef(ref.build)) :: Nil
+    case _           => Nil
   }
 
   def delegates[Proj](
@@ -417,19 +409,17 @@ object Scope:
   ): Seq[ScopeAxis[T]] =
     axis match {
       case Select(x)   => topologicalSort[T](x, appendZero)(inherit)
-      case Zero | This => if (appendZero) Zero :: Nil else Nil
+      case Zero | This => if (appendZero) zeroL else Nil
     }
 
   def topologicalSort[T](node: T, appendZero: Boolean)(
       dependencies: T => Seq[T]
   ): Seq[ScopeAxis[T]] = {
     val o = Dag.topologicalSortUnchecked(node)(dependencies).map(x => Select(x): ScopeAxis[T])
-    if (appendZero) o ::: (Zero: ScopeAxis[T]) :: Nil
+    if (appendZero) o ::: zeroL else o
-    else o
   }
   def globalProjectDelegates(scope: Scope): Seq[Scope] =
-    if (scope == GlobalScope)
+    if (scope == GlobalScope) globalL
-      GlobalScope :: Nil
     else
       for {
         c <- withZeroAxis(scope.config)

main/src/main/scala/sbt/internal/Load.scala

@@ -35,6 +35,7 @@ import java.net.URI
 import java.nio.file.{ Path, Paths }
 import scala.annotation.tailrec
 import scala.collection.mutable
+import sbt.internal.util.Util
 
 private[sbt] object Load {
   // note that there is State passed in but not pulled out
@@ -266,11 +267,16 @@ private[sbt] object Load {
     }
     val projects = loaded.units
     lazy val rootEval = lazyEval(loaded.units(loaded.root).unit)
+    val settings0 = timed("Load.apply: buildConfigurations", log) {
+      buildConfigurations(loaded, getRootProject(projects), config.injectSettings)
+    }
     val settings = timed("Load.apply: finalTransforms", log) {
-      finalTransforms(buildConfigurations(loaded, getRootProject(projects), config.injectSettings))
+      finalTransforms(settings0)
     }
     val delegates = timed("Load.apply: config.delegates", log) {
-      config.delegates(loaded)
+      // We use caching to avoid creating new Scope instances too many times
+      // Creating a new Scope is CPU expensive because of the uniqueness cache
+      Util.withCaching(config.delegates(loaded))
     }
     val (cMap, data) = timed("Load.apply: Def.make(settings)...", log) {
       // When settings.size is 100000, Def.make takes around 10s.

sbt-app/src/main/scala/package.scala

@@ -33,7 +33,6 @@ package object sbt
     fillTaskAxis,
     mapScope,
     transform,
-    transformRef,
     inThisBuild,
     inScope,
     normalizeModuleID

util-collection/src/main/scala/sbt/internal/util/INode.scala

@@ -40,7 +40,10 @@ abstract class EvaluateSettings[ScopeType]:
   private val transform: [A] => Initialize[A] => INode[A] = [A] =>
     (fa: Initialize[A]) =>
       fa match
-        case k: Keyed[s, A]   => single(getStatic(k.scopedKey), k.transform)
+        case g: GetValue[s, A]     => single(getStatic(g.scopedKey), g.transform)
+        case k: KeyedInitialize[A] =>
+          // TODO create a Single node with no transform?
+          single(getStatic(k.scopedKey), identity)
         case u: Uniform[s, A] => UniformNode(u.inputs.map(transform[s]), u.f)
         case a: Apply[k, A] =>
           MixedNode[k, A](TupleMapExtension.transform(a.inputs)(transform), a.f)

util-collection/src/main/scala/sbt/internal/util/Settings.scala

@@ -811,9 +811,13 @@ trait Init[ScopeType]:
    */
   sealed trait Keyed[S, A1] extends Initialize[A1]:
     def scopedKey: ScopedKey[S]
-    def transform: S => A1
-
     override final def dependencies = scopedKey :: Nil
+    private[sbt] override def processAttributes[A2](init: A2)(f: (A2, AttributeMap) => A2): A2 =
+      init
+  end Keyed
+
+  final class GetValue[S, A1](val scopedKey: ScopedKey[S], val transform: S => A1)
+      extends Keyed[S, A1]:
     override final def apply[A2](g: A1 => A2): Initialize[A2] =
       GetValue(scopedKey, g compose transform)
     override final def evaluate(ss: Settings[ScopeType]): A1 = transform(getValue(ss, scopedKey))
@@ -829,20 +833,27 @@ trait Init[ScopeType]:
       g(scopedKey) match
         case None        => this
         case Some(const) => Value(() => transform(const))
-
+  end GetValue
-    private[sbt] override def processAttributes[A2](init: A2)(f: (A2, AttributeMap) => A2): A2 =
-      init
-  end Keyed
-
-  private final class GetValue[S, A1](val scopedKey: ScopedKey[S], val transform: S => A1)
-      extends Keyed[S, A1]
 
   /**
    * A `Keyed` where the type of the value and the associated `ScopedKey` are the same.
    * @tparam A1 the type of both the value this `Initialize` defines and the type of the associated `ScopedKey`.
    */
   trait KeyedInitialize[A1] extends Keyed[A1, A1]:
-    final val transform = identity[A1]
+    override final def apply[A2](g: A1 => A2): Initialize[A2] =
+      GetValue(scopedKey, g)
+    override final def evaluate(ss: Settings[ScopeType]): A1 = getValue(ss, scopedKey)
+    override final def mapReferenced(g: MapScoped): Initialize[A1] = g(scopedKey)
+
+    private[sbt] override final def validateKeyReferenced(g: ValidateKeyRef): ValidatedInit[A1] =
+      g(scopedKey, false) match
+        case Left(un)  => Left(un :: Nil)
+        case Right(nk) => Right(nk)
+
+    override final def mapConstant(g: MapConstant): Initialize[A1] =
+      g(scopedKey) match
+        case None        => this
+        case Some(const) => Value(() => const)
   end KeyedInitialize
 
   private[sbt] final class TransformCapture(val f: [x] => Initialize[x] => Initialize[x])