From 748bf1207f6ef2b9b8e5c952f605eb4c3d1d3910 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 20 Sep 2025 17:44:17 -0400 Subject: [PATCH] Auto aggregate **Problem** For sbt 1.x, the user is forced to pick between having a stable ID for the root project, or having the automatic aggregation of all subprojects. The problem becomes more pronounced for large build that frequent add/remove subprojects. **Solution** This implements `.autoAggregate` method on `Project`, which is implemented as `this.aggregate(LocalAggregate)`. At the loading time, we can automatically expand `LocalAggregate` to a list of subproject references, after we discover all subprojects. The `autoAggregate` will use the base directory of the subproject to pick the parent-child relationship. For example, a root project would aggregate all subprojects, but `bar` might aggregate only `bar/bar1` and `bar/bar2`. --- main-settings/src/main/scala/sbt/Def.scala | 1 + .../src/main/scala/sbt/Project.scala | 17 ++-- .../src/main/scala/sbt/Reference.scala | 4 + main-settings/src/main/scala/sbt/Scope.scala | 2 + .../scala/sbt/internal/BuildStructure.scala | 4 +- main/src/main/scala/sbt/internal/Load.scala | 79 ++++++++++++------- .../project/auto-aggregate/bar/bar1/B1.scala | 1 + .../sbt-test/project/auto-aggregate/build.sbt | 13 +++ .../project/auto-aggregate/foo/A.scala | 1 + .../src/sbt-test/project/auto-aggregate/test | 5 ++ 10 files changed, 93 insertions(+), 34 deletions(-) create mode 100644 sbt-app/src/sbt-test/project/auto-aggregate/bar/bar1/B1.scala create mode 100644 sbt-app/src/sbt-test/project/auto-aggregate/build.sbt create mode 100644 sbt-app/src/sbt-test/project/auto-aggregate/foo/A.scala create mode 100644 sbt-app/src/sbt-test/project/auto-aggregate/test diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala index 682f1daef..e77923a9c 100644 --- a/main-settings/src/main/scala/sbt/Def.scala +++ b/main-settings/src/main/scala/sbt/Def.scala @@ -173,6 +173,7 @@ object Def extends BuildSyntax with Init with InitializeImplicits: case LocalProject(p) => if (p == current.project) "" else p case ThisBuild => "ThisBuild" case LocalRootProject => "" + case LocalAggregate => "" case ThisProject => "" } val str = loop(project) diff --git a/main-settings/src/main/scala/sbt/Project.scala b/main-settings/src/main/scala/sbt/Project.scala index abd3b2b11..b74c057d9 100644 --- a/main-settings/src/main/scala/sbt/Project.scala +++ b/main-settings/src/main/scala/sbt/Project.scala @@ -143,6 +143,12 @@ sealed trait Project extends ProjectDefinition[ProjectReference] with CompositeP def aggregate(refs: ProjectReference*): Project = copy(aggregate = (aggregate: Seq[ProjectReference]) ++ refs) + /** + * Automatically aggregate local subprojects. + */ + def autoAggregate: Project = + this.aggregate(LocalAggregate) + /** Appends settings to the current settings sequence for this project. */ def settings(ss: Def.SettingsDefinition*): Project = copy(settings = (settings: Seq[Def.Setting[?]]) ++ Def.settings(ss*)) @@ -217,11 +223,12 @@ sealed trait Project extends ProjectDefinition[ProjectReference] with CompositeP dependencies = resolveDeps(dependencies), ) - private[sbt] def resolve(resolveRef: ProjectReference => ProjectRef): ResolvedProject = - def resolveRefs(prs: Seq[ProjectReference]) = prs.map(resolveRef) - def resolveDeps(ds: Seq[ClasspathDep[ProjectReference]]) = ds.map(resolveDep) - def resolveDep(d: ClasspathDep[ProjectReference]) = - ClasspathDep.ResolvedClasspathDependency(resolveRef(d.project), d.configuration) + private[sbt] def resolve(resolveRef: ProjectReference => Seq[ProjectRef]): ResolvedProject = + def resolveRefs(prs: Seq[ProjectReference]) = prs.flatMap(resolveRef) + def resolveDeps(ds: Seq[ClasspathDep[ProjectReference]]) = ds.flatMap(resolveDep) + def resolveDep(d: ClasspathDep[ProjectReference]): Seq[ClasspathDep[ProjectRef]] = + resolveRef(d.project).map: ref => + ClasspathDep.ResolvedClasspathDependency(ref, d.configuration) Project.resolved( id, base, diff --git a/main-settings/src/main/scala/sbt/Reference.scala b/main-settings/src/main/scala/sbt/Reference.scala index fb271916e..eb5682e43 100644 --- a/main-settings/src/main/scala/sbt/Reference.scala +++ b/main-settings/src/main/scala/sbt/Reference.scala @@ -71,6 +71,9 @@ case object LocalRootProject extends ProjectReference /** Identifies the project for the current context. */ case object ThisProject extends ProjectReference +/** A placeholder for auto aggregation. */ +case object LocalAggregate extends ProjectReference + object ProjectRef { def apply(base: File, id: String): ProjectRef = ProjectRef(IO toURI base, id) } @@ -108,6 +111,7 @@ object Reference { ref match { case ThisProject => "{}" case LocalRootProject => "{}" + case LocalAggregate => "{}" case LocalProject(id) => "{}" + id case RootProject(uri) => "{" + uri + " }" case ProjectRef(uri, id) => s"""ProjectRef(uri("$uri"), "$id")""" diff --git a/main-settings/src/main/scala/sbt/Scope.scala b/main-settings/src/main/scala/sbt/Scope.scala index fa8b5edd9..2dd328bec 100644 --- a/main-settings/src/main/scala/sbt/Scope.scala +++ b/main-settings/src/main/scala/sbt/Scope.scala @@ -130,6 +130,7 @@ object Scope: case RootProject(uri) => RootProject(resolveBuild(current, uri)) case ProjectRef(uri, id) => ProjectRef(resolveBuild(current, uri), id) case ThisProject => ThisProject // haven't exactly "resolved" anything.. + case LocalAggregate => LocalAggregate } def resolveBuild(current: URI, uri: URI): URI = if (!uri.isAbsolute && current.isOpaque && uri.getSchemeSpecificPart == ".") @@ -158,6 +159,7 @@ object Scope: case RootProject(uri) => val u = resolveBuild(current, uri); ProjectRef(u, rootProject(u)) case ProjectRef(uri, id) => ProjectRef(resolveBuild(current, uri), id) case ThisProject => sys.error("Cannot resolve ThisProject w/o the current project") + case LocalAggregate => sys.error("Cannot resolve LocalAggregate") } def resolveBuildRef(current: URI, ref: BuildReference): BuildRef = ref match { diff --git a/main/src/main/scala/sbt/internal/BuildStructure.scala b/main/src/main/scala/sbt/internal/BuildStructure.scala index 87f0ee407..28e4ba8fb 100644 --- a/main/src/main/scala/sbt/internal/BuildStructure.scala +++ b/main/src/main/scala/sbt/internal/BuildStructure.scala @@ -294,7 +294,8 @@ final class PartBuildUnit( def resolve(f: Project => ResolvedProject): LoadedBuildUnit = new LoadedBuildUnit(unit, defined.view.mapValues(f).toMap, rootProjects, buildSettings) - def resolveRefs(f: ProjectReference => ProjectRef): LoadedBuildUnit = resolve(_.resolve(f)) + def resolveRefs(f: ProjectReference => Seq[ProjectRef]): LoadedBuildUnit = + resolve(_.resolve(f)) } object BuildStreams { @@ -360,6 +361,7 @@ object BuildStreams { case Select(LocalProject(id)) => id case Select(RootProject(_)) => RootPath case Select(LocalRootProject) => LocalRootProject.toString + case Select(LocalAggregate) => LocalAggregate.toString case Select(ThisBuild) | Select(ThisProject) | This => // Don't want to crash if somehow an unresolved key makes it in here. This.toString diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 4b69973ab..a5e77f546 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -657,28 +657,25 @@ private[sbt] object Load { IO createDirectory base } - def resolveAll(builds: Map[URI, PartBuildUnit]): Map[URI, LoadedBuildUnit] = { - val rootProject = getRootProject(builds) - builds map { (uri, unit) => - (uri, unit.resolveRefs(ref => Scope.resolveProjectRef(uri, rootProject, ref))) - } - } - def checkAll( referenced: Map[URI, List[ProjectReference]], builds: Map[URI, PartBuildUnit] - ): Unit = { + ): Unit = val rootProject = getRootProject(builds) - for ((uri, refs) <- referenced; ref <- refs) { - val ProjectRef(refURI, refID) = Scope.resolveProjectRef(uri, rootProject, ref) - val loadedUnit = builds(refURI) - if (!(loadedUnit.defined contains refID)) { - val projectIDs = loadedUnit.defined.keys.toSeq.sorted - sys.error(s"""No project '$refID' in '$refURI'. - |Valid project IDs: ${projectIDs.mkString(", ")}""".stripMargin) - } - } - } + for + (uri, refs) <- referenced + ref <- refs + do + ref match + case LocalAggregate => () + case _ => + val ProjectRef(refURI, refID) = Scope.resolveProjectRef(uri, rootProject, ref) + val loadedUnit = builds(refURI) + if (!loadedUnit.defined.contains(refID)) { + val projectIDs = loadedUnit.defined.keys.toSeq.sorted + sys.error(s"""No project '$refID' in '$refURI'. + |Valid project IDs: ${projectIDs.mkString(", ")}""".stripMargin) + } /** * Returns true when value is the subproject base for root project. @@ -709,16 +706,37 @@ private[sbt] object Load { uri: URI, unit: PartBuildUnit, rootProject: URI => String - ): LoadedBuildUnit = { + ): LoadedBuildUnit = IO.assertAbsolute(uri) - val resolve = (_: Project).resolve(ref => Scope.resolveProjectRef(uri, rootProject, ref)) - new LoadedBuildUnit( + val ps = unit.defined.values.toVector + .map(p => (p, IO.toURI(p.base).toString())) + .sortBy(_._2) + val resolve: Project => ResolvedProject = (p: Project) => + p.resolve: + case LocalAggregate => resolveAutoAggregate(uri, p, ps) + case ref => Vector(Scope.resolveProjectRef(uri, rootProject, ref)) + LoadedBuildUnit( unit.unit, unit.defined.view.mapValues(resolve).toMap, unit.rootProjects, unit.buildSettings ) - } + + /** + * This expands LocalAggregate reference object to all subprojects within the same + * build URI under the current subproject's base directory. + * This should return all subprojects for root. + */ + private def resolveAutoAggregate( + uri: URI, + current: Project, + ps: Vector[(Project, String)] + ): Seq[ProjectRef] = + val base = IO.toURI(current.base).toString() + ps.flatMap: (p, projBase) => + if projBase == base then Nil + else if projBase.startsWith(base) then Vector(ProjectRef(uri, p.id)) + else Nil def projects(unit: BuildUnit): Seq[Project] = { // we don't have the complete build graph loaded, so we don't have the rootProject function yet. @@ -826,7 +844,7 @@ private[sbt] object Load { loadedProjectsRaw.projects.exists(p => isRootPath(p.base, normBase)) || defsScala.exists( _.rootProject.isDefined ) - val (loadedProjects, defaultBuildIfNone, keepClassFiles) = + val (loadedProjects0, defaultBuildIfNone, keepClassFiles) = if (hasRoot) ( loadedProjectsRaw.projects, @@ -847,6 +865,7 @@ private[sbt] object Load { defaultProjects.generatedConfigClassFiles ++ loadedProjectsRaw.generatedConfigClassFiles ) } + val loadedProjects = processAutoAggregate(loadedProjects0, uri) // TODO: Uncomment when we fixed https://github.com/sbt/sbt/issues/7424 // likely keepClassFiles isn't covering enough. // timed("Load.loadUnit: cleanEvalClasses", log) { @@ -870,6 +889,10 @@ private[sbt] object Load { new BuildUnit(uri, normBase, loadedDefs, plugs, converter) } + private def processAutoAggregate(inProjects: Seq[Project], uri: URI): Seq[Project] = + inProjects.map: proj => + proj + private def autoID( localBase: File, context: PluginManagement.Context, @@ -1005,7 +1028,7 @@ private[sbt] object Load { // a. Apply all the project manipulations from .sbt files in order // b. Deduce the auto plugins for the project // c. Finalize a project with all its settings/configuration. - def finalizeProject( + def processProject( p: Project, files: Seq[VirtualFile], extraFiles: Seq[VirtualFile], @@ -1048,14 +1071,14 @@ private[sbt] object Load { // phony. However, we may want to 'merge' the two, or only do this if the original was a // default generated project. val root = rootOpt.getOrElse(p) - val (finalRoot, projectLevelExtra) = finalizeProject(root, files, extraFiles, true) + val (finalRoot, projectLevelExtra) = processProject(root, files, extraFiles, true) val newProjects = rest ++ discovered ++ projectLevelExtra val newAcc = acc :+ finalRoot val newGenerated = generated ++ generatedConfigClassFiles loadTransitive1(newProjects, newAcc, newGenerated, finalRoot.commonSettings) } - // Load all config files AND finalize the project at the root directory, if it exists. + // Load all config files AND process the project at the root directory, if it exists. // Continue loading if we find any more. newProjects match case Seq(next, rest*) => @@ -1093,8 +1116,8 @@ private[sbt] object Load { val refs = existingIds.map(id => ProjectRef(buildUri, id)) (root.aggregate(refs*), false, Nil, otherProjects) val (finalRoot, projectLevelExtra) = - timed(s"Load.loadTransitive: finalizeProject($root)", log) { - finalizeProject(root, files, extraFiles, expand) + timed(s"Load.loadTransitive: processProject($root)", log) { + processProject(root, files, extraFiles, expand) } val newProjects = moreProjects ++ projectLevelExtra val newAcc = finalRoot +: (acc ++ otherProjects.projects) diff --git a/sbt-app/src/sbt-test/project/auto-aggregate/bar/bar1/B1.scala b/sbt-app/src/sbt-test/project/auto-aggregate/bar/bar1/B1.scala new file mode 100644 index 000000000..e246898c3 --- /dev/null +++ b/sbt-app/src/sbt-test/project/auto-aggregate/bar/bar1/B1.scala @@ -0,0 +1 @@ +class B1 diff --git a/sbt-app/src/sbt-test/project/auto-aggregate/build.sbt b/sbt-app/src/sbt-test/project/auto-aggregate/build.sbt new file mode 100644 index 000000000..07cd7a1f0 --- /dev/null +++ b/sbt-app/src/sbt-test/project/auto-aggregate/build.sbt @@ -0,0 +1,13 @@ +lazy val root = (project in file(".")) + .autoAggregate + .settings( + name := "foo-root", + publish / skip := true, + ) + +lazy val foo = project + +lazy val bar = project + .autoAggregate + +lazy val bar1 = (project in file("bar/bar1")) diff --git a/sbt-app/src/sbt-test/project/auto-aggregate/foo/A.scala b/sbt-app/src/sbt-test/project/auto-aggregate/foo/A.scala new file mode 100644 index 000000000..83d15dc73 --- /dev/null +++ b/sbt-app/src/sbt-test/project/auto-aggregate/foo/A.scala @@ -0,0 +1 @@ +class A diff --git a/sbt-app/src/sbt-test/project/auto-aggregate/test b/sbt-app/src/sbt-test/project/auto-aggregate/test new file mode 100644 index 000000000..382e6cd57 --- /dev/null +++ b/sbt-app/src/sbt-test/project/auto-aggregate/test @@ -0,0 +1,5 @@ +> bar/compile +$ exists target/**/bar1/backend/B1.class + +> compile +$ exists target/**/foo/backend/A.class