luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Auto aggregate 

**Problem**
For sbt 1.x, the user is forced to pick between having a stable ID for the root project,
or having the automatic aggregation of all subprojects.
The problem becomes more pronounced for large build that frequent add/remove subprojects.

**Solution**
This implements `.autoAggregate` method on `Project`, which is implemented as
`this.aggregate(LocalAggregate)`.
At the loading time, we can automatically expand `LocalAggregate` to a list of subproject references,
after we discover all subprojects.
The `autoAggregate` will use the base directory of the subproject to pick the parent-child
relationship. For example, a root project would aggregate all subprojects,
but `bar` might aggregate only `bar/bar1` and `bar/bar2`.
This commit is contained in:
Eugene Yokota 2025-09-20 17:44:17 -04:00
parent 84fcb1a3a6
commit 748bf1207f
10 changed files with 93 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
main-settings/src/main/scala/sbt/Def.scala
View File

@ -173,6 +173,7 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
case LocalProject(p) => if (p == current.project) "" else p
case ThisBuild => "ThisBuild"
case LocalRootProject => "<root>"
case LocalAggregate => "<aggregate>"
case ThisProject => "<this>"
}
val str = loop(project)

17
main-settings/src/main/scala/sbt/Project.scala
View File

@ -143,6 +143,12 @@ sealed trait Project extends ProjectDefinition[ProjectReference] with CompositeP
def aggregate(refs: ProjectReference*): Project =
copy(aggregate = (aggregate: Seq[ProjectReference]) ++ refs)
/**
* Automatically aggregate local subprojects.
*/
def autoAggregate: Project =
this.aggregate(LocalAggregate)
/** Appends settings to the current settings sequence for this project. */
def settings(ss: Def.SettingsDefinition*): Project =
copy(settings = (settings: Seq[Def.Setting[?]]) ++ Def.settings(ss*))
@ -217,11 +223,12 @@ sealed trait Project extends ProjectDefinition[ProjectReference] with CompositeP
dependencies = resolveDeps(dependencies),
)
private[sbt] def resolve(resolveRef: ProjectReference => ProjectRef): ResolvedProject =
def resolveRefs(prs: Seq[ProjectReference]) = prs.map(resolveRef)
def resolveDeps(ds: Seq[ClasspathDep[ProjectReference]]) = ds.map(resolveDep)
def resolveDep(d: ClasspathDep[ProjectReference]) =
ClasspathDep.ResolvedClasspathDependency(resolveRef(d.project), d.configuration)
private[sbt] def resolve(resolveRef: ProjectReference => Seq[ProjectRef]): ResolvedProject =
def resolveRefs(prs: Seq[ProjectReference]) = prs.flatMap(resolveRef)
def resolveDeps(ds: Seq[ClasspathDep[ProjectReference]]) = ds.flatMap(resolveDep)
def resolveDep(d: ClasspathDep[ProjectReference]): Seq[ClasspathDep[ProjectRef]] =
resolveRef(d.project).map: ref =>
ClasspathDep.ResolvedClasspathDependency(ref, d.configuration)
Project.resolved(
id,
base,

4
main-settings/src/main/scala/sbt/Reference.scala
View File

@ -71,6 +71,9 @@ case object LocalRootProject extends ProjectReference
/** Identifies the project for the current context. */
case object ThisProject extends ProjectReference
/** A placeholder for auto aggregation. */
case object LocalAggregate extends ProjectReference
object ProjectRef {
def apply(base: File, id: String): ProjectRef = ProjectRef(IO toURI base, id)
}
@ -108,6 +111,7 @@ object Reference {
ref match {
case ThisProject => "{<this>}<this>"
case LocalRootProject => "{<this>}<root>"
case LocalAggregate => "{<this>}<aggregate>"
case LocalProject(id) => "{<this>}" + id
case RootProject(uri) => "{" + uri + " }<root>"
case ProjectRef(uri, id) => s"""ProjectRef(uri("$uri"), "$id")"""

2
main-settings/src/main/scala/sbt/Scope.scala
View File

@ -130,6 +130,7 @@ object Scope:
case RootProject(uri) => RootProject(resolveBuild(current, uri))
case ProjectRef(uri, id) => ProjectRef(resolveBuild(current, uri), id)
case ThisProject => ThisProject // haven't exactly "resolved" anything..
case LocalAggregate => LocalAggregate
}
def resolveBuild(current: URI, uri: URI): URI =
if (!uri.isAbsolute && current.isOpaque && uri.getSchemeSpecificPart == ".")
@ -158,6 +159,7 @@ object Scope:
case RootProject(uri) => val u = resolveBuild(current, uri); ProjectRef(u, rootProject(u))
case ProjectRef(uri, id) => ProjectRef(resolveBuild(current, uri), id)
case ThisProject => sys.error("Cannot resolve ThisProject w/o the current project")
case LocalAggregate => sys.error("Cannot resolve LocalAggregate")
}
def resolveBuildRef(current: URI, ref: BuildReference): BuildRef =
ref match {

4
main/src/main/scala/sbt/internal/BuildStructure.scala
View File

@ -294,7 +294,8 @@ final class PartBuildUnit(
def resolve(f: Project => ResolvedProject): LoadedBuildUnit =
new LoadedBuildUnit(unit, defined.view.mapValues(f).toMap, rootProjects, buildSettings)
def resolveRefs(f: ProjectReference => ProjectRef): LoadedBuildUnit = resolve(_.resolve(f))
def resolveRefs(f: ProjectReference => Seq[ProjectRef]): LoadedBuildUnit =
resolve(_.resolve(f))
}
object BuildStreams {
@ -360,6 +361,7 @@ object BuildStreams {
case Select(LocalProject(id)) => id
case Select(RootProject(_)) => RootPath
case Select(LocalRootProject) => LocalRootProject.toString
case Select(LocalAggregate) => LocalAggregate.toString
case Select(ThisBuild) | Select(ThisProject) | This =>
// Don't want to crash if somehow an unresolved key makes it in here.
This.toString

79
main/src/main/scala/sbt/internal/Load.scala
View File

@ -657,28 +657,25 @@ private[sbt] object Load {
IO createDirectory base
}
def resolveAll(builds: Map[URI, PartBuildUnit]): Map[URI, LoadedBuildUnit] = {
val rootProject = getRootProject(builds)
builds map { (uri, unit) =>
(uri, unit.resolveRefs(ref => Scope.resolveProjectRef(uri, rootProject, ref)))
}
}
def checkAll(
referenced: Map[URI, List[ProjectReference]],
builds: Map[URI, PartBuildUnit]
): Unit = {
): Unit =
val rootProject = getRootProject(builds)
for ((uri, refs) <- referenced; ref <- refs) {
val ProjectRef(refURI, refID) = Scope.resolveProjectRef(uri, rootProject, ref)
val loadedUnit = builds(refURI)
if (!(loadedUnit.defined contains refID)) {
val projectIDs = loadedUnit.defined.keys.toSeq.sorted
sys.error(s"""No project '$refID' in '$refURI'.
|Valid project IDs: ${projectIDs.mkString(", ")}""".stripMargin)
}
}
}
for
(uri, refs) <- referenced
ref <- refs
do
ref match
case LocalAggregate => ()
case _ =>
val ProjectRef(refURI, refID) = Scope.resolveProjectRef(uri, rootProject, ref)
val loadedUnit = builds(refURI)
if (!loadedUnit.defined.contains(refID)) {
val projectIDs = loadedUnit.defined.keys.toSeq.sorted
sys.error(s"""No project '$refID' in '$refURI'.
|Valid project IDs: ${projectIDs.mkString(", ")}""".stripMargin)
}
/**
* Returns true when value is the subproject base for root project.
@ -709,16 +706,37 @@ private[sbt] object Load {
uri: URI,
unit: PartBuildUnit,
rootProject: URI => String
): LoadedBuildUnit = {
): LoadedBuildUnit =
IO.assertAbsolute(uri)
val resolve = (_: Project).resolve(ref => Scope.resolveProjectRef(uri, rootProject, ref))
new LoadedBuildUnit(
val ps = unit.defined.values.toVector
.map(p => (p, IO.toURI(p.base).toString()))
.sortBy(_._2)
val resolve: Project => ResolvedProject = (p: Project) =>
p.resolve:
case LocalAggregate => resolveAutoAggregate(uri, p, ps)
case ref => Vector(Scope.resolveProjectRef(uri, rootProject, ref))
LoadedBuildUnit(
unit.unit,
unit.defined.view.mapValues(resolve).toMap,
unit.rootProjects,
unit.buildSettings
)
}
/**
* This expands LocalAggregate reference object to all subprojects within the same
* build URI under the current subproject's base directory.
* This should return all subprojects for root.
*/
private def resolveAutoAggregate(
uri: URI,
current: Project,
ps: Vector[(Project, String)]
): Seq[ProjectRef] =
val base = IO.toURI(current.base).toString()
ps.flatMap: (p, projBase) =>
if projBase == base then Nil
else if projBase.startsWith(base) then Vector(ProjectRef(uri, p.id))
else Nil
def projects(unit: BuildUnit): Seq[Project] = {
// we don't have the complete build graph loaded, so we don't have the rootProject function yet.
@ -826,7 +844,7 @@ private[sbt] object Load {
loadedProjectsRaw.projects.exists(p => isRootPath(p.base, normBase)) || defsScala.exists(
_.rootProject.isDefined
)
val (loadedProjects, defaultBuildIfNone, keepClassFiles) =
val (loadedProjects0, defaultBuildIfNone, keepClassFiles) =
if (hasRoot)
(
loadedProjectsRaw.projects,
@ -847,6 +865,7 @@ private[sbt] object Load {
defaultProjects.generatedConfigClassFiles ++ loadedProjectsRaw.generatedConfigClassFiles
)
}
val loadedProjects = processAutoAggregate(loadedProjects0, uri)
// TODO: Uncomment when we fixed https://github.com/sbt/sbt/issues/7424
// likely keepClassFiles isn't covering enough.
// timed("Load.loadUnit: cleanEvalClasses", log) {
@ -870,6 +889,10 @@ private[sbt] object Load {
new BuildUnit(uri, normBase, loadedDefs, plugs, converter)
}
private def processAutoAggregate(inProjects: Seq[Project], uri: URI): Seq[Project] =
inProjects.map: proj =>
proj
private def autoID(
localBase: File,
context: PluginManagement.Context,
@ -1005,7 +1028,7 @@ private[sbt] object Load {
// a. Apply all the project manipulations from .sbt files in order
// b. Deduce the auto plugins for the project
// c. Finalize a project with all its settings/configuration.
def finalizeProject(
def processProject(
p: Project,
files: Seq[VirtualFile],
extraFiles: Seq[VirtualFile],
@ -1048,14 +1071,14 @@ private[sbt] object Load {
// phony. However, we may want to 'merge' the two, or only do this if the original was a
// default generated project.
val root = rootOpt.getOrElse(p)
val (finalRoot, projectLevelExtra) = finalizeProject(root, files, extraFiles, true)
val (finalRoot, projectLevelExtra) = processProject(root, files, extraFiles, true)
val newProjects = rest ++ discovered ++ projectLevelExtra
val newAcc = acc :+ finalRoot
val newGenerated = generated ++ generatedConfigClassFiles
loadTransitive1(newProjects, newAcc, newGenerated, finalRoot.commonSettings)
}
// Load all config files AND finalize the project at the root directory, if it exists.
// Load all config files AND process the project at the root directory, if it exists.
// Continue loading if we find any more.
newProjects match
case Seq(next, rest*) =>
@ -1093,8 +1116,8 @@ private[sbt] object Load {
val refs = existingIds.map(id => ProjectRef(buildUri, id))
(root.aggregate(refs*), false, Nil, otherProjects)
val (finalRoot, projectLevelExtra) =
timed(s"Load.loadTransitive: finalizeProject($root)", log) {
finalizeProject(root, files, extraFiles, expand)
timed(s"Load.loadTransitive: processProject($root)", log) {
processProject(root, files, extraFiles, expand)
}
val newProjects = moreProjects ++ projectLevelExtra
val newAcc = finalRoot +: (acc ++ otherProjects.projects)

1
sbt-app/src/sbt-test/project/auto-aggregate/bar/bar1/B1.scala Normal file
View File

@ -0,0 +1 @@
class B1

13
sbt-app/src/sbt-test/project/auto-aggregate/build.sbt Normal file
View File

@ -0,0 +1,13 @@
lazy val root = (project in file("."))
.autoAggregate
.settings(
name := "foo-root",
publish / skip := true,
)
lazy val foo = project
lazy val bar = project
.autoAggregate
lazy val bar1 = (project in file("bar/bar1"))

1
sbt-app/src/sbt-test/project/auto-aggregate/foo/A.scala Normal file
View File

@ -0,0 +1 @@
class A

5
sbt-app/src/sbt-test/project/auto-aggregate/test Normal file
View File

@ -0,0 +1,5 @@
> bar/compile
$ exists target/**/bar1/backend/B1.class
> compile
$ exists target/**/foo/backend/A.class