diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e70192e33..9ae99e494 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,10 +56,10 @@ jobs:
java: 17
distribution: zulu
jobtype: 12
- # - os: ubuntu-latest
- # java: 17
- # distribution: temurin
- # jobtype: 13
+ - os: ubuntu-latest
+ java: 17
+ distribution: temurin
+ jobtype: 13
runs-on: ${{ matrix.os }}
timeout-minutes: 25
env:
@@ -196,8 +196,8 @@ jobs:
shell: bash
run: |
./sbt -v --server "scripted cache/*"
- # - name: Hash Benchmark
- # if: ${{ matrix.jobtype == 13 }}
- # shell: bash
- # run: |
- # ./sbt -v "hashBenchmark/Jmh/run -i 5 -wi 3 -f1 -t1"
+ - name: Community build
+ if: ${{ matrix.jobtype == 13 }}
+ shell: bash
+ run: |
+ ./sbt -v --server "community-build/test"
diff --git a/.gitmodules b/.gitmodules
index 5738b50ae..3e491e33f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,12 @@
[submodule "lm-coursier/metadata"]
path = lm-coursier/metadata
url = https://github.com/coursier/handmade-metadata.git
+[submodule "community-build/community-projects/parboiled2"]
+ path = community-build/community-projects/parboiled2
+ url = https://github.com/sbt-2-builds/parboiled2.git
+[submodule "community-build/community-projects/scalaz"]
+ path = community-build/community-projects/scalaz
+ url = https://github.com/sbt-2-builds/scalaz.git
+[submodule "community-build/community-projects/sbt-compile-benchmark"]
+ path = community-build/community-projects/sbt-compile-benchmark
+ url = https://github.com/sbt-2-builds/sbt-compile-benchmark.git
diff --git a/build.sbt b/build.sbt
index a688fc3a2..b9da33813 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1448,3 +1448,24 @@ lazy val launcherPackageIntegrationTest =
},
Test / parallelExecution := false
)
+
+val prepareCommunityBuild = taskKey[Unit]("Publish local etc")
+lazy val `community-build` = (project in file("community-build"))
+ .settings(
+ scalaVersion := scala3,
+ libraryDependencies ++= Seq(junit % Test, junitInterface % Test),
+ prepareCommunityBuild := {
+ val _ = (sbtRoot / publishLocalBinAll).value
+ IO.write(baseDirectory.value / "target" / "sbt.version", version.value)
+ },
+ (Test / testOptions) += Tests.Argument(
+ TestFrameworks.JUnit,
+ "--include-categories=sbt.internal.communitybuild.TestCategory",
+ "--run-listener=sbt.internal.communitybuild.FailureSummarizer",
+ ),
+ Compile / run := (Compile / run).dependsOn(prepareCommunityBuild).evaluated,
+ Test / testOnly := (Test / testOnly).dependsOn(prepareCommunityBuild).evaluated,
+ Test / testQuick := (Test / testQuick).dependsOn(prepareCommunityBuild).evaluated,
+ publish / skip := true,
+ publishLocalBin / skip := true,
+ )
diff --git a/community-build/community-projects/parboiled2 b/community-build/community-projects/parboiled2
new file mode 160000
index 000000000..6941dbab3
--- /dev/null
+++ b/community-build/community-projects/parboiled2
@@ -0,0 +1 @@
+Subproject commit 6941dbab32f394a12c8633c57a7bc1e0456bed73
diff --git a/community-build/community-projects/sbt-compile-benchmark b/community-build/community-projects/sbt-compile-benchmark
new file mode 160000
index 000000000..e8874c791
--- /dev/null
+++ b/community-build/community-projects/sbt-compile-benchmark
@@ -0,0 +1 @@
+Subproject commit e8874c79126882e1ead501740b2be2f43271a4c6
diff --git a/community-build/community-projects/scalaz b/community-build/community-projects/scalaz
new file mode 160000
index 000000000..39500a87c
--- /dev/null
+++ b/community-build/community-projects/scalaz
@@ -0,0 +1 @@
+Subproject commit 39500a87ca70aa4df5ed5408fa1580e52c510c85
diff --git a/community-build/src/main/scala/sbt/internal/communitybuild/CommunityBuildRunner.scala b/community-build/src/main/scala/sbt/internal/communitybuild/CommunityBuildRunner.scala
new file mode 100644
index 000000000..c493858b5
--- /dev/null
+++ b/community-build/src/main/scala/sbt/internal/communitybuild/CommunityBuildRunner.scala
@@ -0,0 +1,87 @@
+package sbt
+package internal
+package communitybuild
+
+import java.nio.file.*
+
+object CommunityBuildRunner:
+
+ /** Depending on the mode of operation, either
+ * runs the test or updates the project. Updating
+ * means that all the dependencies are fetched but
+ * minimal other extra other work is done. Updating
+ * is necessary since we run tests each time on a fresh
+ * Docker container. We run the update on Docker container
+ * creation time to create the cache of the dependencies
+ * and avoid network overhead.
+ */
+ extension (self: CommunityProject)
+ def run()(using suite: CommunityBuildRunner): Unit =
+ suite.runProject(self)
+ end extension
+
+trait CommunityBuildRunner:
+
+ /** fails the current operation, can be specialised in a concrete Runner
+ * - overridden in `CommunityBuildTest`
+ */
+ def failWith(msg: String): Nothing = throw IllegalStateException(msg)
+
+ /** Build the given project with the published local compiler and sbt plugin.
+ *
+ * This test reads the compiler version from community-build/dotty-bootstrapped.version
+ * and expects community-build/sbt-injected-plugins to set any necessary plugins.
+ *
+ * @param project The project name, should be a git submodule in community-build/
+ * @param command The binary file of the program used to test the project – usually
+ * a build tool like SBT or Mill
+ * @param arguments Arguments to pass to the testing program
+ */
+ def runProject(projectDef: CommunityProject): Unit =
+ val project = projectDef.project
+ val command = projectDef.binaryName
+ val arguments = projectDef.buildCommands
+
+ @annotation.tailrec
+ def execTimes(task: () => Int, timesToRerun: Int): Boolean =
+ val exitCode = task()
+ if exitCode == 0
+ then true
+ else if timesToRerun == 0
+ then false
+ else
+ log(s"Rerunning tests in $project because of a previous run failure.")
+ execTimes(task, timesToRerun - 1)
+
+ log(s"Building $project ...")
+
+ val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
+
+ if !Files.exists(projectDir.resolve(".git")) then
+ failWith(s"""
+ |
+ |Missing $project submodule at $projectDir. You can initialize this module using
+ |
+ | git submodule update --init community-build/community-projects/$project
+ |
+ |""".stripMargin)
+
+ val testsCompletedSuccessfully = execTimes(projectDef.build, 3)
+
+ if !testsCompletedSuccessfully then
+ failWith(s"""
+ |
+ |$command exited with an error code. To reproduce without JUnit, use:
+ |
+ | sbt community-build/prepareCommunityBuild
+ | cd community-build/community-projects/$project
+ | $command ${arguments.init.mkString(" ")} "${arguments.last}"
+ |
+ |For a faster feedback loop on sbt projects, one can try to extract a direct call to dotc
+ |using the sbt export command. For instance, for scalacheck, use
+ | sbt export jvm/Test/compileIncremental
+ |
+ |""".stripMargin)
+ end runProject
+
+end CommunityBuildRunner
diff --git a/community-build/src/main/scala/sbt/internal/communitybuild/Main.scala b/community-build/src/main/scala/sbt/internal/communitybuild/Main.scala
new file mode 100644
index 000000000..de64c1ec7
--- /dev/null
+++ b/community-build/src/main/scala/sbt/internal/communitybuild/Main.scala
@@ -0,0 +1,120 @@
+package sbt
+package internal
+package communitybuild
+
+import java.nio.file.Paths
+import java.nio.file.Path
+import java.nio.file.Files
+import scala.sys.process._
+
+import CommunityBuildRunner.run
+
+object Main:
+
+ private def generateDocs(project: CommunityProject): Seq[Path] =
+ val name = project.project
+ try
+ project.doc()
+ val pathsOut = s"find community-projects/$name/ -name 'scaladoc.version'".!!
+ pathsOut.linesIterator.map(Paths.get(_).getParent).toList
+ catch
+ case e: Exception =>
+ e.printStackTrace()
+ Nil
+
+ def withProjects[T](names: Seq[String], opName: String)(op: CommunityProject => T): Seq[T] =
+ val missing = names.filterNot(projectMap.contains)
+ if missing.nonEmpty then
+ val allNames = allProjects.map(_.project).mkString(", ")
+ println(s"Missing projects: ${missing.mkString(", ")}. All projects: $allNames")
+ sys.exit(1)
+
+ val (failed, completed) = names.flatMap(projectMap.apply).partitionMap( o =>
+ try
+ Right(op(o))
+ catch case e: Throwable =>
+ e.printStackTrace()
+ Left(o)
+ )
+
+ if failed.nonEmpty then
+ println(s"$opName failed for ${failed.mkString(", ")}")
+ sys.exit(1)
+
+ completed
+
+ /** Allows running various commands on community build projects. */
+ def main(args: Array[String]): Unit =
+ args.toList match
+ case "publish" :: names if names.nonEmpty =>
+ withProjects(names, "Publishing")(_.publish())
+
+ case "build" :: names if names.nonEmpty =>
+ withProjects(names, "Build")(_.build())
+
+ case "doc" :: "all" :: destStr :: Nil =>
+ val dest = Paths.get(destStr)
+ Seq("rm", "-rf", destStr).!
+ Files.createDirectory(dest)
+ val (toRun, ignored) =
+ allProjects.partition(_.docCommand != null)
+
+ val paths = toRun.map { project =>
+ val name = project.project
+ val projectDest = dest.resolve(name)
+ val projectRoot = Paths.get(s"community-projects/$name")
+ println(s"generating docs for $name into $projectDest")
+ val generatedDocs = generateDocs(project)
+ if !Files.exists(projectDest) && generatedDocs.nonEmpty then
+ Files.createDirectory(projectDest)
+
+ val docsFiles = generatedDocs.map { docsPath =>
+ val destFileName =
+ docsPath.subpath(2, docsPath.getNameCount).toString.replace('/', '_')
+
+ Seq("cp", "-r", docsPath.toString, projectDest.resolve(destFileName).toString).!
+ destFileName
+ }
+ name -> docsFiles
+ }
+
+ val (failed, withDocs) = paths.partition{ case (_, paths) => paths.isEmpty }
+
+ val indexFile = withDocs.map { case (name, paths) =>
+ paths.map(p => s"""$p\n""")
+ .mkString(s"
$name
","\n", "\n")
+ }.mkString("\n", "\n", "\n")
+
+ Files.write(dest.resolve("index.html"), indexFile.getBytes)
+
+ if ignored.nonEmpty then
+ println(s"Ignored project without doc command: ${ignored.map(_.project)}")
+
+ if failed.nonEmpty then
+ println(s"Documentation not found for ${failed.map(_._1).mkString(", ")}")
+ sys.exit(1)
+
+ case "doc" :: names if names.nonEmpty =>
+ val failed = withProjects(names, "Documenting"){ p =>
+ val docsRoots = generateDocs(p)
+ println(docsRoots)
+ if docsRoots.nonEmpty then println(s"Docs for $p generated in $docsRoots")
+ if docsRoots.isEmpty then Some(p.project) else None
+ }.flatten
+
+ if failed.nonEmpty then
+ println(s"Documentation not found for ${failed.mkString(", ")}")
+ sys.exit(1)
+
+ case "run" :: names if names.nonEmpty =>
+ given CommunityBuildRunner()
+ withProjects(names, "Running")(_.run())
+
+ case args =>
+ println("USAGE: ")
+ println("COMMAND is one of: publish, build, doc, doc all, run")
+ println("Available projects are:")
+ allProjects.foreach { k =>
+ println(s"\t${k.project}")
+ }
+ sys.exit(1)
diff --git a/community-build/src/main/scala/sbt/internal/communitybuild/projects.scala b/community-build/src/main/scala/sbt/internal/communitybuild/projects.scala
new file mode 100644
index 000000000..e630b1944
--- /dev/null
+++ b/community-build/src/main/scala/sbt/internal/communitybuild/projects.scala
@@ -0,0 +1,169 @@
+package sbt
+package internal
+package communitybuild
+
+import java.nio.file._
+import java.io.File
+import java.nio.charset.StandardCharsets.UTF_8
+
+lazy val communitybuildDir: Path =
+ Paths.get(sys.props("user.dir")).resolve("community-build")
+
+lazy val sbtVersion: String =
+ val file = communitybuildDir.resolve("target").resolve("sbt.version")
+ new String(Files.readAllBytes(file), UTF_8)
+
+lazy val bootDir: Path =
+ val dir = communitybuildDir.resolve("target").resolve("boot")
+ Files.createDirectories(dir)
+ dir
+
+lazy val sbtPluginFilePath: String =
+ // Workaround for https://github.com/sbt/sbt/issues/4395
+ new File(sys.props("user.home") + "/config/sbt/2/plugins").mkdirs()
+ communitybuildDir.resolve("sbt-injected-plugins").toAbsolutePath().toString()
+
+def log(msg: String) = println(Console.GREEN + msg + Console.RESET)
+
+/** Executes shell command, returns false in case of error. */
+def exec(projectDir: Path, binary: String, arguments: Seq[String], environment: Map[String, String]): Int =
+ import scala.jdk.CollectionConverters._
+ val command = binary +: arguments
+ log(command.mkString(" "))
+ val builder = new ProcessBuilder(command*).directory(projectDir.toFile).inheritIO()
+ builder.environment.putAll(environment.asJava)
+ val process = builder.start()
+ val exitCode = process.waitFor()
+ exitCode
+
+
+sealed trait CommunityProject:
+ def project: String
+ def testCommand: String
+ def testCompileCommand: String
+ def publishCommand: String
+ def docCommand: String
+ def binaryName: String
+ def runCommandsArgs: List[String] = Nil
+ def environment: Map[String, String] = Map.empty
+
+ final val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
+
+ /** Publish this project to the local Maven repository */
+ final def publish(): Unit =
+ log(s"Publishing $project")
+ if publishCommand eq null then
+ throw RuntimeException(s"Publish command is not specified for $project. Project details:\n$this")
+ val exitCode = exec(projectDir, binaryName, (runCommandsArgs :+ publishCommand), environment)
+ if exitCode != 0 then
+ throw RuntimeException(s"Publish command exited with code $exitCode for project $project. Project details:\n$this")
+
+ final def doc(): Unit =
+ log(s"Documenting $project")
+ if docCommand eq null then
+ throw RuntimeException(s"Doc command is not specified for $project. Project details:\n$this")
+ val exitCode = exec(projectDir, binaryName, (runCommandsArgs :+ docCommand), environment)
+ if exitCode != 0 then
+ throw RuntimeException(s"Doc command exited with code $exitCode for project $project. Project details:\n$this")
+
+ final def build(): Int = exec(projectDir, binaryName, buildCommands, environment)
+
+ final def buildCommands = runCommandsArgs :+ testCompileCommand
+
+end CommunityProject
+
+val sbt1Version = "1.12.1"
+val sbt2Version = "2.0.3"
+
+final case class SbtCommunityProject(
+ project: String,
+ testCmd: String = "test",
+ testCompileCmd: String = "Test/compile",
+ extraSbtArgs: List[String] = Nil,
+ publishCmd: String = "publishLocal",
+ docCmd: String = "doc",
+ scalacOptions: List[String] = SbtCommunityProject.scalacOptions,
+ override val environment: Map[String, String] = Map.empty,
+ ) extends CommunityProject:
+ override val binaryName: String = "sbt"
+
+ private def scalacOptionsString: String =
+ scalacOptions.map("\"" + _ + "\"").mkString("List(", ",", ")")
+
+ private val baseCommand =
+ "set Global/logLevel := Level.Error; "
+ ++ (if scalacOptions.isEmpty then "" else s"""set Global/scalacOptions ++= $scalacOptionsString;""")
+
+ override val testCommand =
+ """set Global/testOptions += Tests.Argument(TestFramework("munit.Framework"), "+l"); """
+ ++ s"$baseCommand$testCmd"
+
+ override val testCompileCommand =
+ s"$baseCommand$testCompileCmd"
+
+ override val publishCommand =
+ if publishCmd eq null then null else s"$baseCommand$publishCmd"
+
+ override val docCommand =
+ if docCmd eq null then null else
+ val cmd = if docCmd.startsWith(";") then docCmd else s";$docCmd"
+ s"$baseCommand set every useScaladoc := true; set every doc/logLevel := Level.Warn $cmd "
+
+ override val runCommandsArgs: List[String] =
+ // Run the sbt command with the compiler version and sbt plugin set in the build
+ val sbtProps = Option(System.getProperty("sbt.ivy.home")) match
+ case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome")
+ case _ => Nil
+ extraSbtArgs ++ sbtProps ++ List(
+ s"-Dsbt.version=$sbtVersion",
+ s"-Dsbt.boot=$bootDir",
+ "-Dsbt.supershell=false",
+ )
+
+object SbtCommunityProject:
+ def scalacOptions = Nil
+
+object projects:
+
+ private def forceDoc(projects: String*) =
+ projects.map(project =>
+ s""";set $project/Compile/doc/sources ++= ($project/Compile/doc/dotty.tools.sbtplugin.DottyPlugin.autoImport.tastyFiles).value ;$project/doc"""
+ ).mkString(" ")
+
+ private def removeRelease8(projects: String*): String =
+ projects.map(project =>
+ s"""set $project/Compile/scalacOptions := ($project/Compile/scalacOptions).value.filterNot(opt => opt == "-release" || opt == "-java-output-version" || opt == "8")"""
+ ).mkString("; ")
+
+ private def aggregateDoc(in: String)(projects: String*) =
+ val tastyFiles =
+ (in +: projects).map(p => s"($p/Compile/doc/dotty.tools.sbtplugin.DottyPlugin.autoImport.tastyFiles).value").mkString(" ++ ")
+ s""";set $in/Compile/doc/sources ++= file("a.scala") +: ($tastyFiles) ;$in/doc"""
+
+ lazy val `sbt-compile-benchmark` = SbtCommunityProject(
+ project = "sbt-compile-benchmark",
+ )
+
+ lazy val scalaz = SbtCommunityProject(
+ project = "scalaz",
+ testCmd = "rootJVM/test",
+ testCompileCmd = "rootJVM/Test/compile",
+ docCmd = forceDoc("effectJVM"),
+ )
+
+ lazy val parboiled2 = SbtCommunityProject(
+ project = "parboiled2",
+ testCmd = "parboiledCoreJVM3/testFull; parboiledJVM3/testFull",
+ testCompileCmd = "parboiledCoreJVM3/Test/compile; parboiledJVM3/Test/compile",
+ publishCmd = "publishLocal",
+ scalacOptions = SbtCommunityProject.scalacOptions.filter(_ != "-Xcheck-macros"),
+ )
+
+end projects
+
+def allProjects = List(
+ projects.parboiled2,
+ projects.scalaz,
+)
+
+lazy val projectMap = allProjects.groupBy(_.project)
diff --git a/community-build/src/test/java/sbt/internal/communitybuild/FailureSummarizer.java b/community-build/src/test/java/sbt/internal/communitybuild/FailureSummarizer.java
new file mode 100644
index 000000000..b60b81617
--- /dev/null
+++ b/community-build/src/test/java/sbt/internal/communitybuild/FailureSummarizer.java
@@ -0,0 +1,36 @@
+package sbt.internal.communitybuild;
+
+import java.util.List;
+
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+public class FailureSummarizer extends RunListener {
+ @Override
+ public void testRunFinished(Result result) throws Exception {
+ super.testRunFinished(result);
+ if (result.getFailureCount() > 0) {
+ Thread.sleep(500); // pause to give sbt log buffers some time to flush
+ summarizeFailures(result.getFailures());
+ }
+ }
+
+ private void summarizeFailures(List failures) {
+ err("********************************************************************************");
+ err("Failed projects:");
+ for (Failure f : failures) {
+ err(" - " + getProjectName(f.getDescription()));
+ }
+ err("********************************************************************************");
+ }
+
+ private String getProjectName(Description desc) {
+ return desc.getClassName() + "." + desc.getMethodName();
+ }
+
+ private void err(String msg) {
+ System.err.println(msg);
+ }
+}
diff --git a/community-build/src/test/scala/sbt/internal/communitybuild/CommunityBuildTest.scala b/community-build/src/test/scala/sbt/internal/communitybuild/CommunityBuildTest.scala
new file mode 100644
index 000000000..3dd037e0d
--- /dev/null
+++ b/community-build/src/test/scala/sbt/internal/communitybuild/CommunityBuildTest.scala
@@ -0,0 +1,21 @@
+package sbt
+package internal
+package communitybuild
+
+import org.junit.Test
+import org.junit.Assert.fail
+import org.junit.experimental.categories.Category
+
+import CommunityBuildRunner.run
+
+class TestCategory
+
+given testRunner: CommunityBuildRunner with
+ override def failWith(msg: String) = { fail(msg); ??? }
+
+@Category(Array(classOf[TestCategory]))
+class CommunityBuildTestA:
+ @Test def parboiled2 = projects.parboiled2.run()
+ @Test def `sbt-compile-benchmark` = projects.`sbt-compile-benchmark`.run()
+ @Test def scalaz = projects.scalaz.run()
+end CommunityBuildTestA
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 05f8234eb..384b91e38 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -93,6 +93,7 @@ object Dependencies {
)
val scalacheck = "org.scalacheck" %% "scalacheck" % "1.19.0"
val junit = "junit" % "junit" % "4.13.2"
+ val junitInterface = "com.github.sbt" % "junit-interface" % "0.13.3"
val scalaVerify = "com.eed3si9n.verify" %% "verify" % "1.0.0"
val templateResolverApi = "org.scala-sbt" % "template-resolver" % "0.1"
val remoteapis =