Merge pull request #5766 from eed3si9n/wip/use_pipelining

Add exportPipelining setting to implement subproject opt-out

build.sbt

@@ -14,6 +14,7 @@ ThisBuild / version := {
 ThisBuild / scalafmtOnCompile := !(Global / insideCI).value
 ThisBuild / Test / scalafmtOnCompile := !(Global / insideCI).value
 ThisBuild / turbo := true
+ThisBuild / SettingKey[Boolean]("usePipelining") := true
 
 val excludeLint = SettingKey[Set[Def.KeyedInitialize[_]]]("excludeLintKeys")
 Global / excludeLint := (Global / excludeLint).?.value.getOrElse(Set.empty)
@@ -796,6 +797,7 @@ lazy val coreMacrosProj = (project in file("core-macros"))
     baseSettings :+ (crossScalaVersions := (scala212 :: scala213 :: Nil)),
     name := "Core Macros",
     libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value,
+    SettingKey[Boolean]("exportPipelining") := false,
     mimaSettings,
   )
 
@@ -913,6 +915,7 @@ lazy val mainProj = (project in file("main"))
     sourceManaged in (Compile, generateContrabands) := baseDirectory.value / "src" / "main" / "contraband-scala",
     testOptions in Test += Tests
       .Argument(TestFrameworks.ScalaCheck, "-minSuccessfulTests", "1000"),
+    SettingKey[Boolean]("usePipelining") := false,
     mimaSettings,
     mimaBinaryIssueFilters ++= Vector(
       // New and changed methods on KeyIndex. internal.

main/src/main/scala/sbt/Defaults.scala

@@ -325,6 +325,7 @@ object Defaults extends BuildCommon {
       },
       turbo :== SysProp.turbo,
       usePipelining :== SysProp.pipelining,
+      exportPipelining := usePipelining.value,
       useScalaReplJLine :== false,
       scalaInstanceTopLoader := {
         if (!useScalaReplJLine.value) classOf[org.jline.terminal.Terminal].getClassLoader
@@ -389,6 +390,7 @@ object Defaults extends BuildCommon {
   )
 
   private[sbt] lazy val buildLevelJvmSettings: Seq[Setting[_]] = Seq(
+    exportPipelining := usePipelining.value,
     rootPaths := {
       val app = appConfiguration.value
       val base = app.baseDirectory.getCanonicalFile
@@ -785,7 +787,7 @@ object Defaults extends BuildCommon {
     scalacOptions := {
       val old = scalacOptions.value
       val converter = fileConverter.value
-      if (usePipelining.value)
+      if (exportPipelining.value)
         Vector("-Ypickle-java", "-Ypickle-write", converter.toPath(earlyOutput.value).toString) ++ old
       else old
     },
@@ -1941,8 +1943,9 @@ object Defaults extends BuildCommon {
     val setup: Setup = compileIncSetup.value
     val useBinary: Boolean = enableBinaryCompileAnalysis.value
     val analysisResult: CompileResult = compileIncremental.value
+    val exportP = exportPipelining.value
     // Save analysis midway if pipelining is enabled
-    if (analysisResult.hasModified && setup.incrementalCompilerOptions.pipelining) {
+    if (analysisResult.hasModified && exportP) {
       val store =
         MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary)
       val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
@@ -2079,6 +2082,10 @@ object Defaults extends BuildCommon {
         val vs = sources.value.toVector map { x =>
           c.toVirtualFile(x.toPath)
         }
+        val eo = CompileOutput(c.toPath(earlyOutput.value))
+        val eoOpt =
+          if (exportPipelining.value) Some(eo)
+          else None
         CompileOptions.of(
           cp.toArray,
           vs.toArray,
@@ -2091,7 +2098,7 @@ object Defaults extends BuildCommon {
           None.toOptional: Optional[NioPath],
           Some(fileConverter.value).toOptional,
           Some(reusableStamper.value).toOptional,
-          Some(CompileOutput(c.toPath(earlyOutput.value))).toOptional,
+          eoOpt.toOptional,
         )
       },
       compilerReporter := {

main/src/main/scala/sbt/Keys.scala

@@ -383,6 +383,7 @@ object Keys {
   val pushRemoteCacheTo = settingKey[Option[Resolver]]("The resolver to publish remote cache to.")
   val remoteCachePom = taskKey[File]("Generates a pom for publishing when publishing Maven-style.")
   val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting)
+  val exportPipelining = settingKey[Boolean]("Product early output so downstream subprojects can do pipelining.").withRank(BSetting)
 
   val bspTargetIdentifier = settingKey[BuildTargetIdentifier]("Id for BSP build target.").withRank(DSetting)
   val bspWorkspace = settingKey[Map[BuildTargetIdentifier, Scope]]("Mapping of BSP build targets to sbt scopes").withRank(DSetting)

main/src/main/scala/sbt/internal/ClasspathImpl.scala

@@ -31,15 +31,22 @@ private[sbt] object ClasspathImpl {
   // we can't reduce the track level.
   def exportedPicklesTask: Initialize[Task[VirtualClasspath]] =
     Def.task {
-      val module = projectID.value
+      // conditional task: do not refactor
-      val config = configuration.value
+      if (exportPipelining.value) {
-      val products = pickleProducts.value
+        val module = projectID.value
-      val analysis = compileEarly.value
+        val config = configuration.value
-      val xs = products map { _ -> analysis }
+        val products = pickleProducts.value
-      for { (f, analysis) <- xs } yield APIMappings
+        val analysis = compileEarly.value
-        .store(analyzed(f, analysis), apiURL.value)
+        val xs = products map { _ -> analysis }
-        .put(moduleID.key, module)
+        for { (f, analysis) <- xs } yield APIMappings
-        .put(configuration.key, config)
+          .store(analyzed(f, analysis), apiURL.value)
+          .put(moduleID.key, module)
+          .put(configuration.key, config)
+      } else {
+        val c = fileConverter.value
+        val ps = exportedProducts.value
+        ps.map(attr => attr.map(x => c.toVirtualFile(x.toPath)))
+      }
     }
 
   def trackedExportedProducts(track: TrackLevel): Initialize[Task[Classpath]] =

project/Dependencies.scala

@@ -14,7 +14,7 @@ object Dependencies {
   private val ioVersion = nightlyVersion.getOrElse("1.4.0-M6")
   private val lmVersion =
     sys.props.get("sbt.build.lm.version").orElse(nightlyVersion).getOrElse("1.4.0-M2")
-  val zincVersion = nightlyVersion.getOrElse("1.4.0-M9")
+  val zincVersion = nightlyVersion.getOrElse("1.4.0-M10")
 
   private val sbtIO = "org.scala-sbt" %% "io" % ioVersion
 

project/Util.scala

@@ -20,7 +20,7 @@ object Util {
 
   lazy val javaOnlySettings: Seq[Setting[_]] = Seq(
     // crossPaths := false,
-    compileOrder := CompileOrder.JavaThenScala,
+    // compileOrder := CompileOrder.JavaThenScala,
     unmanagedSourceDirectories in Compile := Seq((javaSource in Compile).value)
   )