[2.0.x] fix: restore the pipelining early output on an action-cache hit (#9793)

Problem
compileIncremental declares the analysis file and the classes directory
as cache outputs, but not the early (pickle) jar or the early analysis.
A cache hit therefore leaves a subproject with a restored analysis and
no early jar. On the next incremental round Zinc has no jar to merge
scalac's pickles into, so the early jar ends up holding only that
round's TASTy/sig files, afterEarlyOutput(true) fires, and every
downstream subproject is compiled against it and fails with "Not found"
for each upstream type that was not just recompiled.

Solution
- Declare the early jar and the early analysis file as outputs of
  compileIncremental when exportPipelining produced them, so a hit
  restores them.
- Restored outputs are symlinks into the CAS, and Zinc rewrites the
  early jar in place when it merges a round's pickles. Replace a
  symlinked early jar by a real copy before Zinc runs so the cached
  blob is never modified.
- When a previous analysis exists but the early jar does not (a cache
  written before the jar was an output, exportPipelining switched on
  for an existing build, a deleted early directory), recompile the
  subproject from scratch, which writes a complete jar.

main/src/main/scala/sbt/Defaults.scala

@@ -2337,15 +2337,32 @@ object Defaults extends BuildCommon {
       val setup: Setup = (TaskZero / compileIncSetup).value
       val c = fileConverter.value
       val store = analysisStore(compileAnalysisFile.value.toPath(), c)
+      val earlyAnalysisFile = (earlyCompileAnalysisFile.value: @nowarn("msg=transient")).toPath()
+      // Present iff exportPipelining: the jar scalac's -Ypickle-write pickles end up in.
+      val earlyJar = ci.options.earlyOutput.toScala.flatMap(_.getSingleOutputAsPath.toScala)
+      val ci1 = earlyJar.fold(ci)(Compiler.prepareEarlyOutput(ci, _, s.log))
       // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
-      val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci, ping))
+      val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci1, ping))
+      val dir = ci.options.classesDirectory
+      val vfDir = c.toVirtualFile(dir)
+      val dirZip = ActionCache.dirZipPath(dir)
+      // Zinc leaves the class directory alone when it invalidates nothing, so the zip the previous
+      // run left behind still describes it and re-packing only reproduces a blob the store has.
+      val packedDir =
+        if analysisResult.hasModified() || !Files.exists(dirZip) then
+          Def.declareOutputDirectory(vfDir)
+        else Def.declareOutput(c.toVirtualFile(dirZip))
       val analysisOut = c.toVirtualFile(setup.cachePath())
       val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
       store.set(contents)
       Def.declareOutput(analysisOut)
-      val dir = ci.options.classesDirectory
-      val vfDir = c.toVirtualFile(dir)
-      val packedDir = Def.declareOutputDirectory(vfDir)
+      // Downstream pipelined compiles read the early jar and the early analysis, so a cache hit
+      // has to bring them back too. Otherwise the hit leaves no early jar and the next incremental
+      // round builds one from its own pickles alone, and downstream then fails to resolve every
+      // type this subproject did not just recompile.
+      earlyJar.filter(Files.exists(_)).foreach(jar => Def.declareOutput(c.toVirtualFile(jar)))
+      if earlyJar.isDefined && Files.exists(earlyAnalysisFile) then
+        Def.declareOutput(c.toVirtualFile(earlyAnalysisFile))
       s.log.debug(s"wrote $vfDir")
       (analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef)
     }

main/src/main/scala/sbt/internal/Compiler.scala

@@ -10,12 +10,12 @@ package sbt
 package internal
 
 import java.io.{ File, PrintWriter }
-import java.nio.file.{ Path, Paths }
-import java.util.ArrayList
+import java.nio.file.{ Files, Path, Paths, StandardCopyOption }
+import java.util.{ ArrayList, Optional }
 import sbt.BuildExtra.*
 import sbt.Keys.Classpath
 import sbt.internal.CommandStrings
-import sbt.internal.inc.{ AnalyzingCompiler, ScalaInstance, ZincLmUtil }
+import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ScalaInstance, ZincLmUtil }
 import sbt.internal.inc.classpath.ClasspathUtil
 import sbt.internal.worker.{ ClientJobParams, ScalaInstanceConfig }
 import sbt.internal.worker1.{ ConsoleInfo, WorkerMain }
@@ -34,8 +34,10 @@ import sbt.librarymanagement.{
 }
 import sbt.util.Logger
 import scala.jdk.CollectionConverters.*
+import scala.jdk.OptionConverters.*
 import scala.util.Random
 import xsbti.{ HashedVirtualFileRef, ScalaProvider }
+import xsbti.compile.{ CompileAnalysis, Inputs, PreviousResult }
 
 object Compiler:
   private val r = Random()
@@ -481,4 +483,30 @@ object Compiler:
 
     options.map(_.split(":").map(_.split(",").map(convertValue).mkString(",")).mkString(":"))
 
+  /**
+   * Gets the early output into a state Zinc can update incrementally.
+   *
+   * An action-cache hit restores the early jar as a symlink into the CAS. Zinc rewrites the jar in
+   * place when it merges a round's pickles, which would corrupt the cached blob, so Zinc gets a
+   * real copy. When there is a previous analysis but no early jar (a cache populated before the
+   * jar was an output, `exportPipelining` switched on for an existing build, a deleted `early`
+   * directory), an incremental round would create the jar from its own pickles alone, so the
+   * subproject is recompiled from scratch instead.
+   */
+  private[sbt] def prepareEarlyOutput(ci: Inputs, earlyJar: Path, log: Logger): Inputs =
+    if Files.isSymbolicLink(earlyJar) then
+      val tmp = earlyJar.resolveSibling(earlyJar.getFileName.toString + ".tmp")
+      Files.copy(earlyJar.toRealPath(), tmp, StandardCopyOption.REPLACE_EXISTING)
+      Files.move(tmp, earlyJar, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE)
+      ci
+    else if !Files.exists(earlyJar) && hasCompilations(ci.previousResult.analysis.toScala) then
+      log.debug(s"early output $earlyJar is missing, recompiling from scratch")
+      ci.withPreviousResult(PreviousResult.of(Optional.empty(), Optional.empty()))
+    else ci
+
+  private def hasCompilations(analysis: Option[CompileAnalysis]): Boolean =
+    analysis match
+      case Some(a: Analysis) => a.compilations.allCompilations.nonEmpty
+      case _                 => false
+
 end Compiler

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala

@@ -0,0 +1,6 @@
+package app
+
+import core.*
+
+object Use:
+  val other: Other[Base] = Other(List(Base("a")))

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt

@@ -0,0 +1,23 @@
+import java.util.zip.ZipFile
+import scala.jdk.CollectionConverters.*
+
+ThisBuild / scalaVersion := "3.9.0"
+ThisBuild / usePipelining := true
+
+Global / localCacheDirectory := baseDirectory.value / "diskcache"
+
+lazy val core = project
+
+lazy val app = project
+  .dependsOn(core)
+  .settings(
+    // The early jar core exports must hold every TASTy of core, not just the last round's.
+    TaskKey[Unit]("checkEarlyJar") := Def.uncached {
+      val c = fileConverter.value
+      val jar = c.toPath((core / Compile / earlyOutput).value).toFile
+      assert(jar.exists, s"early jar $jar is missing")
+      val entries = new ZipFile(jar).entries.asScala.map(_.getName).filter(_.endsWith(".tasty")).toSet
+      val expected = Set("core/Base.tasty", "core/Other.tasty", "core/Added.tasty")
+      assert(entries == expected, s"early jar entries = $entries, expected $expected")
+    },
+  )

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala

@@ -0,0 +1,3 @@
+package core
+
+final case class Added[T](others: List[Other[T]], n: Int)

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala

@@ -0,0 +1,6 @@
+package app
+
+import core.*
+
+object UseAdded:
+  val added: Added[Base] = Added(List(Use.other), 0)

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala

@@ -0,0 +1,3 @@
+package core
+
+final case class Base(name: String)

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala

@@ -0,0 +1,3 @@
+package core
+
+final case class Other[T](items: List[T])

sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test

@@ -0,0 +1,12 @@
+> compile
+
+# wipe the build output but keep the disk cache: core's next compile is an action-cache hit
+$ delete target/out
+> compile
+$ exists target/out/jvm/scala-3.9.0/core/early/core_3-0.1.0-SNAPSHOT.jar
+
+# an incremental round in core after the hit: app must still see core's unchanged types
+$ copy-file changes/Added.scala core/Added.scala
+$ copy-file changes/UseAdded.scala app/UseAdded.scala
+> compile
+> app/checkEarlyJar

util-cache/src/main/scala/sbt/util/ActionCache.scala

@@ -383,6 +383,10 @@ object ActionCache:
     outputs += vf
     vf
 
+  /** The zip `packageDirectory` writes for `dirPath`, as a sibling of the directory itself. */
+  def dirZipPath(dirPath: Path): Path =
+    Paths.get(dirPath.toString + dirZipExt)
+
   def packageDirectory(
       dir: VirtualFileRef,
       conv: FileConverter,