From 56ff180cdfee01da22c8b9fcbf8f9fa6ca1d2427 Mon Sep 17 00:00:00 2001 From: PhoenixmitX <39215472+PhoenixmitX@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:49:30 +0200 Subject: [PATCH] [2.0.x] fix: restore the pipelining early output on an action-cache hit (#9793) Problem compileIncremental declares the analysis file and the classes directory as cache outputs, but not the early (pickle) jar or the early analysis. A cache hit therefore leaves a subproject with a restored analysis and no early jar. On the next incremental round Zinc has no jar to merge scalac's pickles into, so the early jar ends up holding only that round's TASTy/sig files, afterEarlyOutput(true) fires, and every downstream subproject is compiled against it and fails with "Not found" for each upstream type that was not just recompiled. Solution - Declare the early jar and the early analysis file as outputs of compileIncremental when exportPipelining produced them, so a hit restores them. - Restored outputs are symlinks into the CAS, and Zinc rewrites the early jar in place when it merges a round's pickles. Replace a symlinked early jar by a real copy before Zinc runs so the cached blob is never modified. - When a previous analysis exists but the early jar does not (a cache written before the jar was an output, exportPipelining switched on for an existing build, a deleted early directory), recompile the subproject from scratch, which writes a complete jar. --- main/src/main/scala/sbt/Defaults.scala | 25 +++++++++++--- .../main/scala/sbt/internal/Compiler.scala | 34 +++++++++++++++++-- .../app/Use.scala | 6 ++++ .../pipelining-early-jar-restored/build.sbt | 23 +++++++++++++ .../changes/Added.scala | 3 ++ .../changes/UseAdded.scala | 6 ++++ .../core/Base.scala | 3 ++ .../core/Other.scala | 3 ++ .../cache/pipelining-early-jar-restored/test | 12 +++++++ .../pipelining-java/{disabled => test} | 0 .../src/main/scala/sbt/util/ActionCache.scala | 4 +++ 11 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala create mode 100644 sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test rename sbt-app/src/sbt-test/source-dependencies/pipelining-java/{disabled => test} (100%) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 7c89c67aa..b641e38f3 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -2337,15 +2337,32 @@ object Defaults extends BuildCommon { val setup: Setup = (TaskZero / compileIncSetup).value val c = fileConverter.value val store = analysisStore(compileAnalysisFile.value.toPath(), c) + val earlyAnalysisFile = (earlyCompileAnalysisFile.value: @nowarn("msg=transient")).toPath() + // Present iff exportPipelining: the jar scalac's -Ypickle-write pickles end up in. + val earlyJar = ci.options.earlyOutput.toScala.flatMap(_.getSingleOutputAsPath.toScala) + val ci1 = earlyJar.fold(ci)(Compiler.prepareEarlyOutput(ci, _, s.log)) // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too? - val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci, ping)) + val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci1, ping)) + val dir = ci.options.classesDirectory + val vfDir = c.toVirtualFile(dir) + val dirZip = ActionCache.dirZipPath(dir) + // Zinc leaves the class directory alone when it invalidates nothing, so the zip the previous + // run left behind still describes it and re-packing only reproduces a blob the store has. + val packedDir = + if analysisResult.hasModified() || !Files.exists(dirZip) then + Def.declareOutputDirectory(vfDir) + else Def.declareOutput(c.toVirtualFile(dirZip)) val analysisOut = c.toVirtualFile(setup.cachePath()) val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup()) store.set(contents) Def.declareOutput(analysisOut) - val dir = ci.options.classesDirectory - val vfDir = c.toVirtualFile(dir) - val packedDir = Def.declareOutputDirectory(vfDir) + // Downstream pipelined compiles read the early jar and the early analysis, so a cache hit + // has to bring them back too. Otherwise the hit leaves no early jar and the next incremental + // round builds one from its own pickles alone, and downstream then fails to resolve every + // type this subproject did not just recompile. + earlyJar.filter(Files.exists(_)).foreach(jar => Def.declareOutput(c.toVirtualFile(jar))) + if earlyJar.isDefined && Files.exists(earlyAnalysisFile) then + Def.declareOutput(c.toVirtualFile(earlyAnalysisFile)) s.log.debug(s"wrote $vfDir") (analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef) } diff --git a/main/src/main/scala/sbt/internal/Compiler.scala b/main/src/main/scala/sbt/internal/Compiler.scala index 5ecd5acdc..45cd8919e 100644 --- a/main/src/main/scala/sbt/internal/Compiler.scala +++ b/main/src/main/scala/sbt/internal/Compiler.scala @@ -10,12 +10,12 @@ package sbt package internal import java.io.{ File, PrintWriter } -import java.nio.file.{ Path, Paths } -import java.util.ArrayList +import java.nio.file.{ Files, Path, Paths, StandardCopyOption } +import java.util.{ ArrayList, Optional } import sbt.BuildExtra.* import sbt.Keys.Classpath import sbt.internal.CommandStrings -import sbt.internal.inc.{ AnalyzingCompiler, ScalaInstance, ZincLmUtil } +import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ScalaInstance, ZincLmUtil } import sbt.internal.inc.classpath.ClasspathUtil import sbt.internal.worker.{ ClientJobParams, ScalaInstanceConfig } import sbt.internal.worker1.{ ConsoleInfo, WorkerMain } @@ -34,8 +34,10 @@ import sbt.librarymanagement.{ } import sbt.util.Logger import scala.jdk.CollectionConverters.* +import scala.jdk.OptionConverters.* import scala.util.Random import xsbti.{ HashedVirtualFileRef, ScalaProvider } +import xsbti.compile.{ CompileAnalysis, Inputs, PreviousResult } object Compiler: private val r = Random() @@ -481,4 +483,30 @@ object Compiler: options.map(_.split(":").map(_.split(",").map(convertValue).mkString(",")).mkString(":")) + /** + * Gets the early output into a state Zinc can update incrementally. + * + * An action-cache hit restores the early jar as a symlink into the CAS. Zinc rewrites the jar in + * place when it merges a round's pickles, which would corrupt the cached blob, so Zinc gets a + * real copy. When there is a previous analysis but no early jar (a cache populated before the + * jar was an output, `exportPipelining` switched on for an existing build, a deleted `early` + * directory), an incremental round would create the jar from its own pickles alone, so the + * subproject is recompiled from scratch instead. + */ + private[sbt] def prepareEarlyOutput(ci: Inputs, earlyJar: Path, log: Logger): Inputs = + if Files.isSymbolicLink(earlyJar) then + val tmp = earlyJar.resolveSibling(earlyJar.getFileName.toString + ".tmp") + Files.copy(earlyJar.toRealPath(), tmp, StandardCopyOption.REPLACE_EXISTING) + Files.move(tmp, earlyJar, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) + ci + else if !Files.exists(earlyJar) && hasCompilations(ci.previousResult.analysis.toScala) then + log.debug(s"early output $earlyJar is missing, recompiling from scratch") + ci.withPreviousResult(PreviousResult.of(Optional.empty(), Optional.empty())) + else ci + + private def hasCompilations(analysis: Option[CompileAnalysis]): Boolean = + analysis match + case Some(a: Analysis) => a.compilations.allCompilations.nonEmpty + case _ => false + end Compiler diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala new file mode 100644 index 000000000..4c15c1f8e --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala @@ -0,0 +1,6 @@ +package app + +import core.* + +object Use: + val other: Other[Base] = Other(List(Base("a"))) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt new file mode 100644 index 000000000..8dba90876 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt @@ -0,0 +1,23 @@ +import java.util.zip.ZipFile +import scala.jdk.CollectionConverters.* + +ThisBuild / scalaVersion := "3.9.0" +ThisBuild / usePipelining := true + +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +lazy val core = project + +lazy val app = project + .dependsOn(core) + .settings( + // The early jar core exports must hold every TASTy of core, not just the last round's. + TaskKey[Unit]("checkEarlyJar") := Def.uncached { + val c = fileConverter.value + val jar = c.toPath((core / Compile / earlyOutput).value).toFile + assert(jar.exists, s"early jar $jar is missing") + val entries = new ZipFile(jar).entries.asScala.map(_.getName).filter(_.endsWith(".tasty")).toSet + val expected = Set("core/Base.tasty", "core/Other.tasty", "core/Added.tasty") + assert(entries == expected, s"early jar entries = $entries, expected $expected") + }, + ) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala new file mode 100644 index 000000000..1551bfb50 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala @@ -0,0 +1,3 @@ +package core + +final case class Added[T](others: List[Other[T]], n: Int) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala new file mode 100644 index 000000000..cc6956837 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala @@ -0,0 +1,6 @@ +package app + +import core.* + +object UseAdded: + val added: Added[Base] = Added(List(Use.other), 0) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala new file mode 100644 index 000000000..c9020f39d --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala @@ -0,0 +1,3 @@ +package core + +final case class Base(name: String) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala new file mode 100644 index 000000000..dbf4b13f4 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala @@ -0,0 +1,3 @@ +package core + +final case class Other[T](items: List[T]) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test new file mode 100644 index 000000000..340f838a0 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test @@ -0,0 +1,12 @@ +> compile + +# wipe the build output but keep the disk cache: core's next compile is an action-cache hit +$ delete target/out +> compile +$ exists target/out/jvm/scala-3.9.0/core/early/core_3-0.1.0-SNAPSHOT.jar + +# an incremental round in core after the hit: app must still see core's unchanged types +$ copy-file changes/Added.scala core/Added.scala +$ copy-file changes/UseAdded.scala app/UseAdded.scala +> compile +> app/checkEarlyJar diff --git a/sbt-app/src/sbt-test/source-dependencies/pipelining-java/disabled b/sbt-app/src/sbt-test/source-dependencies/pipelining-java/test similarity index 100% rename from sbt-app/src/sbt-test/source-dependencies/pipelining-java/disabled rename to sbt-app/src/sbt-test/source-dependencies/pipelining-java/test diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 292fdadff..1f46b8f85 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -383,6 +383,10 @@ object ActionCache: outputs += vf vf + /** The zip `packageDirectory` writes for `dirPath`, as a sibling of the directory itself. */ + def dirZipPath(dirPath: Path): Path = + Paths.get(dirPath.toString + dirZipExt) + def packageDirectory( dir: VirtualFileRef, conv: FileConverter,