diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 7c89c67aa..b641e38f3 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -2337,15 +2337,32 @@ object Defaults extends BuildCommon { val setup: Setup = (TaskZero / compileIncSetup).value val c = fileConverter.value val store = analysisStore(compileAnalysisFile.value.toPath(), c) + val earlyAnalysisFile = (earlyCompileAnalysisFile.value: @nowarn("msg=transient")).toPath() + // Present iff exportPipelining: the jar scalac's -Ypickle-write pickles end up in. + val earlyJar = ci.options.earlyOutput.toScala.flatMap(_.getSingleOutputAsPath.toScala) + val ci1 = earlyJar.fold(ci)(Compiler.prepareEarlyOutput(ci, _, s.log)) // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too? - val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci, ping)) + val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci1, ping)) + val dir = ci.options.classesDirectory + val vfDir = c.toVirtualFile(dir) + val dirZip = ActionCache.dirZipPath(dir) + // Zinc leaves the class directory alone when it invalidates nothing, so the zip the previous + // run left behind still describes it and re-packing only reproduces a blob the store has. + val packedDir = + if analysisResult.hasModified() || !Files.exists(dirZip) then + Def.declareOutputDirectory(vfDir) + else Def.declareOutput(c.toVirtualFile(dirZip)) val analysisOut = c.toVirtualFile(setup.cachePath()) val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup()) store.set(contents) Def.declareOutput(analysisOut) - val dir = ci.options.classesDirectory - val vfDir = c.toVirtualFile(dir) - val packedDir = Def.declareOutputDirectory(vfDir) + // Downstream pipelined compiles read the early jar and the early analysis, so a cache hit + // has to bring them back too. Otherwise the hit leaves no early jar and the next incremental + // round builds one from its own pickles alone, and downstream then fails to resolve every + // type this subproject did not just recompile. + earlyJar.filter(Files.exists(_)).foreach(jar => Def.declareOutput(c.toVirtualFile(jar))) + if earlyJar.isDefined && Files.exists(earlyAnalysisFile) then + Def.declareOutput(c.toVirtualFile(earlyAnalysisFile)) s.log.debug(s"wrote $vfDir") (analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef) } diff --git a/main/src/main/scala/sbt/internal/Compiler.scala b/main/src/main/scala/sbt/internal/Compiler.scala index 5ecd5acdc..45cd8919e 100644 --- a/main/src/main/scala/sbt/internal/Compiler.scala +++ b/main/src/main/scala/sbt/internal/Compiler.scala @@ -10,12 +10,12 @@ package sbt package internal import java.io.{ File, PrintWriter } -import java.nio.file.{ Path, Paths } -import java.util.ArrayList +import java.nio.file.{ Files, Path, Paths, StandardCopyOption } +import java.util.{ ArrayList, Optional } import sbt.BuildExtra.* import sbt.Keys.Classpath import sbt.internal.CommandStrings -import sbt.internal.inc.{ AnalyzingCompiler, ScalaInstance, ZincLmUtil } +import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ScalaInstance, ZincLmUtil } import sbt.internal.inc.classpath.ClasspathUtil import sbt.internal.worker.{ ClientJobParams, ScalaInstanceConfig } import sbt.internal.worker1.{ ConsoleInfo, WorkerMain } @@ -34,8 +34,10 @@ import sbt.librarymanagement.{ } import sbt.util.Logger import scala.jdk.CollectionConverters.* +import scala.jdk.OptionConverters.* import scala.util.Random import xsbti.{ HashedVirtualFileRef, ScalaProvider } +import xsbti.compile.{ CompileAnalysis, Inputs, PreviousResult } object Compiler: private val r = Random() @@ -481,4 +483,30 @@ object Compiler: options.map(_.split(":").map(_.split(",").map(convertValue).mkString(",")).mkString(":")) + /** + * Gets the early output into a state Zinc can update incrementally. + * + * An action-cache hit restores the early jar as a symlink into the CAS. Zinc rewrites the jar in + * place when it merges a round's pickles, which would corrupt the cached blob, so Zinc gets a + * real copy. When there is a previous analysis but no early jar (a cache populated before the + * jar was an output, `exportPipelining` switched on for an existing build, a deleted `early` + * directory), an incremental round would create the jar from its own pickles alone, so the + * subproject is recompiled from scratch instead. + */ + private[sbt] def prepareEarlyOutput(ci: Inputs, earlyJar: Path, log: Logger): Inputs = + if Files.isSymbolicLink(earlyJar) then + val tmp = earlyJar.resolveSibling(earlyJar.getFileName.toString + ".tmp") + Files.copy(earlyJar.toRealPath(), tmp, StandardCopyOption.REPLACE_EXISTING) + Files.move(tmp, earlyJar, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) + ci + else if !Files.exists(earlyJar) && hasCompilations(ci.previousResult.analysis.toScala) then + log.debug(s"early output $earlyJar is missing, recompiling from scratch") + ci.withPreviousResult(PreviousResult.of(Optional.empty(), Optional.empty())) + else ci + + private def hasCompilations(analysis: Option[CompileAnalysis]): Boolean = + analysis match + case Some(a: Analysis) => a.compilations.allCompilations.nonEmpty + case _ => false + end Compiler diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala new file mode 100644 index 000000000..4c15c1f8e --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/app/Use.scala @@ -0,0 +1,6 @@ +package app + +import core.* + +object Use: + val other: Other[Base] = Other(List(Base("a"))) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt new file mode 100644 index 000000000..8dba90876 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/build.sbt @@ -0,0 +1,23 @@ +import java.util.zip.ZipFile +import scala.jdk.CollectionConverters.* + +ThisBuild / scalaVersion := "3.9.0" +ThisBuild / usePipelining := true + +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +lazy val core = project + +lazy val app = project + .dependsOn(core) + .settings( + // The early jar core exports must hold every TASTy of core, not just the last round's. + TaskKey[Unit]("checkEarlyJar") := Def.uncached { + val c = fileConverter.value + val jar = c.toPath((core / Compile / earlyOutput).value).toFile + assert(jar.exists, s"early jar $jar is missing") + val entries = new ZipFile(jar).entries.asScala.map(_.getName).filter(_.endsWith(".tasty")).toSet + val expected = Set("core/Base.tasty", "core/Other.tasty", "core/Added.tasty") + assert(entries == expected, s"early jar entries = $entries, expected $expected") + }, + ) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala new file mode 100644 index 000000000..1551bfb50 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/Added.scala @@ -0,0 +1,3 @@ +package core + +final case class Added[T](others: List[Other[T]], n: Int) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala new file mode 100644 index 000000000..cc6956837 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/changes/UseAdded.scala @@ -0,0 +1,6 @@ +package app + +import core.* + +object UseAdded: + val added: Added[Base] = Added(List(Use.other), 0) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala new file mode 100644 index 000000000..c9020f39d --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Base.scala @@ -0,0 +1,3 @@ +package core + +final case class Base(name: String) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala new file mode 100644 index 000000000..dbf4b13f4 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/core/Other.scala @@ -0,0 +1,3 @@ +package core + +final case class Other[T](items: List[T]) diff --git a/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test new file mode 100644 index 000000000..340f838a0 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/pipelining-early-jar-restored/test @@ -0,0 +1,12 @@ +> compile + +# wipe the build output but keep the disk cache: core's next compile is an action-cache hit +$ delete target/out +> compile +$ exists target/out/jvm/scala-3.9.0/core/early/core_3-0.1.0-SNAPSHOT.jar + +# an incremental round in core after the hit: app must still see core's unchanged types +$ copy-file changes/Added.scala core/Added.scala +$ copy-file changes/UseAdded.scala app/UseAdded.scala +> compile +> app/checkEarlyJar diff --git a/sbt-app/src/sbt-test/source-dependencies/pipelining-java/disabled b/sbt-app/src/sbt-test/source-dependencies/pipelining-java/test similarity index 100% rename from sbt-app/src/sbt-test/source-dependencies/pipelining-java/disabled rename to sbt-app/src/sbt-test/source-dependencies/pipelining-java/test diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 292fdadff..1f46b8f85 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -383,6 +383,10 @@ object ActionCache: outputs += vf vf + /** The zip `packageDirectory` writes for `dirPath`, as a sibling of the directory itself. */ + def dirZipPath(dirPath: Path): Path = + Paths.get(dirPath.toString + dirZipExt) + def packageDirectory( dir: VirtualFileRef, conv: FileConverter,