From 0712811dff4d29df345050807ed06f5cc8f85c0e Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 11 May 2026 04:05:01 -0400 Subject: [PATCH] [2.0.x] Cache analysis using file last modified **Problem** Current MixedAnalyzingCompiler analysis cache caches using the last write, assuming all writing happens via it. However, this doesn't work with sbt 2.x caching system where the gz file under it can switch. **Solution** This switches to using caffeine caching, which includes file size and file timestamp for local analysis caching. --- main/src/main/scala/sbt/Defaults.scala | 37 +++++++--------- .../main/scala/sbt/internal/BuildDef.scala | 43 +++++++++++++++++-- project/Dependencies.scala | 4 +- .../source-dependencies/move-file/A.scala | 5 +++ .../source-dependencies/move-file/build.sbt | 16 +++++++ .../source-dependencies/move-file/test | 16 +++++++ 6 files changed, 95 insertions(+), 26 deletions(-) create mode 100644 sbt-app/src/sbt-test/source-dependencies/move-file/A.scala create mode 100644 sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt create mode 100644 sbt-app/src/sbt-test/source-dependencies/move-file/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 0bb4c181c..dd97ae531 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -79,18 +79,13 @@ import sjsonnew.* import scala.annotation.nowarn import scala.collection.immutable.ListMap import scala.concurrent.duration.* +import scala.jdk.OptionConverters.* import scala.util.control.NonFatal import scala.xml.NodeSeq // incremental compiler import sbt.SlashSyntax0.* -import sbt.internal.inc.{ - Analysis, - AnalyzingCompiler, - ManagedLoggedReporter, - MixedAnalyzingCompiler, - ScalaInstance -} +import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ManagedLoggedReporter, ScalaInstance } import sbt.internal.io.Retry import xsbti.{ AppConfiguration, @@ -2181,8 +2176,9 @@ object Defaults extends BuildCommon { val setup: Setup = compileIncSetup.value val _ = compileIncremental.value val exportP = exportPipelining.value + val c = fileConverter.value // Save analysis midway if pipelining is enabled - val store = analysisStore(compileAnalysisFile) + val store = analysisStore(compileAnalysisFile.value.toPath(), c) val contents = store.unsafeGet() if (exportP) { // this stores the early analysis (again) in case the subproject contains a macro @@ -2207,8 +2203,9 @@ object Defaults extends BuildCommon { .debug(s"${name.value}: compileEarly: blocking on earlyOutputPing") earlyOutputPing.await.value }) { - val store = analysisStore(earlyCompileAnalysisFile) - store.get.toOption match { + val c = fileConverter.value + val store = analysisStore(earlyCompileAnalysisFile.value.toPath(), c) + store.get.toScala match { case Some(contents) => contents.getAnalysis case _ => Analysis.empty } @@ -2219,8 +2216,8 @@ object Defaults extends BuildCommon { def compileTask: Initialize[Task[CompileAnalysis]] = Def.task { val setup: Setup = compileIncSetup.value - val store = analysisStore(compileAnalysisFile) val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) // TODO - expose bytecode manipulation phase. val analysisResult: CompileResult = manipulateBytecode.value if (analysisResult.hasModified) { @@ -2247,7 +2244,7 @@ object Defaults extends BuildCommon { val dir = c.toPath(backendOutput.value).toFile result match case Result.Value(res) => - val store = analysisStore(compileAnalysisFile) + val store = analysisStore(compileAnalysisFile.value.toPath(), c) val analysis = store.unsafeGet().getAnalysis() reporter.sendSuccessReport(analysis) bspTask.notifySuccess(analysis) @@ -2270,8 +2267,8 @@ object Defaults extends BuildCommon { val ci2 = (compile / compileInputs2).value val ping = (TaskZero / earlyOutputPing).value val setup: Setup = (TaskZero / compileIncSetup).value - val store = analysisStore(compileAnalysisFile) val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too? val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci, ping)) val analysisOut = c.toVirtualFile(setup.cachePath()) @@ -2354,7 +2351,7 @@ object Defaults extends BuildCommon { override def definesClass(classpathEntry: VirtualFile): DefinesClass = cachedPerEntryDefinesClassLookup(classpathEntry) val extra = extraIncOptions.value.map(t2) - val store = analysisStore(earlyCompileAnalysisFile) + val store = analysisStore(earlyCompileAnalysisFile.value.toPath(), converter) val eaOpt = if exportPipelining.value then Some(store) else None Setup.of( lookup, @@ -2469,8 +2466,9 @@ object Defaults extends BuildCommon { def compileAnalysisSettings: Seq[Setting[?]] = Seq( previousCompile := Def.uncached { val setup = compileIncSetup.value - val store = analysisStore(compileAnalysisFile) - val prev = store.get().toOption match { + val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) + val prev = store.get().toScala match { case Some(contents) => val analysis = Option(contents.getAnalysis).toOptional val setup = Option(contents.getMiniSetup).toOptional @@ -2481,11 +2479,8 @@ object Defaults extends BuildCommon { } ) - private inline def analysisStore(inline analysisFile: TaskKey[File]): AnalysisStore = - MixedAnalyzingCompiler.staticCachedStore( - analysisFile = analysisFile.value.toPath, - useTextAnalysis = false, - ) + private def analysisStore(path: NioPath, converter: FileConverter): AnalysisStore = + BuildDef.cachedAnalysisStore(path, converter) def printWarningsTask: Initialize[Task[Unit]] = Def.task { diff --git a/main/src/main/scala/sbt/internal/BuildDef.scala b/main/src/main/scala/sbt/internal/BuildDef.scala index 6ac7b226e..0a42f74bf 100644 --- a/main/src/main/scala/sbt/internal/BuildDef.scala +++ b/main/src/main/scala/sbt/internal/BuildDef.scala @@ -11,17 +11,19 @@ package internal import com.github.benmanes.caffeine.cache.{ Cache as CCache, Caffeine, Weigher } import java.io.File -import java.nio.file.{ Files, NoSuchFileException } +import java.nio.file.{ Files, NoSuchFileException, Path as NioPath } import java.nio.file.attribute.BasicFileAttributes +import java.util.Optional import Keys.{ organization, thisProject, autoGeneratedProject, publish, publishLocal, skip } import Def.Setting // import sbt.ProjectExtra.apply import sbt.io.Hash import sbt.internal.util.{ Attributed, StringAttributeMap } -import sbt.internal.inc.{ FileAnalysisStore, ReflectUtilities } +import sbt.internal.inc.{ FileAnalysisStore, MixedAnalyzingCompiler, ReflectUtilities } import sbt.util.CacheImplicits.given +import scala.jdk.OptionConverters.* import xsbti.{ FileConverter, VirtualFileRef } -import xsbti.compile.{ AnalysisContents, CompileAnalysis } +import xsbti.compile.{ AnalysisContents, AnalysisStore, CompileAnalysis } trait BuildDef { def projectDefinitions(@deprecated("unused", "") baseDirectory: File): Seq[Project] = projects @@ -134,4 +136,39 @@ private[sbt] object BuildDef: content <- getContents(VirtualFileRef.of(ref)) yield content.getAnalysis + private[sbt] def cachedAnalysisStore(path: NioPath, converter: FileConverter): AnalysisStore = + CachedAnalysisStore(path, converter) + + private class CachedAnalysisStore(path: NioPath, converter: FileConverter) extends AnalysisStore: + private val underlying = MixedAnalyzingCompiler.staticCachedStore( + analysisFile = path, + useTextAnalysis = false, + cacheLast = false, + ) + override def get: Optional[AnalysisContents] = + val ref: VirtualFileRef = converter.toVirtualFile(path) + try + val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) + if attrs.isDirectory then underlying.get + else + val lastModified = attrs.lastModifiedTime().toMillis() + val sizeBytes = attrs.size() + getOrElseUpdate(ref, lastModified, sizeBytes)(underlying.get.toScala).toJava + catch case _: NoSuchFileException => underlying.get + + override def unsafeGet: AnalysisContents = get.toScala.get + + override def set(contents: AnalysisContents): Unit = + val ref: VirtualFileRef = converter.toVirtualFile(path) + try + val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) + if attrs.isDirectory then underlying.set(contents) + else + val lastModified = attrs.lastModifiedTime().toMillis() + val sizeBytes = attrs.size() + inMemoryAnalysisCache.put(ref.id(), (Some(contents), lastModified, sizeBytes)) + underlying.set(contents) + catch case _: NoSuchFileException => underlying.set(contents) + + end CachedAnalysisStore end BuildDef diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ef4b9baba..948985b9b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,8 +12,8 @@ object Dependencies { sys.env.get("BUILD_VERSION") orElse sys.props.get("sbt.build.version") // sbt modules - val ioVersion = nightlyVersion.getOrElse("1.10.5") - val zincVersion = nightlyVersion.getOrElse("2.0.0-M16") + val ioVersion = nightlyVersion.getOrElse("1.12.0") + val zincVersion = nightlyVersion.getOrElse("2.0.0-M17") private val sbtIO = "org.scala-sbt" %% "io" % ioVersion diff --git a/sbt-app/src/sbt-test/source-dependencies/move-file/A.scala b/sbt-app/src/sbt-test/source-dependencies/move-file/A.scala new file mode 100644 index 000000000..83b986f33 --- /dev/null +++ b/sbt-app/src/sbt-test/source-dependencies/move-file/A.scala @@ -0,0 +1,5 @@ +package example + +object Main { + def main(args: Array[String]): Unit = () +} diff --git a/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt b/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt new file mode 100644 index 000000000..d53253473 --- /dev/null +++ b/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt @@ -0,0 +1,16 @@ +import sbt.internal.inc.Analysis +import complete.DefaultParsers.* + +scalaVersion := "2.13.18" + +val checkStampSize = inputKey[Unit]("Verifies the accumulated number of iterations of incremental compilation.") +checkStampSize := { + val s = streams.value + val expected: Int = (Space ~> NatBasic).parsed + val analysis = (Compile / compile).value match + case a: Analysis => a + s.log.info(s"analysis: $analysis") + val sourceStampSize = analysis.readStamps.getAllSourceStamps.size + assert(sourceStampSize == expected, s"sourceStampSize = $sourceStampSize") +} + diff --git a/sbt-app/src/sbt-test/source-dependencies/move-file/test b/sbt-app/src/sbt-test/source-dependencies/move-file/test new file mode 100644 index 000000000..309eebbd8 --- /dev/null +++ b/sbt-app/src/sbt-test/source-dependencies/move-file/test @@ -0,0 +1,16 @@ +> checkStampSize 1 + +# move A.scala elsewhere +$ copy-file A.scala A.txt +$ delete A.scala +> checkStampSize 0 + +# move A.scala back +$ copy-file A.txt A.scala +$ delete A.txt +$ exists A.scala + +> show Compile/sources + +# if we recompile, we should regain stamp size 1 +> checkStampSize 1