From 566f88804571b64e4a4c664679c8d6699285418f Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 11 May 2026 04:05:01 -0400 Subject: [PATCH] [2.x] Cache analysis using file last modified **Problem** Current MixedAnalyzingCompiler analysis cache caches using the last write, assuming all writing happens via it. However, this doesn't work with sbt 2.x caching system where the gz file under it can switch. **Solution** This switches to using caffeine caching, which includes file size and file timestamp for local analysis caching. --- main/src/main/scala/sbt/Defaults.scala | 32 ++++++-------- .../main/scala/sbt/internal/BuildDef.scala | 43 +++++++++++++++++-- project/Dependencies.scala | 2 +- .../source-dependencies/move-file/build.sbt | 3 +- .../move-file/{pending => test} | 1 - 5 files changed, 56 insertions(+), 25 deletions(-) rename sbt-app/src/sbt-test/source-dependencies/move-file/{pending => test} (97%) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index b49511f0e..24a9dd2c4 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -82,13 +82,7 @@ import scala.xml.NodeSeq // incremental compiler import sbt.SlashSyntax0.* -import sbt.internal.inc.{ - Analysis, - AnalyzingCompiler, - ManagedLoggedReporter, - MixedAnalyzingCompiler, - ScalaInstance -} +import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ManagedLoggedReporter, ScalaInstance } import sbt.internal.io.Retry import xsbti.{ AppConfiguration, @@ -2136,8 +2130,9 @@ object Defaults extends BuildCommon with DefExtra { val setup: Setup = compileIncSetup.value val _ = compileIncremental.value val exportP = exportPipelining.value + val c = fileConverter.value // Save analysis midway if pipelining is enabled - val store = analysisStore(compileAnalysisFile) + val store = analysisStore(compileAnalysisFile.value.toPath(), c) val contents = store.unsafeGet() if (exportP) { // this stores the early analysis (again) in case the subproject contains a macro @@ -2162,7 +2157,8 @@ object Defaults extends BuildCommon with DefExtra { .debug(s"${name.value}: compileEarly: blocking on earlyOutputPing") earlyOutputPing.await.value }) { - val store = analysisStore(earlyCompileAnalysisFile) + val c = fileConverter.value + val store = analysisStore(earlyCompileAnalysisFile.value.toPath(), c) store.get.toScala match { case Some(contents) => contents.getAnalysis case _ => Analysis.empty @@ -2174,8 +2170,8 @@ object Defaults extends BuildCommon with DefExtra { def compileTask: Initialize[Task[CompileAnalysis]] = Def.task { val setup: Setup = compileIncSetup.value - val store = analysisStore(compileAnalysisFile) val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) // TODO - expose bytecode manipulation phase. val analysisResult: CompileResult = manipulateBytecode.value if (analysisResult.hasModified) { @@ -2202,7 +2198,7 @@ object Defaults extends BuildCommon with DefExtra { val dir = c.toPath(backendOutput.value).toFile result match case Result.Value(res) => - val store = analysisStore(compileAnalysisFile) + val store = analysisStore(compileAnalysisFile.value.toPath(), c) val analysis = store.unsafeGet().getAnalysis() reporter.sendSuccessReport(analysis) bspTask.notifySuccess(analysis) @@ -2232,8 +2228,8 @@ object Defaults extends BuildCommon with DefExtra { val ci2 = (compile / compileInputs2).value val ping = (TaskZero / earlyOutputPing).value val setup: Setup = (TaskZero / compileIncSetup).value - val store = analysisStore(compileAnalysisFile) val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too? val analysisResult = Retry.io(compileIncrementalTaskImpl(bspTask, s, ci, ping, projectId)) val analysisOut = c.toVirtualFile(setup.cachePath()) @@ -2320,7 +2316,7 @@ object Defaults extends BuildCommon with DefExtra { override def definesClass(classpathEntry: VirtualFile): DefinesClass = cachedPerEntryDefinesClassLookup(classpathEntry) val extra = extraIncOptions.value.map(t2) - val store = analysisStore(earlyCompileAnalysisFile) + val store = analysisStore(earlyCompileAnalysisFile.value.toPath(), converter) val eaOpt = if exportPipelining.value then Some(store) else None Setup.of( lookup, @@ -2435,7 +2431,8 @@ object Defaults extends BuildCommon with DefExtra { def compileAnalysisSettings: Seq[Setting[?]] = Seq( previousCompile := Def.uncached { val setup = compileIncSetup.value - val store = analysisStore(compileAnalysisFile) + val c = fileConverter.value + val store = analysisStore(compileAnalysisFile.value.toPath(), c) val prev = store.get().toScala match { case Some(contents) => val analysis = Option(contents.getAnalysis).toJava @@ -2447,11 +2444,8 @@ object Defaults extends BuildCommon with DefExtra { } ) - private inline def analysisStore(inline analysisFile: TaskKey[File]): AnalysisStore = - MixedAnalyzingCompiler.staticCachedStore( - analysisFile = analysisFile.value.toPath, - useTextAnalysis = false, - ) + private def analysisStore(path: NioPath, converter: FileConverter): AnalysisStore = + BuildDef.cachedAnalysisStore(path, converter) def printWarningsTask: Initialize[Task[Unit]] = Def.task { diff --git a/main/src/main/scala/sbt/internal/BuildDef.scala b/main/src/main/scala/sbt/internal/BuildDef.scala index 6ac7b226e..0a42f74bf 100644 --- a/main/src/main/scala/sbt/internal/BuildDef.scala +++ b/main/src/main/scala/sbt/internal/BuildDef.scala @@ -11,17 +11,19 @@ package internal import com.github.benmanes.caffeine.cache.{ Cache as CCache, Caffeine, Weigher } import java.io.File -import java.nio.file.{ Files, NoSuchFileException } +import java.nio.file.{ Files, NoSuchFileException, Path as NioPath } import java.nio.file.attribute.BasicFileAttributes +import java.util.Optional import Keys.{ organization, thisProject, autoGeneratedProject, publish, publishLocal, skip } import Def.Setting // import sbt.ProjectExtra.apply import sbt.io.Hash import sbt.internal.util.{ Attributed, StringAttributeMap } -import sbt.internal.inc.{ FileAnalysisStore, ReflectUtilities } +import sbt.internal.inc.{ FileAnalysisStore, MixedAnalyzingCompiler, ReflectUtilities } import sbt.util.CacheImplicits.given +import scala.jdk.OptionConverters.* import xsbti.{ FileConverter, VirtualFileRef } -import xsbti.compile.{ AnalysisContents, CompileAnalysis } +import xsbti.compile.{ AnalysisContents, AnalysisStore, CompileAnalysis } trait BuildDef { def projectDefinitions(@deprecated("unused", "") baseDirectory: File): Seq[Project] = projects @@ -134,4 +136,39 @@ private[sbt] object BuildDef: content <- getContents(VirtualFileRef.of(ref)) yield content.getAnalysis + private[sbt] def cachedAnalysisStore(path: NioPath, converter: FileConverter): AnalysisStore = + CachedAnalysisStore(path, converter) + + private class CachedAnalysisStore(path: NioPath, converter: FileConverter) extends AnalysisStore: + private val underlying = MixedAnalyzingCompiler.staticCachedStore( + analysisFile = path, + useTextAnalysis = false, + cacheLast = false, + ) + override def get: Optional[AnalysisContents] = + val ref: VirtualFileRef = converter.toVirtualFile(path) + try + val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) + if attrs.isDirectory then underlying.get + else + val lastModified = attrs.lastModifiedTime().toMillis() + val sizeBytes = attrs.size() + getOrElseUpdate(ref, lastModified, sizeBytes)(underlying.get.toScala).toJava + catch case _: NoSuchFileException => underlying.get + + override def unsafeGet: AnalysisContents = get.toScala.get + + override def set(contents: AnalysisContents): Unit = + val ref: VirtualFileRef = converter.toVirtualFile(path) + try + val attrs = Files.readAttributes(path, classOf[BasicFileAttributes]) + if attrs.isDirectory then underlying.set(contents) + else + val lastModified = attrs.lastModifiedTime().toMillis() + val sizeBytes = attrs.size() + inMemoryAnalysisCache.put(ref.id(), (Some(contents), lastModified, sizeBytes)) + underlying.set(contents) + catch case _: NoSuchFileException => underlying.set(contents) + + end CachedAnalysisStore end BuildDef diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0e3aa44a2..c093e1a10 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -11,7 +11,7 @@ object Dependencies { // sbt modules val ioVersion = nightlyVersion.getOrElse("1.12.0") - val zincVersion = nightlyVersion.getOrElse("2.0.0-M15") + val zincVersion = nightlyVersion.getOrElse("2.0.0-M17") private val sbtIO = "org.scala-sbt" %% "io" % ioVersion diff --git a/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt b/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt index f62cfa56f..d53253473 100644 --- a/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt +++ b/sbt-app/src/sbt-test/source-dependencies/move-file/build.sbt @@ -5,10 +5,11 @@ scalaVersion := "2.13.18" val checkStampSize = inputKey[Unit]("Verifies the accumulated number of iterations of incremental compilation.") checkStampSize := { + val s = streams.value val expected: Int = (Space ~> NatBasic).parsed val analysis = (Compile / compile).value match case a: Analysis => a - println(s"analysis: $analysis") + s.log.info(s"analysis: $analysis") val sourceStampSize = analysis.readStamps.getAllSourceStamps.size assert(sourceStampSize == expected, s"sourceStampSize = $sourceStampSize") } diff --git a/sbt-app/src/sbt-test/source-dependencies/move-file/pending b/sbt-app/src/sbt-test/source-dependencies/move-file/test similarity index 97% rename from sbt-app/src/sbt-test/source-dependencies/move-file/pending rename to sbt-app/src/sbt-test/source-dependencies/move-file/test index 358ab6596..309eebbd8 100644 --- a/sbt-app/src/sbt-test/source-dependencies/move-file/pending +++ b/sbt-app/src/sbt-test/source-dependencies/move-file/test @@ -13,5 +13,4 @@ $ exists A.scala > show Compile/sources # if we recompile, we should regain stamp size 1 -> debug > checkStampSize 1