From 544f56695ae7334d33ecb8ea400a3fae1e013a0f Mon Sep 17 00:00:00 2001 From: Dream <42954461+eureka928@users.noreply.github.com> Date: Tue, 31 Mar 2026 22:23:10 -0400 Subject: [PATCH] [2.x] feat: Add cacheVersion setting for global cache invalidation (#8993) * [2.x] feat: Add cacheVersion setting for global cache invalidation **Problem** There was no escape hatch to invalidate all task caches when needed. **Solution** Add `Global / cacheVersion` setting that incorporates into the cache key hash. Changing it invalidates all caches. Defaults to reading system property `sbt.cacheversion`, or else 0L. When 0L, the hash is identical to the previous behavior (backward compatible). Fixes #8992 * [2.x] refactor: Simplify BuildWideCacheConfiguration and add cacheVersion test - Replace auxiliary constructors with default parameter values - Add unit test verifying cacheVersion invalidates the cache * [2.x] fix: Restore auxiliary constructors for binary compatibility * [2.x] test: Improve cacheVersion scripted test and add release note - Scripted test now verifies cache invalidation via a counter that increments only when the task body actually executes - Add release note documenting the cacheVersion setting --- .../src/main/scala/sbt/BasicKeys.scala | 6 +++ main-settings/src/main/scala/sbt/Def.scala | 6 ++- main/src/main/scala/sbt/Keys.scala | 1 + main/src/main/scala/sbt/RemoteCache.scala | 1 + notes/2.0.0/cache-version.md | 24 ++++++++++ .../sbt-test/actions/cache-version/build.sbt | 20 ++++++++ .../src/sbt-test/actions/cache-version/test | 21 +++++++++ .../src/main/scala/sbt/util/ActionCache.scala | 39 ++++++++++++---- .../test/scala/sbt/util/ActionCacheTest.scala | 46 ++++++++++++++++++- 9 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 notes/2.0.0/cache-version.md create mode 100644 sbt-app/src/sbt-test/actions/cache-version/build.sbt create mode 100644 sbt-app/src/sbt-test/actions/cache-version/test diff --git a/main-command/src/main/scala/sbt/BasicKeys.scala b/main-command/src/main/scala/sbt/BasicKeys.scala index 5b4decfff..d2a6972ea 100644 --- a/main-command/src/main/scala/sbt/BasicKeys.scala +++ b/main-command/src/main/scala/sbt/BasicKeys.scala @@ -129,6 +129,12 @@ object BasicKeys { 10000 ) + val cacheVersion = AttributeKey[Long]( + "cacheVersion", + "A version number that invalidates all task caches when changed.", + 10000 + ) + // Unlike other BasicKeys, this is not used directly as a setting key, // and severLog / logLevel is used instead. private[sbt] val serverLogLevel = diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala index c75d19960..3bd06398a 100644 --- a/main-settings/src/main/scala/sbt/Def.scala +++ b/main-settings/src/main/scala/sbt/Def.scala @@ -283,6 +283,8 @@ object Def extends BuildSyntax with Init with InitializeImplicits: private[sbt] val cacheEventLog: CacheEventLog = CacheEventLog() private[sbt] val localDigestCacheByteSizeKey = SettingKey[Long](BasicKeys.localDigestCacheByteSize) + private[sbt] val cacheVersionKey = + SettingKey[Long](BasicKeys.cacheVersion) @cacheLevel(include = Array.empty) val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task { val state = stateKey.value @@ -298,13 +300,15 @@ object Def extends BuildSyntax with Init with InitializeImplicits: DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter) ) val cacheByteSize = localDigestCacheByteSizeKey.value + val cv = cacheVersionKey.value BuildWideCacheConfiguration( cacheStore, outputDirectory, fileConverter, state.log, cacheEventLog, - cacheByteSize + cacheByteSize, + cv, ) } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index a1658b56b..28a1db68e 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -457,6 +457,7 @@ object Keys { val allowZombieClassLoaders = settingKey[Boolean]("Allow a classloader that has previously been closed by `run` or `test` to continue loading classes.") val localCacheDirectory = settingKey[File]("Operating system specific cache directory.") val localDigestCacheByteSize = SettingKey[Long](BasicKeys.localDigestCacheByteSize).withRank(DSetting) + val cacheVersion = SettingKey[Long](BasicKeys.cacheVersion).withRank(BSetting) val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting) val exportPipelining = settingKey[Boolean]("Produce early output so downstream subprojects can do pipelining.").withRank(BSetting) diff --git a/main/src/main/scala/sbt/RemoteCache.scala b/main/src/main/scala/sbt/RemoteCache.scala index 91f5b616c..ab2f50de1 100644 --- a/main/src/main/scala/sbt/RemoteCache.scala +++ b/main/src/main/scala/sbt/RemoteCache.scala @@ -30,6 +30,7 @@ object RemoteCache: lazy val globalSettings: Seq[Def.Setting[?]] = Seq( localCacheDirectory :== defaultCacheLocation, localDigestCacheByteSize :== CacheImplicits.defaultLocalDigestCacheByteSize, + cacheVersion :== sys.props.get("sbt.cacheversion").flatMap(_.toLongOption).getOrElse(0L), rootOutputDirectory := { appConfiguration.value.baseDirectory .toPath() diff --git a/notes/2.0.0/cache-version.md b/notes/2.0.0/cache-version.md new file mode 100644 index 000000000..4dad2b536 --- /dev/null +++ b/notes/2.0.0/cache-version.md @@ -0,0 +1,24 @@ +## cacheVersion setting + +sbt 2.x caches task results by default. `cacheVersion` provides an escape hatch to invalidate all caches when needed. + +### Usage + +In `build.sbt`: + +```scala +Global / cacheVersion := 1L +``` + +Or via system property: + +``` +sbt -Dsbt.cacheversion=1 +``` + +### Details + +- Defaults to reading system property `sbt.cacheversion`, or else `0L` +- When `cacheVersion` is `0L`, caching behaves identically to previous versions +- Changing the value invalidates all task caches, forcing recomputation +- The value is incorporated into every cache key via `BuildWideCacheConfiguration` diff --git a/sbt-app/src/sbt-test/actions/cache-version/build.sbt b/sbt-app/src/sbt-test/actions/cache-version/build.sbt new file mode 100644 index 000000000..2a8cc00f3 --- /dev/null +++ b/sbt-app/src/sbt-test/actions/cache-version/build.sbt @@ -0,0 +1,20 @@ +scalaVersion := "3.7.4" + +lazy val checkCounter = inputKey[Unit]("assert counter file value") + +checkCounter := { + val expected = complete.DefaultParsers.spaceDelimited("").parsed.head.toInt + val f = baseDirectory.value / "target" / "counter.txt" + val actual = if (f.exists()) IO.read(f).trim.toInt else 0 + assert(actual == expected, s"Expected counter=$expected but got $actual") +} + +lazy val bumpCounter = taskKey[Int]("cached task that bumps a counter file on each execution") + +bumpCounter := { + val f = baseDirectory.value / "target" / "counter.txt" + val n = if (f.exists()) IO.read(f).trim.toInt else 0 + val next = n + 1 + IO.write(f, next.toString) + next +} diff --git a/sbt-app/src/sbt-test/actions/cache-version/test b/sbt-app/src/sbt-test/actions/cache-version/test new file mode 100644 index 000000000..3830e2fc0 --- /dev/null +++ b/sbt-app/src/sbt-test/actions/cache-version/test @@ -0,0 +1,21 @@ +# bumpCounter is a cached task that increments a file counter on each execution. +# When the cache is hit, the task body is NOT re-executed, so the counter stays the same. + +# First call: body runs, counter goes to 1 +> bumpCounter +> checkCounter 1 + +# Second call: cache hit, body does NOT run, counter stays at 1 +> bumpCounter +> checkCounter 1 + +# Change cacheVersion: invalidates all caches +> set Global / cacheVersion := 1L + +# Third call: cache miss (version changed), body runs, counter goes to 2 +> bumpCounter +> checkCounter 2 + +# Fourth call: cache hit again, counter stays at 2 +> bumpCounter +> checkCounter 2 diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index e08cfd1d1..04c62f8f9 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -41,9 +41,10 @@ object ActionCache: * all we need from the input is to generate some hash value. * - codeContentHash: This hash represents the Scala code of the task. * Even if the input tasks are the same, the code part needs to be tracked. - * - extraHash: Reserved for later, which we might use to invalidate the cache. + * - extraHash: Extra hash for cache invalidation (combined with config.cacheVersion). * - tags: Tags to track cache level. * - config: The configuration that's used to store where the cache backends are. + * config.cacheVersion is incorporated into the cache key to allow global invalidation. * - action: The actual action to be cached. */ def cache[I: HashWriter, O: JsonFormat]( @@ -62,7 +63,7 @@ object ActionCache: // This fixes https://github.com/sbt/sbt/issues/7662 // Use the same input digest as success, distinguished by exitCode cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val cachedFailure = CachedCompileFailure.fromException(e) val json = Converter.toJsonUnsafe(cachedFailure) val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json)) @@ -101,7 +102,7 @@ object ActionCache: result else cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) val newOutputs = Vector(valueFile) ++ outputs.toVector try @@ -160,7 +161,7 @@ object ActionCache: catch case _: Exception => None // Optimization: Check if we can read directly from symlinked value file - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath)) def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] = @@ -240,7 +241,7 @@ object ActionCache: ): Either[Throwable, ActionResult] = // val logger = config.logger CacheImplicits.setCacheSize(config.localDigestCacheByteSize) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val getRequest = GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath)) config.store.get(getRequest) @@ -248,10 +249,18 @@ object ActionCache: private inline def mkInput[I: HashWriter]( key: I, codeContentHash: Digest, - extraHash: Digest + extraHash: Digest, + cacheVersion: Long, ): (Digest, String) = + val effectiveExtraHash = + if cacheVersion != 0L then Digest.sha256Hash(extraHash, Digest.dummy(cacheVersion)) + else extraHash val input = - Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key))) + Digest.sha256Hash( + codeContentHash, + effectiveExtraHash, + Digest.dummy(Hasher.hashUnsafe[I](key)) + ) (input, s"$${OUT}/value/$input.json") def manifestFromFile(manifest: Path): Manifest = @@ -332,13 +341,14 @@ class BuildWideCacheConfiguration( val logger: Logger, val cacheEventLog: CacheEventLog, val localDigestCacheByteSize: Long, + val cacheVersion: Long, ): def this( store: ActionCacheStore, outputDirectory: Path, fileConverter: FileConverter, logger: Logger, - cacheEventLog: CacheEventLog + cacheEventLog: CacheEventLog, ) = this( store, @@ -346,9 +356,20 @@ class BuildWideCacheConfiguration( fileConverter, logger, cacheEventLog, - CacheImplicits.defaultLocalDigestCacheByteSize + CacheImplicits.defaultLocalDigestCacheByteSize, + 0L ) + def this( + store: ActionCacheStore, + outputDirectory: Path, + fileConverter: FileConverter, + logger: Logger, + cacheEventLog: CacheEventLog, + localDigestCacheByteSize: Long, + ) = + this(store, outputDirectory, fileConverter, logger, cacheEventLog, localDigestCacheByteSize, 0L) + override def toString(): String = s"BuildWideCacheConfiguration(store = $store, outputDirectory = $outputDirectory)" end BuildWideCacheConfiguration diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index 7d387b71b..708e9a646 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -259,6 +259,36 @@ object ActionCacheTest extends BasicTestSuite: assert(v2 == 42) assert(called == 2) + test("Changing cacheVersion invalidates the cache"): + withDiskCache(testCacheVersionInvalidation) + + def testCacheVersionInvalidation(cache: ActionCacheStore): Unit = + import sjsonnew.BasicJsonProtocol.* + var called = 0 + val action: ((Int, Int)) => InternalActionResult[Int] = { (a, b) => + called += 1 + InternalActionResult(a + b, Nil) + } + IO.withTemporaryDirectory: tempDir => + val config0 = getCacheConfig(cache, tempDir) + // First call: computes the result + val v1 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action) + assert(v1 == 2) + assert(called == 1) + // Second call with same config: hits cache + val v2 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action) + assert(v2 == 2) + assert(called == 1) + // Third call with different cacheVersion: cache miss, recomputes + val config1 = getCacheConfig(cache, tempDir, cacheVersion = 1L) + val v3 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action) + assert(v3 == 2) + assert(called == 2) + // Fourth call with same cacheVersion=1: hits cache again + val v4 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action) + assert(v4 == 2) + assert(called == 2) + def withInMemoryCache(f: InMemoryActionCacheStore => Unit): Unit = val cache = InMemoryActionCacheStore() f(cache) @@ -273,12 +303,24 @@ object ActionCacheTest extends BasicTestSuite: keepDirectory = false ) - def getCacheConfig(cache: ActionCacheStore, outputDir: File): BuildWideCacheConfiguration = + def getCacheConfig( + cache: ActionCacheStore, + outputDir: File, + cacheVersion: Long = 0L, + ): BuildWideCacheConfiguration = val logger = new Logger: override def trace(t: => Throwable): Unit = () override def success(message: => String): Unit = () override def log(level: Level.Value, message: => String): Unit = () - BuildWideCacheConfiguration(cache, outputDir.toPath(), fileConverter, logger, CacheEventLog()) + BuildWideCacheConfiguration( + cache, + outputDir.toPath(), + fileConverter, + logger, + CacheEventLog(), + CacheImplicits.defaultLocalDigestCacheByteSize, + cacheVersion, + ) def fileConverter = new FileConverter: override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id)