[2.x] feat: Add cacheVersion setting for global cache invalidation (#8993)

* [2.x] feat: Add cacheVersion setting for global cache invalidation

**Problem**
There was no escape hatch to invalidate all task caches when needed.

**Solution**
Add `Global / cacheVersion` setting that incorporates into the cache key
hash. Changing it invalidates all caches. Defaults to reading system
property `sbt.cacheversion`, or else 0L. When 0L, the hash is identical
to the previous behavior (backward compatible).

Fixes #8992

* [2.x] refactor: Simplify BuildWideCacheConfiguration and add cacheVersion test

- Replace auxiliary constructors with default parameter values
- Add unit test verifying cacheVersion invalidates the cache

* [2.x] fix: Restore auxiliary constructors for binary compatibility

* [2.x] test: Improve cacheVersion scripted test and add release note

- Scripted test now verifies cache invalidation via a counter
  that increments only when the task body actually executes
- Add release note documenting the cacheVersion setting

main-command/src/main/scala/sbt/BasicKeys.scala

@@ -129,6 +129,12 @@ object BasicKeys {
     10000
   )
 
+  val cacheVersion = AttributeKey[Long](
+    "cacheVersion",
+    "A version number that invalidates all task caches when changed.",
+    10000
+  )
+
   // Unlike other BasicKeys, this is not used directly as a setting key,
   // and severLog / logLevel is used instead.
   private[sbt] val serverLogLevel =

main-settings/src/main/scala/sbt/Def.scala

@@ -283,6 +283,8 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
   private[sbt] val cacheEventLog: CacheEventLog = CacheEventLog()
   private[sbt] val localDigestCacheByteSizeKey =
     SettingKey[Long](BasicKeys.localDigestCacheByteSize)
+  private[sbt] val cacheVersionKey =
+    SettingKey[Long](BasicKeys.cacheVersion)
   @cacheLevel(include = Array.empty)
   val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task {
     val state = stateKey.value
@@ -298,13 +300,15 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
         DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter)
       )
     val cacheByteSize = localDigestCacheByteSizeKey.value
+    val cv = cacheVersionKey.value
     BuildWideCacheConfiguration(
       cacheStore,
       outputDirectory,
       fileConverter,
       state.log,
       cacheEventLog,
-      cacheByteSize
+      cacheByteSize,
+      cv,
     )
   }
 

main/src/main/scala/sbt/Keys.scala

@@ -457,6 +457,7 @@ object Keys {
   val allowZombieClassLoaders = settingKey[Boolean]("Allow a classloader that has previously been closed by `run` or `test` to continue loading classes.")
   val localCacheDirectory = settingKey[File]("Operating system specific cache directory.")
   val localDigestCacheByteSize = SettingKey[Long](BasicKeys.localDigestCacheByteSize).withRank(DSetting)
+  val cacheVersion = SettingKey[Long](BasicKeys.cacheVersion).withRank(BSetting)
   val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting)
   val exportPipelining = settingKey[Boolean]("Produce early output so downstream subprojects can do pipelining.").withRank(BSetting)
 

main/src/main/scala/sbt/RemoteCache.scala

@@ -30,6 +30,7 @@ object RemoteCache:
   lazy val globalSettings: Seq[Def.Setting[?]] = Seq(
     localCacheDirectory :== defaultCacheLocation,
     localDigestCacheByteSize :== CacheImplicits.defaultLocalDigestCacheByteSize,
+    cacheVersion :== sys.props.get("sbt.cacheversion").flatMap(_.toLongOption).getOrElse(0L),
     rootOutputDirectory := {
       appConfiguration.value.baseDirectory
         .toPath()

notes/2.0.0/cache-version.md

@@ -0,0 +1,24 @@
+## cacheVersion setting
+
+sbt 2.x caches task results by default. `cacheVersion` provides an escape hatch to invalidate all caches when needed.
+
+### Usage
+
+In `build.sbt`:
+
+```scala
+Global / cacheVersion := 1L
+```
+
+Or via system property:
+
+```
+sbt -Dsbt.cacheversion=1
+```
+
+### Details
+
+- Defaults to reading system property `sbt.cacheversion`, or else `0L`
+- When `cacheVersion` is `0L`, caching behaves identically to previous versions
+- Changing the value invalidates all task caches, forcing recomputation
+- The value is incorporated into every cache key via `BuildWideCacheConfiguration`

sbt-app/src/sbt-test/actions/cache-version/build.sbt

@@ -0,0 +1,20 @@
+scalaVersion := "3.7.4"
+
+lazy val checkCounter = inputKey[Unit]("assert counter file value")
+
+checkCounter := {
+  val expected = complete.DefaultParsers.spaceDelimited("<arg>").parsed.head.toInt
+  val f = baseDirectory.value / "target" / "counter.txt"
+  val actual = if (f.exists()) IO.read(f).trim.toInt else 0
+  assert(actual == expected, s"Expected counter=$expected but got $actual")
+}
+
+lazy val bumpCounter = taskKey[Int]("cached task that bumps a counter file on each execution")
+
+bumpCounter := {
+  val f = baseDirectory.value / "target" / "counter.txt"
+  val n = if (f.exists()) IO.read(f).trim.toInt else 0
+  val next = n + 1
+  IO.write(f, next.toString)
+  next
+}

sbt-app/src/sbt-test/actions/cache-version/test

@@ -0,0 +1,21 @@
+# bumpCounter is a cached task that increments a file counter on each execution.
+# When the cache is hit, the task body is NOT re-executed, so the counter stays the same.
+
+# First call: body runs, counter goes to 1
+> bumpCounter
+> checkCounter 1
+
+# Second call: cache hit, body does NOT run, counter stays at 1
+> bumpCounter
+> checkCounter 1
+
+# Change cacheVersion: invalidates all caches
+> set Global / cacheVersion := 1L
+
+# Third call: cache miss (version changed), body runs, counter goes to 2
+> bumpCounter
+> checkCounter 2
+
+# Fourth call: cache hit again, counter stays at 2
+> bumpCounter
+> checkCounter 2

util-cache/src/main/scala/sbt/util/ActionCache.scala

@@ -41,9 +41,10 @@ object ActionCache:
    *   all we need from the input is to generate some hash value.
    * - codeContentHash: This hash represents the Scala code of the task.
    *   Even if the input tasks are the same, the code part needs to be tracked.
-   * - extraHash: Reserved for later, which we might use to invalidate the cache.
+   * - extraHash: Extra hash for cache invalidation (combined with config.cacheVersion).
    * - tags: Tags to track cache level.
    * - config: The configuration that's used to store where the cache backends are.
+   *   config.cacheVersion is incorporated into the cache key to allow global invalidation.
    * - action: The actual action to be cached.
    */
   def cache[I: HashWriter, O: JsonFormat](
@@ -62,7 +63,7 @@ object ActionCache:
       // This fixes https://github.com/sbt/sbt/issues/7662
       // Use the same input digest as success, distinguished by exitCode
       cacheEventLog.append(ActionCacheEvent.OnsiteTask)
-      val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
+      val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
       val cachedFailure = CachedCompileFailure.fromException(e)
       val json = Converter.toJsonUnsafe(cachedFailure)
       val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json))
@@ -101,7 +102,7 @@ object ActionCache:
         result
       else
         cacheEventLog.append(ActionCacheEvent.OnsiteTask)
-        val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
+        val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
         val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json))
         val newOutputs = Vector(valueFile) ++ outputs.toVector
         try
@@ -160,7 +161,7 @@ object ActionCache:
       catch case _: Exception => None
 
     // Optimization: Check if we can read directly from symlinked value file
-    val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
+    val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
     val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath))
 
     def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] =
@@ -240,7 +241,7 @@ object ActionCache:
   ): Either[Throwable, ActionResult] =
     // val logger = config.logger
     CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
-    val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
+    val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
     val getRequest =
       GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
     config.store.get(getRequest)
@@ -248,10 +249,18 @@ object ActionCache:
   private inline def mkInput[I: HashWriter](
       key: I,
       codeContentHash: Digest,
-      extraHash: Digest
+      extraHash: Digest,
+      cacheVersion: Long,
   ): (Digest, String) =
+    val effectiveExtraHash =
+      if cacheVersion != 0L then Digest.sha256Hash(extraHash, Digest.dummy(cacheVersion))
+      else extraHash
     val input =
-      Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key)))
+      Digest.sha256Hash(
+        codeContentHash,
+        effectiveExtraHash,
+        Digest.dummy(Hasher.hashUnsafe[I](key))
+      )
     (input, s"$${OUT}/value/$input.json")
 
   def manifestFromFile(manifest: Path): Manifest =
@@ -332,13 +341,14 @@ class BuildWideCacheConfiguration(
     val logger: Logger,
     val cacheEventLog: CacheEventLog,
     val localDigestCacheByteSize: Long,
+    val cacheVersion: Long,
 ):
   def this(
       store: ActionCacheStore,
       outputDirectory: Path,
       fileConverter: FileConverter,
       logger: Logger,
-      cacheEventLog: CacheEventLog
+      cacheEventLog: CacheEventLog,
   ) =
     this(
       store,
@@ -346,9 +356,20 @@ class BuildWideCacheConfiguration(
       fileConverter,
       logger,
       cacheEventLog,
-      CacheImplicits.defaultLocalDigestCacheByteSize
+      CacheImplicits.defaultLocalDigestCacheByteSize,
+      0L
     )
 
+  def this(
+      store: ActionCacheStore,
+      outputDirectory: Path,
+      fileConverter: FileConverter,
+      logger: Logger,
+      cacheEventLog: CacheEventLog,
+      localDigestCacheByteSize: Long,
+  ) =
+    this(store, outputDirectory, fileConverter, logger, cacheEventLog, localDigestCacheByteSize, 0L)
+
   override def toString(): String =
     s"BuildWideCacheConfiguration(store = $store, outputDirectory = $outputDirectory)"
 end BuildWideCacheConfiguration

util-cache/src/test/scala/sbt/util/ActionCacheTest.scala

@@ -259,6 +259,36 @@ object ActionCacheTest extends BasicTestSuite:
     assert(v2 == 42)
     assert(called == 2)
 
+  test("Changing cacheVersion invalidates the cache"):
+    withDiskCache(testCacheVersionInvalidation)
+
+  def testCacheVersionInvalidation(cache: ActionCacheStore): Unit =
+    import sjsonnew.BasicJsonProtocol.*
+    var called = 0
+    val action: ((Int, Int)) => InternalActionResult[Int] = { (a, b) =>
+      called += 1
+      InternalActionResult(a + b, Nil)
+    }
+    IO.withTemporaryDirectory: tempDir =>
+      val config0 = getCacheConfig(cache, tempDir)
+      // First call: computes the result
+      val v1 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action)
+      assert(v1 == 2)
+      assert(called == 1)
+      // Second call with same config: hits cache
+      val v2 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action)
+      assert(v2 == 2)
+      assert(called == 1)
+      // Third call with different cacheVersion: cache miss, recomputes
+      val config1 = getCacheConfig(cache, tempDir, cacheVersion = 1L)
+      val v3 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action)
+      assert(v3 == 2)
+      assert(called == 2)
+      // Fourth call with same cacheVersion=1: hits cache again
+      val v4 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action)
+      assert(v4 == 2)
+      assert(called == 2)
+
   def withInMemoryCache(f: InMemoryActionCacheStore => Unit): Unit =
     val cache = InMemoryActionCacheStore()
     f(cache)
@@ -273,12 +303,24 @@ object ActionCacheTest extends BasicTestSuite:
       keepDirectory = false
     )
 
-  def getCacheConfig(cache: ActionCacheStore, outputDir: File): BuildWideCacheConfiguration =
+  def getCacheConfig(
+      cache: ActionCacheStore,
+      outputDir: File,
+      cacheVersion: Long = 0L,
+  ): BuildWideCacheConfiguration =
     val logger = new Logger:
       override def trace(t: => Throwable): Unit = ()
       override def success(message: => String): Unit = ()
       override def log(level: Level.Value, message: => String): Unit = ()
-    BuildWideCacheConfiguration(cache, outputDir.toPath(), fileConverter, logger, CacheEventLog())
+    BuildWideCacheConfiguration(
+      cache,
+      outputDir.toPath(),
+      fileConverter,
+      logger,
+      CacheEventLog(),
+      CacheImplicits.defaultLocalDigestCacheByteSize,
+      cacheVersion,
+    )
 
   def fileConverter = new FileConverter:
     override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id)