[2.x] fix: Fixes cache invalidation on version change (#9471)

**Problem**
packageBin includes version into the file name, which ends up
invalidating the cache.

**Solution**
Define packageInternal, which does not include version in the file name, and used during Compile or Test/compile. Note that publishing and Runtime classpath would continue to use packageBin.

lm-core/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala

@@ -112,6 +112,25 @@ private[librarymanagement] abstract class ArtifactFunctions {
     base + "-" + module.revision + classifierStr + "." + artifact.extension
   }
 
+  /**
+   * Like `artifactName`, but omits the module's version.
+   */
+  def internalArtifactName(
+      scalaVersion: ScalaVersion,
+      module: ModuleID,
+      artifact: Artifact
+  ): String =
+    import artifact.*
+    val classifierStr = classifier match
+      case None    => ""
+      case Some(c) => s"-${c}"
+    val cross = CrossVersion(module.crossVersion, scalaVersion.full, scalaVersion.binary)
+    val withPlatform = module.crossVersion match
+      case _: Disabled => artifact.name
+      case _           => CrossVersion.addPlatformSuffix(artifact.name, module.platformOpt, None)
+    val base = CrossVersion.applyCross(withPlatform, cross)
+    s"${base}${classifierStr}.${artifact.extension}"
+
   val classifierTypeMap = Map(SourceClassifier -> SourceType, DocClassifier -> DocType)
   @deprecated("Configuration should not be decided from the classifier.", "1.0")
   def classifierConf(classifier: String): Configuration =

main/src/main/scala/sbt/Defaults.scala

@@ -1812,6 +1812,8 @@ object Defaults extends BuildCommon with DefExtra {
       packageTaskSettings(packageBin, packageBinMappings) ++
       packageTaskSettings(packageSrc, packageSrcMappings) ++
       packageTaskSettings(packageDoc, packageDocMappings) ++
+      packageTaskSettings(packageInternal, packageBin / mappings) ++
+      inTask(packageInternal)(Seq(artifactName :== Artifact.internalArtifactName)) ++
       Seq(Keys.`package` := packageBin.value)
 
   def packageBinMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] =
@@ -2884,6 +2886,15 @@ object Classpaths {
       exportedProductsNoTracking := ClasspathImpl
         .trackedExportedProducts(TrackLevel.NoTracking)
         .value,
+      exportedProductsVersioned := Def.uncached(
+        ClasspathImpl.trackedExportedProductsVersioned(TrackLevel.TrackAlways).value
+      ),
+      exportedProductsVersionedIfMissing := ClasspathImpl
+        .trackedExportedProductsVersioned(TrackLevel.TrackIfMissing)
+        .value,
+      exportedProductsVersionedNoTracking := ClasspathImpl
+        .trackedExportedProductsVersioned(TrackLevel.NoTracking)
+        .value,
       exportedProductJars := ClasspathImpl.trackedExportedJarProducts(TrackLevel.TrackAlways).value,
       exportedProductJarsIfMissing := ClasspathImpl
         .trackedExportedJarProducts(TrackLevel.TrackIfMissing)
@@ -2891,6 +2902,15 @@ object Classpaths {
       exportedProductJarsNoTracking := ClasspathImpl
         .trackedExportedJarProducts(TrackLevel.NoTracking)
         .value,
+      exportedProductJarsVersioned := ClasspathImpl
+        .trackedExportedJarProductsVersioned(TrackLevel.TrackAlways)
+        .value,
+      exportedProductJarsVersionedIfMissing := ClasspathImpl
+        .trackedExportedJarProductsVersioned(TrackLevel.TrackIfMissing)
+        .value,
+      exportedProductJarsVersionedNoTracking := ClasspathImpl
+        .trackedExportedJarProductsVersioned(TrackLevel.NoTracking)
+        .value,
       internalDependencyAsJars := Def.uncached(internalDependencyJarsTask.value),
       dependencyClasspathAsJars := Def.uncached(
         concat(

main/src/main/scala/sbt/Keys.scala

@@ -324,6 +324,7 @@ object Keys {
 
   // package keys
   val packageBin = taskKey[HashedVirtualFileRef]("Produces a main artifact, such as a binary jar.").withRank(ATask)
+  val packageInternal = taskKey[HashedVirtualFileRef]("Produces a binary JAR for internal use (inter-project classpaths, BSP).").withRank(DTask)
   val `package` = taskKey[HashedVirtualFileRef]("Produces the main artifact, such as a binary jar.  This is typically an alias for the task that actually does the packaging.").withRank(APlusTask)
   val packageDoc = taskKey[HashedVirtualFileRef]("Produces a documentation artifact, such as a jar containing API documentation.").withRank(AMinusTask)
   val packageSrc = taskKey[HashedVirtualFileRef]("Produces a source artifact, such as a jar containing sources and resources.").withRank(AMinusTask)
@@ -442,6 +443,9 @@ object Keys {
   val exportedProducts = taskKey[Classpath]("Build products that go on the exported classpath.").withRank(CTask)
   val exportedProductsIfMissing = taskKey[Classpath]("Build products that go on the exported classpath if missing.").withRank(CTask)
   val exportedProductsNoTracking = taskKey[Classpath]("Just the exported classpath without triggering the compilation.").withRank(CTask)
+  val exportedProductsVersioned = taskKey[Classpath]("Build products that go on the exported classpath, packaged with the versioned artifact (used by the Runtime configuration).").withRank(CTask)
+  val exportedProductsVersionedIfMissing = taskKey[Classpath]("Build products that go on the exported classpath, packaged with the versioned artifact, if missing.").withRank(CTask)
+  val exportedProductsVersionedNoTracking = taskKey[Classpath]("Just the exported classpath, packaged with the versioned artifact, without triggering the compilation.").withRank(CTask)
   val unmanagedClasspath = taskKey[Classpath]("Classpath entries (deep) that are manually managed.").withRank(BPlusTask)
   val unmanagedJars = taskKey[Classpath]("Classpath entries for the current project (shallow) that are manually managed.").withRank(BPlusTask)
   val managedClasspath = taskKey[Classpath]("The classpath consisting of external, managed library dependencies.").withRank(BMinusTask)
@@ -458,6 +462,9 @@ object Keys {
   val exportedProductJars = taskKey[Classpath]("Build products that go on the exported classpath as JARs.")
   val exportedProductJarsIfMissing = taskKey[Classpath]("Build products that go on the exported classpath as JARs if missing.")
   val exportedProductJarsNoTracking = taskKey[Classpath]("Just the exported classpath as JARs without triggering the compilation.")
+  val exportedProductJarsVersioned = taskKey[Classpath]("Build products that go on the exported classpath as JARs, packaged with the versioned artifact (used by the Runtime configuration).")
+  val exportedProductJarsVersionedIfMissing = taskKey[Classpath]("Build products that go on the exported classpath as JARs, packaged with the versioned artifact, if missing.")
+  val exportedProductJarsVersionedNoTracking = taskKey[Classpath]("Just the exported classpath as JARs, packaged with the versioned artifact, without triggering the compilation.")
   val exportedPickles = taskKey[Classpath]("Build products that go on the exported compilation classpath as JARs. Note this is promise-blocked.").withRank(DTask)
   val pickleProducts = taskKey[Seq[VirtualFile]]("Pickle JARs").withRank(DTask)
   val internalDependencyAsJars = taskKey[Classpath]("The internal (inter-project) classpath as JARs.")

main/src/main/scala/sbt/internal/ClasspathImpl.scala

@@ -25,7 +25,7 @@ import sbt.librarymanagement.{
   TrackLevel,
   UpdateReport
 }
-import sbt.librarymanagement.Configurations.names
+import sbt.librarymanagement.Configurations.{ names, Runtime }
 import sbt.SlashSyntax0.*
 import sbt.std.TaskExtra.*
 import sbt.util.*
@@ -56,13 +56,24 @@ private[sbt] object ClasspathImpl {
       else exportedProducts.value
     }
 
+  // Compile/Test: version-free, so a dependency's version bump doesn't bust the cache.
   def trackedExportedProducts(track: TrackLevel): Initialize[Task[Classpath]] =
+    trackedExportedProductsFor(packageInternal, track)
+
+  // Runtime: versioned, matching what `run` and sbt-native-packager-style tooling expect.
+  def trackedExportedProductsVersioned(track: TrackLevel): Initialize[Task[Classpath]] =
+    trackedExportedProductsFor(packageBin, track)
+
+  private def trackedExportedProductsFor(
+      key: TaskKey[HashedVirtualFileRef],
+      track: TrackLevel
+  ): Initialize[Task[Classpath]] =
     Def.task {
       val _ = (packageBin / dynamicDependency).value
       val art = (packageBin / artifact).value
       val module = projectID.value
       val config = configuration.value
-      for (f, analysis) <- trackedExportedProductsImplTask(track).value
+      for (f, analysis) <- trackedExportedProductsImplTask(key, track).value
       yield APIMappings
         .store(Classpaths.analyzed(f, analysis), apiURL.value)
         .put(Keys.artifactStr, RemoteCache.artifactToStr(art))
@@ -71,12 +82,21 @@ private[sbt] object ClasspathImpl {
     }
 
   def trackedExportedJarProducts(track: TrackLevel): Initialize[Task[Classpath]] =
+    trackedExportedJarProductsFor(packageInternal, track)
+
+  def trackedExportedJarProductsVersioned(track: TrackLevel): Initialize[Task[Classpath]] =
+    trackedExportedJarProductsFor(packageBin, track)
+
+  private def trackedExportedJarProductsFor(
+      key: TaskKey[HashedVirtualFileRef],
+      track: TrackLevel
+  ): Initialize[Task[Classpath]] =
     Def.task {
       val _ = (packageBin / dynamicDependency).value
       val art = (packageBin / artifact).value
       val module = projectID.value
       val config = configuration.value
-      for (f, analysis) <- trackedJarProductsImplTask(track).value
+      for (f, analysis) <- jarProductsForTask(key, track).value
       yield APIMappings
         .store(Classpaths.analyzed(f, analysis), apiURL.value)
         .put(Keys.artifactStr, RemoteCache.artifactToStr(art))
@@ -85,13 +105,14 @@ private[sbt] object ClasspathImpl {
     }
 
   private def trackedExportedProductsImplTask(
+      key: TaskKey[HashedVirtualFileRef],
       track: TrackLevel
   ): Initialize[Task[Seq[(HashedVirtualFileRef, VirtualFile)]]] =
     Def.taskIf {
       if {
         val _ = (packageBin / dynamicDependency).value
         exportJars.value
-      } then trackedJarProductsImplTask(track).value
+      } then jarProductsForTask(key, track).value
       else trackedNonJarProductsImplTask(track).value
     }
 
@@ -126,13 +147,14 @@ private[sbt] object ClasspathImpl {
           }
       }
 
-  private def trackedJarProductsImplTask(
+  private def jarProductsForTask(
+      key: TaskKey[HashedVirtualFileRef],
       track: TrackLevel
   ): Initialize[Task[Seq[(HashedVirtualFileRef, VirtualFile)]]] =
     (Def
       .task {
         val converter = fileConverter.value
-        val vf = (packageBin / artifactPath).value
+        val vf = (key / artifactPath).value
         val jar = converter.toPath(vf)
         (TrackLevel.intersection(track, exportToInternal.value), vf, jar)
       })
@@ -141,13 +163,13 @@ private[sbt] object ClasspathImpl {
           Def.task {
             val converter = fileConverter.value
             val analysisFile = converter.toVirtualFile(compileAnalysisFile.value.toPath)
-            Seq((packageBin.value, analysisFile))
+            Seq((key.value, analysisFile))
           }
         case (TrackLevel.TrackIfMissing, _, jar) if !jar.toFile().exists =>
           Def.task {
             val converter = fileConverter.value
             val analysisFile = converter.toVirtualFile(compileAnalysisFile.value.toPath)
-            Seq((packageBin.value, analysisFile))
+            Seq((key.value, analysisFile))
           }
         case (_, vf, _) =>
           Def.task {
@@ -164,6 +186,9 @@ private[sbt] object ClasspathImpl {
           (exportedProductsNoTracking / transitiveClasspathDependency).value,
           (exportedProductsIfMissing / transitiveClasspathDependency).value,
           (exportedProducts / transitiveClasspathDependency).value,
+          (exportedProductsVersionedNoTracking / transitiveClasspathDependency).value,
+          (exportedProductsVersionedIfMissing / transitiveClasspathDependency).value,
+          (exportedProductsVersioned / transitiveClasspathDependency).value,
           (exportedProductJarsNoTracking / transitiveClasspathDependency).value,
           (exportedProductJarsIfMissing / transitiveClasspathDependency).value,
           (exportedProductJars / transitiveClasspathDependency).value
@@ -194,11 +219,18 @@ private[sbt] object ClasspathImpl {
       log: Logger
   ): Initialize[Task[Classpath]] =
     Def.value[Task[Classpath]] {
-      interDependencies(projectRef, deps, conf, self, data, track, false, log)(
+      if self == Runtime then
-        exportedProductsNoTracking,
+        interDependencies(projectRef, deps, conf, self, data, track, false, log)(
-        exportedProductsIfMissing,
+          exportedProductsVersionedNoTracking,
-        exportedProducts
+          exportedProductsVersionedIfMissing,
-      )
+          exportedProductsVersioned
+        )
+      else
+        interDependencies(projectRef, deps, conf, self, data, track, false, log)(
+          exportedProductsNoTracking,
+          exportedProductsIfMissing,
+          exportedProducts
+        )
     }
 
   def internalDependencyPicklePathTask: Initialize[Task[Classpath]] = {
@@ -258,20 +290,36 @@ private[sbt] object ClasspathImpl {
       log: Logger
   ): Initialize[Task[Classpath]] =
     Def.value[Task[Classpath]] {
-      interDependencies[Attributed[HashedVirtualFileRef]](
+      if self == Runtime then
-        projectRef,
+        interDependencies[Attributed[HashedVirtualFileRef]](
-        deps,
+          projectRef,
-        conf,
+          deps,
-        self,
+          conf,
-        data,
+          self,
-        track,
+          data,
-        false,
+          track,
-        log,
+          false,
-      )(
+          log,
-        exportedProductJarsNoTracking,
+        )(
-        exportedProductJarsIfMissing,
+          exportedProductJarsVersionedNoTracking,
-        exportedProductJars
+          exportedProductJarsVersionedIfMissing,
-      ): Task[Classpath]
+          exportedProductJarsVersioned
+        ): Task[Classpath]
+      else
+        interDependencies[Attributed[HashedVirtualFileRef]](
+          projectRef,
+          deps,
+          conf,
+          self,
+          data,
+          track,
+          false,
+          log,
+        )(
+          exportedProductJarsNoTracking,
+          exportedProductJarsIfMissing,
+          exportedProductJars
+        ): Task[Classpath]
     }
 
   def unmanagedDependenciesTask: Initialize[Task[Classpath]] =

sbt-app/src/sbt-test/cache/runtime-classpath-versioned/app/App.scala

@@ -0,0 +1,5 @@
+package example
+
+@main
+def main(args: String*): Unit =
+  println(Foo.greeting)

sbt-app/src/sbt-test/cache/runtime-classpath-versioned/build.sbt

@@ -0,0 +1,50 @@
+Global / localCacheDirectory := baseDirectory.value / "diskcache"
+
+scalaVersion := "3.8.4"
+
+// packageInternal is used for the Compile/Test-time internal classpath (what app compiles
+// against), so a dependency's version bump alone doesn't bust downstream compile caches.
+// packageBin (versioned) must still back the Runtime classpath, since that's what `run`
+// uses and what tools like sbt-native-packager read to assemble a runnable image.
+lazy val checkClasspaths = taskKey[Unit]("Assert Compile classpath uses packageInternal, Runtime classpath uses packageBin")
+
+lazy val foo = project
+
+lazy val app = project
+  .dependsOn(foo)
+  .settings(
+    checkClasspaths := {
+      val converter = fileConverter.value
+      val binPath = converter.toPath((foo / Compile / packageBin).value)
+      val internalPath = converter.toPath((foo / Compile / packageInternal).value)
+      val s = streams.value
+
+      s.log.info(s"packageBin      = $binPath")
+      s.log.info(s"packageInternal = $internalPath")
+      assert(binPath != internalPath, "packageBin and packageInternal unexpectedly produced the same path")
+
+      def check(name: String, cp: Seq[HashedVirtualFileRef], expectVersioned: Boolean): Unit =
+        val paths = cp.map(converter.toPath)
+        s.log.info(s"$name = $paths")
+        val (expected, unexpected) = if expectVersioned then (binPath, internalPath) else (internalPath, binPath)
+        val expectedDesc = if expectVersioned then "packageBin's" else "packageInternal's"
+        val unexpectedDesc = if expectVersioned then "packageInternal's" else "packageBin's"
+        assert(paths.contains(expected), s"$name should contain $expectedDesc jar ($expected), got: $paths")
+        assert(!paths.contains(unexpected), s"$name should NOT contain $unexpectedDesc jar ($unexpected), got: $paths")
+
+      check("Compile/dependencyClasspath", (Compile / dependencyClasspath).value.map(_.data), expectVersioned = false)
+      check("Runtime/dependencyClasspath", (Runtime / dependencyClasspath).value.map(_.data), expectVersioned = true)
+
+      check("Compile/internalDependencyAsJars", (Compile / internalDependencyAsJars).value.map(_.data), expectVersioned = false)
+      check("Runtime/internalDependencyAsJars", (Runtime / internalDependencyAsJars).value.map(_.data), expectVersioned = true)
+
+      check("Compile/dependencyClasspathAsJars", (Compile / dependencyClasspathAsJars).value.map(_.data), expectVersioned = false)
+      check("Runtime/dependencyClasspathAsJars", (Runtime / dependencyClasspathAsJars).value.map(_.data), expectVersioned = true)
+
+      check("Compile/fullClasspathAsJars", (Compile / fullClasspathAsJars).value.map(_.data), expectVersioned = false)
+      check("Runtime/fullClasspathAsJars", (Runtime / fullClasspathAsJars).value.map(_.data), expectVersioned = true)
+    }
+  )
+
+lazy val root = (project in file("."))
+  .aggregate(foo, app)

sbt-app/src/sbt-test/cache/runtime-classpath-versioned/foo/Foo.scala

@@ -0,0 +1,5 @@
+package example
+
+object Foo:
+  def greeting: String = "hello"
+end Foo

sbt-app/src/sbt-test/cache/runtime-classpath-versioned/test

@@ -0,0 +1,8 @@
+> app/checkClasspaths
+> app/run
+
+# bumping foo's version must not change which task backs each classpath:
+# Compile still packageInternal (version-free), Runtime still packageBin (versioned)
+> set foo / version := "0.2.0"
+> app/checkClasspaths
+> app/run

sbt-app/src/sbt-test/cache/version-bump-compile/app/App.scala

@@ -0,0 +1,5 @@
+package example
+
+@main
+def main(args: String*): Unit =
+  println(Foo.greeting)

sbt-app/src/sbt-test/cache/version-bump-compile/build.sbt

@@ -0,0 +1,27 @@
+import sbt.internal.util.CacheEventSummary
+import complete.DefaultParsers.*
+
+Global / localCacheDirectory := baseDirectory.value / "diskcache"
+
+scalaVersion := "3.8.4"
+
+lazy val checkMiss = inputKey[Unit]("Assert the exact onsite/miss count of the previous run")
+
+lazy val foo = project
+
+lazy val app = project.dependsOn(foo)
+
+lazy val root = (project in file("."))
+  .aggregate(foo, app)
+  .settings(
+    checkMiss := {
+      val expected: Int = (Space ~> NatBasic).parsed
+      val s = streams.value
+      val config = Def.cacheConfiguration.value
+      val prev = config.cacheEventLog.previous match
+        case d: CacheEventSummary.Data => d
+        case CacheEventSummary.Empty   => sys.error("empty event log")
+      s.log.info(s"missCount = ${prev.missCount}")
+      assert(prev.missCount == expected, s"prev.missCount = ${prev.missCount} (expected $expected)")
+    }
+  )

sbt-app/src/sbt-test/cache/version-bump-compile/foo/Foo.scala

@@ -0,0 +1,5 @@
+package example
+
+object Foo:
+  def greeting: String = "hello"
+end Foo

sbt-app/src/sbt-test/cache/version-bump-compile/test

@@ -0,0 +1,14 @@
+# warm the disk cache
+> app/compile
+
+# after a clean, recompiling app (and foo) should fully restore from the disk cache
+> clean
+> app/compile
+> checkMiss 0
+
+# bumping foo's version alone (e.g. what sbt-dynver would do on a new commit) must not
+# invalidate app's compile cache entry, since foo's jar content hasn't changed
+> set foo / version := "0.2.0"
+> clean
+> app/compile
+> checkMiss 0

sbt-app/src/sbt-test/plugins/sbt-native-packager/build.sbt

@@ -3,6 +3,7 @@ ThisBuild / scalaVersion := "3.8.4"
 name := "hello"
 enablePlugins(JavaAppPackaging)
 
+@transient
 lazy val check = taskKey[Unit]("")
 
 check := {

sbt-app/src/sbt-test/project-matrix/jvm-sandwich/build.sbt

@@ -42,8 +42,8 @@ lazy val bazApp = (projectMatrix in file("baz-app"))
     name := "baz app",
     check := {
       val cp = (Compile / fullClasspath).value.map(_.data.id)
-      assert(cp.exists(_.endsWith("baz-core_2.13-0.1.0-SNAPSHOT.jar")), cp)
+      assert(cp.exists(_.endsWith("baz-core_2.13.jar")), cp)
-      assert(!cp.exists(_.endsWith("baz-core_3.0.0-M1-0.1.0-SNAPSHOT.jar")), cp)
+      assert(!cp.exists(_.endsWith("baz-core_3.0.0-M1.jar")), cp)
       assert(projectMatrixBaseDirectory.value == (ThisBuild / baseDirectory).value / "baz-app",
         s"projectMatrixBaseDirectory is ${projectMatrixBaseDirectory.value}")
     },