Use fileConverter in cacheStore

Otherwise the store cannot sync files that are not in the out folder.

main/src/main/scala/sbt/Defaults.scala

@@ -88,8 +88,6 @@ import sbt.util.CacheImplicits.given
 import sbt.util.InterfaceUtil.{ t2, toJavaFunction => f1 }
 import sbt.util._
 import sjsonnew._
-import xsbti.compile.TastyFiles
-import xsbti.{ FileConverter, Position }
 
 import scala.annotation.nowarn
 import scala.collection.immutable.ListMap
@@ -108,7 +106,16 @@ import sbt.internal.inc.{
   ScalaInstance
 }
 import sbt.internal.io.Retry
-import xsbti.{ CompileFailed, CrossValue, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
+import xsbti.{
+  AppConfiguration,
+  CompileFailed,
+  CrossValue,
+  FileConverter,
+  HashedVirtualFileRef,
+  Position,
+  VirtualFile,
+  VirtualFileRef
+}
 import xsbti.compile.{
   AnalysisContents,
   AnalysisStore,
@@ -129,6 +136,7 @@ import xsbti.compile.{
   PerClasspathEntryLookup,
   PreviousResult,
   Setup,
+  TastyFiles,
   TransactionalManagerType
 }
 
@@ -232,8 +240,28 @@ object Defaults extends BuildCommon {
       closeClassLoaders :== SysProp.closeClassLoaders,
       allowZombieClassLoaders :== true,
       packageTimestamp :== Pkg.defaultTimestamp,
+      rootPaths := {
+        val app = appConfiguration.value
+        val coursierCache = csrCacheDirectory.value.toPath
+        val out = rootOutputDirectory.value
+        getRootPaths(out, app) + ("CSR_CACHE" -> coursierCache)
+      },
+      fileConverter := MappedFileConverter(rootPaths.value, allowMachinePath.value)
     ) ++ BuildServerProtocol.globalSettings
 
+  private[sbt] def getRootPaths(out: NioPath, app: AppConfiguration): ListMap[String, NioPath] =
+    val base = app.baseDirectory.getCanonicalFile.toPath
+    val boot = app.provider.scalaProvider.launcher.bootDirectory.toPath
+    val ih = app.provider.scalaProvider.launcher.ivyHome.toPath
+    val javaHome = Paths.get(sys.props("java.home"))
+    ListMap(
+      "OUT" -> out,
+      "BASE" -> base,
+      "SBT_BOOT" -> boot,
+      "IVY_HOME" -> ih,
+      "JAVA_HOME" -> javaHome
+    )
+
   private[sbt] lazy val globalIvyCore: Seq[Setting[_]] =
     Seq(
       internalConfigurationMap :== Configurations.internalMap _,
@@ -282,6 +310,10 @@ object Defaults extends BuildCommon {
       csrLogger := LMCoursier.coursierLoggerTask.value,
       csrMavenProfiles :== Set.empty,
       csrReconciliations :== LMCoursier.relaxedForAllModules,
+      csrCacheDirectory := {
+        if (useCoursier.value) LMCoursier.defaultCacheLocation
+        else Classpaths.dummyCoursierDirectory(appConfiguration.value)
+      }
     )
 
   /** Core non-plugin settings for sbt builds.  These *must* be on every build or the sbt engine will fail to run at all. */
@@ -410,24 +442,6 @@ object Defaults extends BuildCommon {
 
   private[sbt] lazy val buildLevelJvmSettings: Seq[Setting[_]] = Seq(
     exportPipelining := usePipelining.value,
-    rootPaths := {
-      val app = appConfiguration.value
-      val base = app.baseDirectory.getCanonicalFile
-      val boot = app.provider.scalaProvider.launcher.bootDirectory
-      val ih = app.provider.scalaProvider.launcher.ivyHome
-      val coursierCache = csrCacheDirectory.value
-      val javaHome = Paths.get(sys.props("java.home"))
-      val out = rootOutputDirectory.value
-      ListMap(
-        "OUT" -> out,
-        "BASE" -> base.toPath,
-        "SBT_BOOT" -> boot.toPath,
-        "CSR_CACHE" -> coursierCache.toPath,
-        "IVY_HOME" -> ih.toPath,
-        "JAVA_HOME" -> javaHome,
-      )
-    },
-    fileConverter := MappedFileConverter(rootPaths.value, allowMachinePath.value),
     sourcePositionMappers := Nil, // Never set a default sourcePositionMapper, see #6352! Whatever you are trying to solve, do it in the foldMappers method.
     // The virtual file value cache needs to be global or sbt will run out of direct byte buffer memory.
     classpathDefinesClassCache := VirtualFileValueCache.definesClassCache(fileConverter.value),
@@ -526,14 +540,6 @@ object Defaults extends BuildCommon {
       .getOrElse(pos)
   }
 
-  // csrCacheDirectory is scoped to ThisBuild to allow customization.
-  private[sbt] lazy val buildLevelIvySettings: Seq[Setting[_]] = Seq(
-    csrCacheDirectory := {
-      if (useCoursier.value) LMCoursier.defaultCacheLocation
-      else Classpaths.dummyCoursierDirectory(appConfiguration.value)
-    },
-  )
-
   def defaultTestTasks(key: Scoped): Seq[Setting[_]] =
     inTask(key)(
       Seq(

main/src/main/scala/sbt/RemoteCache.scala

@@ -22,7 +22,7 @@ import sbt.ProjectExtra.*
 import sbt.ScopeFilter.Make._
 import sbt.SlashSyntax0._
 import sbt.coursierint.LMCoursier
-import sbt.internal.inc.{ HashUtil, JarUtils }
+import sbt.internal.inc.{ MappedFileConverter, HashUtil, JarUtils }
 import sbt.internal.librarymanagement._
 import sbt.internal.remotecache._
 import sbt.io.IO
@@ -36,7 +36,7 @@ import sbt.std.TaskExtra._
 import sbt.util.InterfaceUtil.toOption
 import sbt.util.{ ActionCacheStore, AggregateActionCacheStore, DiskActionCacheStore, Logger }
 import sjsonnew.JsonFormat
-import xsbti.{ HashedVirtualFileRef, VirtualFileRef }
+import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFileRef }
 import xsbti.compile.CompileAnalysis
 
 import scala.collection.mutable
@@ -46,8 +46,6 @@ object RemoteCache {
   final val cachedTestClassifier = "cached-test"
   final val commitLength = 10
 
-  def cacheStore: ActionCacheStore = Def.cacheStore
-
   // TODO: cap with caffeine
   private[sbt] val analysisStore: mutable.Map[HashedVirtualFileRef, CompileAnalysis] =
     mutable.Map.empty
@@ -56,14 +54,22 @@ object RemoteCache {
   // currently this is called twice so metabuild can call compile with a minimal setting
   private[sbt] def initializeRemoteCache(s: State): Unit =
     val outDir =
-      s.get(BasicKeys.rootOutputDirectory).getOrElse((s.baseDir / "target" / "out").toPath())
+      s.get(BasicKeys.rootOutputDirectory).getOrElse((s.baseDir / "target" / "out").toPath)
     Def._outputDirectory = Some(outDir)
-    val caches = s.get(BasicKeys.cacheStores)
-    caches match
-      case Some(xs) if xs.nonEmpty => Def._cacheStore = AggregateActionCacheStore(xs)
-      case _ =>
-        val tempDiskCache = (s.baseDir / "target" / "bootcache").toPath()
-        Def._cacheStore = DiskActionCacheStore(tempDiskCache)
+    def defaultCache =
+      val fileConverter = s
+        .get(Keys.fileConverter.key)
+        .getOrElse {
+          MappedFileConverter(
+            Defaults.getRootPaths(outDir, s.configuration),
+            allowMachinePath = true
+          )
+        }
+      DiskActionCacheStore((s.baseDir / "target" / "bootcache").toPath, fileConverter)
+    Def._cacheStore = s
+      .get(BasicKeys.cacheStores)
+      .collect { case xs if xs.nonEmpty => AggregateActionCacheStore(xs) }
+      .getOrElse(defaultCache)
 
   private[sbt] def artifactToStr(art: Artifact): String = {
     import LibraryManagementCodec._
@@ -104,7 +110,7 @@ object RemoteCache {
     },
     cacheStores := {
       List(
-        DiskActionCacheStore(localCacheDirectory.value.toPath())
+        DiskActionCacheStore(localCacheDirectory.value.toPath(), fileConverter.value)
       )
     },
   )

main/src/main/scala/sbt/plugins/IvyPlugin.scala

@@ -28,8 +28,6 @@ object IvyPlugin extends AutoPlugin {
 
   override lazy val globalSettings: Seq[Setting[_]] =
     Defaults.globalIvyCore
-  override lazy val buildSettings: Seq[Setting[_]] =
-    Defaults.buildLevelIvySettings
   override lazy val projectSettings: Seq[Setting[_]] =
     Classpaths.ivyPublishSettings ++ Classpaths.ivyBaseSettings
 

util-cache/src/main/scala/sbt/util/ActionCacheStore.scala

@@ -10,7 +10,7 @@ import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 import sbt.io.IO
 import sbt.io.syntax.*
-import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile }
+import xsbti.{ FileConverter, HashedVirtualFileRef, PathBasedFile, VirtualFile }
 
 /**
  * An abstraction of a remote or local cache store.
@@ -129,7 +129,7 @@ class InMemoryActionCacheStore extends ActionCacheStore:
     underlying.toString()
 end InMemoryActionCacheStore
 
-class DiskActionCacheStore(base: Path) extends ActionCacheStore:
+class DiskActionCacheStore(base: Path, fileConverter: FileConverter) extends ActionCacheStore:
   lazy val casBase: Path = {
     val dir = base.resolve("cas")
     IO.createDirectory(dir.toFile)
@@ -181,13 +181,10 @@ class DiskActionCacheStore(base: Path) extends ActionCacheStore:
       else None
 
   override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] =
-    refs.flatMap: r =>
-      val casFile = casBase.toFile / Digest(r.contentHashStr).toString
+    refs.flatMap: ref =>
+      val casFile = casBase.toFile / Digest(ref.contentHashStr).toString
       if casFile.exists then
-        val shortPath =
-          if r.id.startsWith("${OUT}/") then r.id.drop(7)
-          else r.id
-        val outPath = outputDirectory.resolve(shortPath)
+        val outPath = fileConverter.toPath(ref)
         Files.createDirectories(outPath.getParent())
         if outPath.toFile().exists() then IO.delete(outPath.toFile())
         Some(Files.createSymbolicLink(outPath, casFile.toPath))

util-cache/src/test/scala/sbt/util/ActionCacheTest.scala

@@ -4,7 +4,13 @@ import sbt.internal.util.StringVirtualFile1
 import sbt.io.IO
 import sbt.io.syntax.*
 import verify.BasicTestSuite
+import xsbti.FileConverter
 import xsbti.VirtualFile
+import xsbti.VirtualFileRef
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
 
 object ActionCacheTest extends BasicTestSuite:
   val tags = CacheLevelTag.all.toList
@@ -13,10 +19,10 @@ object ActionCacheTest extends BasicTestSuite:
     withDiskCache(testHoldBlob)
 
   def testHoldBlob(cache: ActionCacheStore): Unit =
-    val in = StringVirtualFile1("a.txt", "foo")
-    val hashRefs = cache.putBlobs(in :: Nil)
-    assert(hashRefs.size == 1)
     IO.withTemporaryDirectory: tempDir =>
+      val in = StringVirtualFile1(s"$tempDir/a.txt", "foo")
+      val hashRefs = cache.putBlobs(in :: Nil)
+      assert(hashRefs.size == 1)
       val actual = cache.syncBlobs(hashRefs, tempDir.toPath()).head
       assert(actual.getFileName().toString() == "a.txt")
 
@@ -48,14 +54,14 @@ object ActionCacheTest extends BasicTestSuite:
     withDiskCache(testActionCacheWithBlob)
 
   def testActionCacheWithBlob(cache: ActionCacheStore): Unit =
-    import sjsonnew.BasicJsonProtocol.*
-    var called = 0
-    val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
-      called += 1
-      val out = StringVirtualFile1("a.txt", (a + b).toString)
-      (a + b, Seq(out))
-    }
     IO.withTemporaryDirectory: (tempDir) =>
+      import sjsonnew.BasicJsonProtocol.*
+      var called = 0
+      val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
+        called += 1
+        val out = StringVirtualFile1(s"$tempDir/a.txt", (a + b).toString)
+        (a + b, Seq(out))
+      }
       val config = BuildWideCacheConfiguration(cache, tempDir.toPath())
       val v1 =
         ActionCache.cache[(Int, Int), Int]((1, 1), Digest.zero, Digest.zero, tags)(action)(config)
@@ -81,9 +87,15 @@ object ActionCacheTest extends BasicTestSuite:
     IO.withTemporaryDirectory(
       { tempDir0 =>
         val tempDir = tempDir0.toPath
-        val cache = DiskActionCacheStore(tempDir)
+        val cache = DiskActionCacheStore(tempDir, fileConverter)
         f(cache)
       },
       keepDirectory = false
     )
+
+  def fileConverter = new FileConverter:
+    override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id)
+    override def toVirtualFile(path: Path): VirtualFile =
+      val content = if Files.isRegularFile(path) then new String(Files.readAllBytes(path)) else ""
+      StringVirtualFile1(path.toString, content)
 end ActionCacheTest