diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e97f8f07a..f765e58b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,7 @@ jobs: if: ${{ matrix.jobtype == 2 }} shell: bash run: | - ./sbt -v "scripted actions/* apiinfo/* compiler-project/* ivy-deps-management/* reporter/* tests/* classloader-cache/* package/*" + ./sbt -v "scripted actions/* apiinfo/* cache/* compiler-project/* ivy-deps-management/* reporter/* tests/* classloader-cache/* package/*" # ./sbt -v "scripted watch/*" - name: Build and test (3) if: ${{ matrix.jobtype == 3 }} diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala index d39f93077..67259e8d7 100644 --- a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala +++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala @@ -294,7 +294,7 @@ trait Cont: ).asTerm.changeOwner(sym) else val tags = CacheLevelTag.all.toList - callActionCache(outputBuf.toList, cacheConfigExpr, tags)( + callActionCache(outputBuf.toList, modifiedCacheConfigExpr, tags)( body = modifiedBody, input = unitExpr, ).asTerm.changeOwner(sym) diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala index 0489bff91..57d5ca174 100644 --- a/main-settings/src/main/scala/sbt/Def.scala +++ b/main-settings/src/main/scala/sbt/Def.scala @@ -254,18 +254,23 @@ object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits: @cacheLevel(include = Array.empty) val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task { val state = stateKey.value - val outputDirectory = state.get(BasicKeys.rootOutputDirectory) + val outputDirectory = state + .get(BasicKeys.rootOutputDirectory) + .getOrElse(sys.error("outputDirectory has not been set")) + val fileConverter = + state.get(BasicKeys.fileConverter).getOrElse(sys.error("outputDirectory has not been set")) val cacheStore = state .get(BasicKeys.cacheStores) .collect { case xs if xs.nonEmpty => AggregateActionCacheStore(xs) } - .getOrElse(DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"))) - val fileConverter = state.get(BasicKeys.fileConverter) + .getOrElse( + DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter) + ) BuildWideCacheConfiguration( cacheStore, - outputDirectory.getOrElse(sys.error("outputDirectory has not been set")), - fileConverter.getOrElse(sys.error("outputDirectory has not been set")), + outputDirectory, + fileConverter, state.log, - cacheEventLog, + cacheEventLog ) } diff --git a/main/src/main/scala/sbt/RemoteCache.scala b/main/src/main/scala/sbt/RemoteCache.scala index 40113ec5d..be4864eec 100644 --- a/main/src/main/scala/sbt/RemoteCache.scala +++ b/main/src/main/scala/sbt/RemoteCache.scala @@ -89,8 +89,9 @@ object RemoteCache { .resolve("out") }, cacheStores := { + val c = fileConverter.value List( - DiskActionCacheStore(localCacheDirectory.value.toPath()) + DiskActionCacheStore(localCacheDirectory.value.toPath, c) ) }, remoteCache := SysProp.remoteCache, diff --git a/sbt-app/src/sbt-test/cache/basic/build.sbt b/sbt-app/src/sbt-test/cache/basic/build.sbt index ccb49f110..86374ddee 100644 --- a/sbt-app/src/sbt-test/cache/basic/build.sbt +++ b/sbt-app/src/sbt-test/cache/basic/build.sbt @@ -5,18 +5,18 @@ val pure1 = taskKey[Unit]("") val map1 = taskKey[String]("") val mapN1 = taskKey[Unit]("") -Global / localCacheDirectory := new File("/tmp/sbt/diskcache/") +Global / localCacheDirectory := baseDirectory.value / "diskcache" pure1 := (Def.cachedTask { - val output = StringVirtualFile1("a.txt", "foo") + val output = StringVirtualFile1("target/out/a.txt", "foo") Def.declareOutput(output) () }).value map1 := (Def.cachedTask { pure1.value - val output1 = StringVirtualFile1("b1.txt", "foo") - val output2 = StringVirtualFile1("b2.txt", "foo") + val output1 = StringVirtualFile1("target/out/b1.txt", "foo") + val output2 = StringVirtualFile1("target/out/b2.txt", "foo") Def.declareOutput(output1) Def.declareOutput(output2) "something" @@ -25,7 +25,7 @@ map1 := (Def.cachedTask { mapN1 := (Def.cachedTask { pure1.value map1.value - val output = StringVirtualFile1("c.txt", "foo") + val output = StringVirtualFile1("target/out/c.txt", "foo") Def.declareOutput(output) () }).value diff --git a/sbt-app/src/sbt-test/cache/optout/build.sbt b/sbt-app/src/sbt-test/cache/optout/build.sbt index cb07d73e3..c3ea57220 100644 --- a/sbt-app/src/sbt-test/cache/optout/build.sbt +++ b/sbt-app/src/sbt-test/cache/optout/build.sbt @@ -2,15 +2,15 @@ import sbt.internal.util.StringVirtualFile1 import sjsonnew.BasicJsonProtocol.* import CustomKeys.* -Global / localCacheDirectory := new File("/tmp/sbt/diskcache/") +Global / localCacheDirectory := baseDirectory.value / "diskcache" aa := A() -// This tests that pure1 is opt'ed out from caching +// This tests that aa is opt'ed out from caching map1 := (Def.cachedTask { aa.value - val output1 = StringVirtualFile1("b1.txt", "foo") - val output2 = StringVirtualFile1("b2.txt", "foo") + val output1 = StringVirtualFile1("target/out/b1.txt", "foo") + val output2 = StringVirtualFile1("target/out/b2.txt", "foo") Def.declareOutput(output1) Def.declareOutput(output2) "something" @@ -19,7 +19,7 @@ map1 := (Def.cachedTask { mapN1 := (Def.cachedTask { aa.value map1.value - val output = StringVirtualFile1("c.txt", "foo") + val output = StringVirtualFile1("target/out/c.txt", "foo") Def.declareOutput(output) () }).value diff --git a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala index 84fc0524d..ee178df78 100644 --- a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala +++ b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala @@ -241,10 +241,7 @@ class GrpcActionCacheStore( // per spec, Clients SHOULD NOT populate [contents] when uploading to the cache. private def toOutputFile(ref: HashedVirtualFileRef): OutputFile = val b = OutputFile.newBuilder() - val shortPath = - if ref.id.startsWith("${OUT}/") then ref.id.drop(7) - else ref.id - b.setPath(shortPath) + b.setPath(ref.id) b.setDigest(toXDigest(Digest(ref))) b.build() diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 33c3042a2..daa4edc70 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -54,7 +54,7 @@ object ActionCache: throw e val json = Converter.toJsonUnsafe(result) val uncacheableOutputs = - outputs.filter(f => !fileConverter.toPath(f).startsWith(outputDirectory)) + outputs.filter(f => !fileConverter.toPath(f).toAbsolutePath.startsWith(outputDirectory)) if uncacheableOutputs.nonEmpty then cacheEventLog.append(ActionCacheEvent.Error) logger.error( @@ -64,8 +64,8 @@ object ActionCache: result else cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val input = mkInput(key, codeContentHash, extraHash) - val valueFile = StringVirtualFile1(s"value/${input}.json", CompactPrinter(json)) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config) + val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) val newOutputs = Vector(valueFile) ++ outputs.toVector store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match case Right(cachedResult) => @@ -126,8 +126,7 @@ object ActionCache: extraHash: Digest, config: BuildWideCacheConfiguration, ): Either[Throwable, ActionResult] = - val input = mkInput(key, codeContentHash, extraHash) - val valuePath = s"value/${input}.json" + val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config) val getRequest = GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath)) config.store.get(getRequest) @@ -135,9 +134,12 @@ object ActionCache: private inline def mkInput[I: HashWriter]( key: I, codeContentHash: Digest, - extraHash: Digest - ): Digest = - Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key))) + extraHash: Digest, + config: BuildWideCacheConfiguration + ): (Digest, String) = + val input = + Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key))) + (input, s"${config.outputDirectory}/value/$input.json") def manifestFromFile(manifest: Path): Manifest = import sbt.internal.util.codec.ManifestCodec.given diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index 0b243663c..c292ce9c4 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -17,7 +17,7 @@ import sbt.nio.file.{ **, FileTreeView } import sbt.nio.file.syntax.* import sbt.internal.util.{ StringVirtualFile1, Util } import sbt.internal.util.codec.ActionResultCodec.given -import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile } +import xsbti.{ FileConverter, HashedVirtualFileRef, PathBasedFile, VirtualFile } import java.io.InputStream /** @@ -170,7 +170,7 @@ class InMemoryActionCacheStore extends AbstractActionCacheStore: underlying.toString() end InMemoryActionCacheStore -class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: +class DiskActionCacheStore(base: Path, converter: FileConverter) extends AbstractActionCacheStore: lazy val casBase: Path = { val dir = base.resolve("cas") IO.createDirectory(dir.toFile) @@ -262,9 +262,6 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: else None def syncFile(ref: HashedVirtualFileRef, casFile: Path, outputDirectory: Path): Path = - val shortPath = - if ref.id.startsWith("${OUT}/") then ref.id.drop(7) - else ref.id val d = Digest(ref) def copyFile(outPath: Path): Path = Files.copy( @@ -293,7 +290,7 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: else copyFile(outPath) afterFileWrite(ref, result, outputDirectory) result - outputDirectory.resolve(shortPath) match + converter.toPath(ref) match case p if !Files.exists(p) => // println(s"- syncFile: $p does not exist") writeFileAndNotify(p) @@ -335,18 +332,17 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: // manifest contains the list of files in the dirzip, and their hashes val m = ActionCache.manifestFromFile(mPath) m.outputFiles.foreach: ref => - val shortPath = - if ref.id.startsWith("${OUT}/") then ref.id.drop(7) - else ref.id - val currentItem = outputDirectory.resolve(shortPath) + val currentItem = converter.toPath(ref) + val shortPath = outputDirectory.relativize(currentItem).toString allPaths.remove(currentItem) val d = Digest(ref) + def tempPath = tempDir.toPath.resolve(shortPath) currentItem match - case p if !Files.exists(p) => doSync(ref, tempDir.toPath().resolve(shortPath)) + case p if !Files.exists(p) => doSync(ref, tempPath) case p if Digest.sameDigest(p, d) => () case p => IO.delete(p.toFile()) - doSync(ref, tempDir.toPath().resolve(shortPath)) + doSync(ref, tempPath) // sync deleted files allPaths.foreach: path => IO.delete(path.toFile()) diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index dea419e1a..2dc77ddd5 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -20,10 +20,10 @@ object ActionCacheTest extends BasicTestSuite: withDiskCache(testHoldBlob) def testHoldBlob(cache: ActionCacheStore): Unit = - val in = StringVirtualFile1("a.txt", "foo") - val hashRefs = cache.putBlobs(in :: Nil) - assert(hashRefs.size == 1) IO.withTemporaryDirectory: tempDir => + val in = StringVirtualFile1(s"$tempDir/a.txt", "foo") + val hashRefs = cache.putBlobs(in :: Nil) + assert(hashRefs.size == 1) val actual = cache.syncBlobs(hashRefs, tempDir.toPath()).head assert(actual.getFileName().toString() == "a.txt") @@ -88,7 +88,7 @@ object ActionCacheTest extends BasicTestSuite: IO.withTemporaryDirectory( { tempDir0 => val tempDir = tempDir0.toPath - val cache = DiskActionCacheStore(tempDir) + val cache = DiskActionCacheStore(tempDir, fileConverter) f(cache) }, keepDirectory = false