diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index fb6a07378..e08cfd1d1 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -8,9 +8,9 @@ package sbt.util -import java.io.File +import java.io.{ File, IOException } import java.nio.charset.StandardCharsets -import java.nio.file.{ Files, NoSuchFileException, Path, Paths, StandardCopyOption } +import java.nio.file.{ Files, Path, Paths, StandardCopyOption } import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 } import sbt.io.syntax.* import sbt.io.IO @@ -19,7 +19,7 @@ import sbt.nio.file.syntax.* import sbt.util.CacheImplicits import scala.reflect.ClassTag import scala.annotation.{ meta, StaticAnnotation } -import scala.util.control.Exception +import scala.util.control.{ Exception, NonFatal } import sjsonnew.{ HashWriter, JsonFormat } import sjsonnew.support.murmurhash.Hasher import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } @@ -81,40 +81,40 @@ object ActionCache: case e: Exception => cacheEventLog.append(ActionCacheEvent.Error) throw e - try - val json = Converter.toJsonUnsafe(result) - val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() - val uncacheableOutputs = - outputs.filter(f => - f match - case vf if vf.id.endsWith(ActionCache.dirZipExt) => - false - case _ => - val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() - !outputPath.startsWith(normalizedOutputDir) - ) - if uncacheableOutputs.nonEmpty then - cacheEventLog.append(ActionCacheEvent.Error) - logger.error( - s"Cannot cache task because its output files are outside the output directory: \n" + - uncacheableOutputs.mkString(" - ", "\n - ", "") - ) - result - else - cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) - val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) - val newOutputs = Vector(valueFile) ++ outputs.toVector + val json = Converter.toJsonUnsafe(result) + val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() + val uncacheableOutputs = + outputs.filter(f => + f match + case vf if vf.id.endsWith(ActionCache.dirZipExt) => + false + case _ => + val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() + !outputPath.startsWith(normalizedOutputDir) + ) + if uncacheableOutputs.nonEmpty then + cacheEventLog.append(ActionCacheEvent.Error) + logger.error( + s"Cannot cache task because its output files are outside the output directory: \n" + + uncacheableOutputs.mkString(" - ", "\n - ", "") + ) + result + else + cacheEventLog.append(ActionCacheEvent.OnsiteTask) + val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) + val newOutputs = Vector(valueFile) ++ outputs.toVector + try store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match case Right(cachedResult) => store.syncBlobs(cachedResult.outputFiles, outputDirectory) result case Left(e) => throw e - catch - case e: NoSuchFileException => - logger.debug(s"Skipping cache storage due to missing file: ${e.getMessage}") - cacheEventLog.append(ActionCacheEvent.Error) - result + catch + case e: IOException => + logger.debug(s"Skipping cache storage due to error: ${e.getMessage}") + cacheEventLog.append(ActionCacheEvent.Error) + result // Single cache lookup - use exitCode to distinguish success from failure getWithFailure(key, codeContentHash, extraHash, tags, config) match @@ -172,8 +172,10 @@ object ActionCache: .flatMap: str => findActionResult(key, codeContentHash, extraHash, config) match case Right(result) => - store.syncBlobs(result.outputFiles, config.outputDirectory) - parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode)) + try + store.syncBlobs(result.outputFiles, config.outputDirectory) + parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode)) + catch case NonFatal(_) => None case Left(_) => None else None @@ -182,18 +184,25 @@ object ActionCache: case None => findActionResult(key, codeContentHash, extraHash, config) match case Right(result) => - val isFailure = result.exitCode.contains(failureExitCode) - result.contents.headOption match - case Some(head) => - store.syncBlobs(result.outputFiles, config.outputDirectory) - val str = String(head.array(), StandardCharsets.UTF_8) - parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) - case _ => - val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) - if paths.isEmpty then Left(None) - else - val str = IO.read(paths.head.toFile()) + try + val isFailure = result.exitCode.contains(failureExitCode) + result.contents.headOption match + case Some(head) => + store.syncBlobs(result.outputFiles, config.outputDirectory) + val str = String(head.array(), StandardCharsets.UTF_8) parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) + case _ => + val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) + if paths.isEmpty then Left(None) + else + val str = IO.read(paths.head.toFile()) + parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) + catch + case NonFatal(e) => + config.logger.debug( + s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}" + ) + Left(None) case Left(_) => Left(None) /** diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index 733aa32e7..ccb6546ce 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -1,6 +1,6 @@ package sbt.util -import java.io.RandomAccessFile +import java.io.{ IOException, RandomAccessFile } import java.nio.ByteBuffer import java.nio.file.{ Files, @@ -221,7 +221,7 @@ class DiskActionCacheStore(base: Path, converter: FileConverter) extends Abstrac val json = Converter.toJsonUnsafe(v) IO.write(acFile, CompactPrinter(json)) Right(v) - catch case NonFatal(e) => Left(e) + catch case e: IOException => Left(e) override def putBlobs(blobs: Seq[VirtualFile]): Seq[HashedVirtualFileRef] = blobs.map: (b: VirtualFile) => diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index 1ebd5e586..7d387b71b 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -7,6 +7,7 @@ import sbt.io.syntax.* import verify.BasicTestSuite import xsbti.{ CompileFailed, + HashedVirtualFileRef, Problem, Position, Severity, @@ -168,6 +169,96 @@ object ActionCacheTest extends BasicTestSuite: assert(caught2.problems()(0).message() == "Test error message") assert(caught2.getMessage() == "Compilation failed") + test("Cache falls back to recompute when syncBlobs throws FileNotFoundException"): + withDiskCache(testSyncBlobsThrowsFallback) + + def testSyncBlobsThrowsFallback(underlying: DiskActionCacheStore): Unit = + import sjsonnew.BasicJsonProtocol.* + var called = 0 + val action: ((Int, Int)) => InternalActionResult[Int] = { (a, b) => + called += 1 + InternalActionResult(a + b, Nil) + } + class ThrowingSyncStore extends AbstractActionCacheStore: + override def storeName: String = "throwing-sync" + override def get(request: GetActionResultRequest): Either[Throwable, ActionResult] = + underlying.get(request) + override def put(request: UpdateActionResultRequest): Either[Throwable, ActionResult] = + underlying.put(request) + override def putBlobs(blobs: Seq[VirtualFile]): Seq[HashedVirtualFileRef] = + underlying.putBlobs(blobs) + override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] = + throw new java.io.FileNotFoundException("simulated missing CAS entry") + override def findBlobs(refs: Seq[HashedVirtualFileRef]): Seq[HashedVirtualFileRef] = + underlying.findBlobs(refs) + IO.withTemporaryDirectory: tempDir => + val config = getCacheConfig(ThrowingSyncStore(), tempDir) + val v1 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config)(action) + assert(v1 == 2) + assert(called == 1) + val v2 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config)(action) + assert(v2 == 2) + assert(called == 2) + + test( + "readFromSymlink fast path falls back to recompute when syncBlobs throws FileNotFoundException" + ): + IO.withTemporaryDirectory: cacheDir => + IO.withTemporaryDirectory: outputDir => + testReadFromSymlinkFallback(cacheDir, outputDir) + + def testReadFromSymlinkFallback(cacheDir: File, outputDir: File): Unit = + import sjsonnew.BasicJsonProtocol.* + val absConverter: FileConverter = new FileConverter: + override def toPath(ref: VirtualFileRef): Path = outputDir.toPath.resolve(ref.id) + override def toVirtualFile(path: Path): VirtualFile = + val content = if Files.isRegularFile(path) then new String(Files.readAllBytes(path)) else "" + StringVirtualFile1(path.toString, content) + val diskCache = DiskActionCacheStore(cacheDir.toPath, absConverter) + var called = 0 + val action: Unit => InternalActionResult[Int] = { _ => + called += 1 + InternalActionResult(42, Nil) + } + val logger = new Logger: + override def trace(t: => Throwable): Unit = () + override def success(message: => String): Unit = () + override def log(level: Level.Value, message: => String): Unit = () + val config1 = + BuildWideCacheConfiguration( + diskCache, + outputDir.toPath, + absConverter, + logger, + CacheEventLog() + ) + val v1 = ActionCache.cache((), Digest.zero, Digest.zero, tags, config1)(action) + assert(v1 == 42) + assert(called == 1) + class ThrowingSyncStore extends AbstractActionCacheStore: + override def storeName: String = "throwing-sync" + override def get(request: GetActionResultRequest): Either[Throwable, ActionResult] = + diskCache.get(request) + override def put(request: UpdateActionResultRequest): Either[Throwable, ActionResult] = + diskCache.put(request) + override def putBlobs(blobs: Seq[VirtualFile]): Seq[HashedVirtualFileRef] = + diskCache.putBlobs(blobs) + override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] = + throw new java.io.FileNotFoundException("simulated missing CAS entry") + override def findBlobs(refs: Seq[HashedVirtualFileRef]): Seq[HashedVirtualFileRef] = + diskCache.findBlobs(refs) + val config2 = + BuildWideCacheConfiguration( + ThrowingSyncStore(), + outputDir.toPath, + absConverter, + logger, + CacheEventLog() + ) + val v2 = ActionCache.cache((), Digest.zero, Digest.zero, tags, config2)(action) + assert(v2 == 42) + assert(called == 2) + def withInMemoryCache(f: InMemoryActionCacheStore => Unit): Unit = val cache = InMemoryActionCacheStore() f(cache)