From fc15e03c33a3abff27519463b3c7cdc3ae7fee1e Mon Sep 17 00:00:00 2001 From: eugene yokota Date: Wed, 13 May 2026 23:27:38 -0700 Subject: [PATCH] [2.0.x] feat: Execution log (#9203) **Problem** We need some tooling to debug caching issues. **Solution** This adds an exeprimental execution log support, which shows input and output of cached tasks. --- main/src/main/scala/sbt/Main.scala | 19 +- project/DatatypeConfig.scala | 2 +- .../sbt/internal/util/SpawnExec.scala | 50 ++++ .../sbt/internal/util/SpawnInput.scala | 55 ++++ .../sbt/internal/util/codec/SpawnCodec.scala | 11 + .../util/codec/SpawnExecFormats.scala | 33 +++ .../util/codec/SpawnInputFormats.scala | 35 +++ util-cache/src/main/contraband/spawn.contra | 21 ++ .../src/main/scala/sbt/util/ActionCache.scala | 278 +++++++++++------- .../scala/sbt/util/ActionCacheStore.scala | 6 +- 10 files changed, 394 insertions(+), 116 deletions(-) create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala create mode 100644 util-cache/src/main/contraband/spawn.contra diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 7ef89b4ea..fea308de3 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -11,7 +11,7 @@ package sbt import java.io.{ File, IOException } import java.net.URI import java.nio.channels.ClosedChannelException -import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files } +import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files, Paths } import java.util.Properties import java.util.concurrent.ForkJoinPool import java.util.concurrent.atomic.AtomicBoolean @@ -32,7 +32,7 @@ import sbt.internal.util.complete.Parser import sbt.internal.util.{ RunningProcesses, Terminal as ITerminal, * } import sbt.io.* import sbt.io.syntax.* -import sbt.util.{ Level, Logger, Show } +import sbt.util.{ ActionCache, Level, Logger, Show } import xsbti.AppProvider import scala.annotation.{ nowarn, tailrec } @@ -260,12 +260,21 @@ object StandardMain { args.flatMap(levelFromArg).headOption.getOrElse(Level.Info) private def initialGlobalLogging(file: Option[File], initialLevel: Level.Value): GlobalLogging = - def createTemp(attempt: Int = 0): File = Retry: + def createTemp(prefix: String)(attempt: Int = 0): File = Retry: file.foreach(f => if (!f.exists()) IO.createDirectory(f)) - File.createTempFile("sbt-global-log", ".log", file.orNull) + File.createTempFile(prefix, ".log", file.orNull) + + // Call this experimental since we don't want to commit to the log format for now + val execLogProp = "sbt.experimental_execution_log" + val execLog = + if java.lang.Boolean.getBoolean(execLogProp) then + Option(ActionCache.setExecLog(createTemp("exec-log")().toPath())) + else sys.props.get(execLogProp).map(Paths.get(_)).map(ActionCache.setExecLog) + execLog.foreach: log => + ShutdownHooks.add(() => log.close()) GlobalLogging.initial( MainAppender.globalDefault(ConsoleOut.globalProxy), - createTemp(), + createTemp("sbt-global-log")(), ConsoleOut.globalProxy, initialLevel ) diff --git a/project/DatatypeConfig.scala b/project/DatatypeConfig.scala index 6327fdd27..ecc6d7e20 100644 --- a/project/DatatypeConfig.scala +++ b/project/DatatypeConfig.scala @@ -85,7 +85,7 @@ object DatatypeConfig { case "Map" | "Tuple2" | "scala.Tuple2" => { tpe => twoArgs(tpe).flatMap(getFormats) } - case "Int" | "Long" => { _ => + case "Int" | "Long" | "sbt.util.Digest" => { _ => Nil } } diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala new file mode 100644 index 000000000..92a5348cb --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala @@ -0,0 +1,50 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util +final class SpawnExec private ( + val input: sbt.internal.util.SpawnInput, + val cacheHit: Boolean, + val exitCode: Option[Int], + val outputs: Vector[xsbti.HashedVirtualFileRef]) extends Serializable { + + private def this(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]) = this(input, cacheHit, exitCode, Vector()) + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: SpawnExec => (this.input == x.input) && (this.cacheHit == x.cacheHit) && (this.exitCode == x.exitCode) && (this.outputs == x.outputs) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnExec".##) + input.##) + cacheHit.##) + exitCode.##) + outputs.##) + } + override def toString: String = { + "SpawnExec(" + input + ", " + cacheHit + ", " + exitCode + ", " + outputs + ")" + } + private def copy(input: sbt.internal.util.SpawnInput = input, cacheHit: Boolean = cacheHit, exitCode: Option[Int] = exitCode, outputs: Vector[xsbti.HashedVirtualFileRef] = outputs): SpawnExec = { + new SpawnExec(input, cacheHit, exitCode, outputs) + } + def withInput(input: sbt.internal.util.SpawnInput): SpawnExec = { + copy(input = input) + } + def withCacheHit(cacheHit: Boolean): SpawnExec = { + copy(cacheHit = cacheHit) + } + def withExitCode(exitCode: Option[Int]): SpawnExec = { + copy(exitCode = exitCode) + } + def withExitCode(exitCode: Int): SpawnExec = { + copy(exitCode = Option(exitCode)) + } + def withOutputs(outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = { + copy(outputs = outputs) + } +} +object SpawnExec { + + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]): SpawnExec = new SpawnExec(input, cacheHit, exitCode) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode)) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int], outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, exitCode, outputs) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int, outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode), outputs) +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala new file mode 100644 index 000000000..97561e582 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala @@ -0,0 +1,55 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util +final class SpawnInput private ( + val digest: sbt.util.Digest, + val codeContentHash: sbt.util.Digest, + val extraHash: sbt.util.Digest, + val cacheVersion: Option[Long], + val str: Option[String]) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: SpawnInput => (this.digest == x.digest) && (this.codeContentHash == x.codeContentHash) && (this.extraHash == x.extraHash) && (this.cacheVersion == x.cacheVersion) && (this.str == x.str) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnInput".##) + digest.##) + codeContentHash.##) + extraHash.##) + cacheVersion.##) + str.##) + } + override def toString: String = { + "SpawnInput(" + digest + ", " + codeContentHash + ", " + extraHash + ", " + cacheVersion + ", " + str + ")" + } + private def copy(digest: sbt.util.Digest = digest, codeContentHash: sbt.util.Digest = codeContentHash, extraHash: sbt.util.Digest = extraHash, cacheVersion: Option[Long] = cacheVersion, str: Option[String] = str): SpawnInput = { + new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + } + def withDigest(digest: sbt.util.Digest): SpawnInput = { + copy(digest = digest) + } + def withCodeContentHash(codeContentHash: sbt.util.Digest): SpawnInput = { + copy(codeContentHash = codeContentHash) + } + def withExtraHash(extraHash: sbt.util.Digest): SpawnInput = { + copy(extraHash = extraHash) + } + def withCacheVersion(cacheVersion: Option[Long]): SpawnInput = { + copy(cacheVersion = cacheVersion) + } + def withCacheVersion(cacheVersion: Long): SpawnInput = { + copy(cacheVersion = Option(cacheVersion)) + } + def withStr(str: Option[String]): SpawnInput = { + copy(str = str) + } + def withStr(str: String): SpawnInput = { + copy(str = Option(str)) + } +} +object SpawnInput { + + def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Option[Long], str: Option[String]): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Long, str: String): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, Option(cacheVersion), Option(str)) +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala new file mode 100644 index 000000000..143e9a077 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala @@ -0,0 +1,11 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +trait SpawnCodec extends sjsonnew.BasicJsonProtocol + with sbt.internal.util.codec.SpawnInputFormats + with sbt.internal.util.codec.HashedVirtualFileRefFormats + with sbt.internal.util.codec.SpawnExecFormats +object SpawnCodec extends SpawnCodec \ No newline at end of file diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala new file mode 100644 index 000000000..b086518b7 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala @@ -0,0 +1,33 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait SpawnExecFormats { self: sbt.internal.util.codec.SpawnInputFormats & sjsonnew.BasicJsonProtocol & sbt.internal.util.codec.HashedVirtualFileRefFormats => +given SpawnExecFormat: JsonFormat[sbt.internal.util.SpawnExec] = new JsonFormat[sbt.internal.util.SpawnExec] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnExec = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val input = unbuilder.readField[sbt.internal.util.SpawnInput]("input") + val cacheHit = unbuilder.readField[Boolean]("cacheHit") + val exitCode = unbuilder.readField[Option[Int]]("exitCode") + val outputs = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputs") + unbuilder.endObject() + sbt.internal.util.SpawnExec(input, cacheHit, exitCode, outputs) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.internal.util.SpawnExec, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("input", obj.input) + builder.addField("cacheHit", obj.cacheHit) + builder.addField("exitCode", obj.exitCode) + builder.addField("outputs", obj.outputs) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala new file mode 100644 index 000000000..ec3878999 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala @@ -0,0 +1,35 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait SpawnInputFormats { self: sjsonnew.BasicJsonProtocol => +given SpawnInputFormat: JsonFormat[sbt.internal.util.SpawnInput] = new JsonFormat[sbt.internal.util.SpawnInput] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnInput = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val digest = unbuilder.readField[sbt.util.Digest]("digest") + val codeContentHash = unbuilder.readField[sbt.util.Digest]("codeContentHash") + val extraHash = unbuilder.readField[sbt.util.Digest]("extraHash") + val cacheVersion = unbuilder.readField[Option[Long]]("cacheVersion") + val str = unbuilder.readField[Option[String]]("str") + unbuilder.endObject() + sbt.internal.util.SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.internal.util.SpawnInput, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("digest", obj.digest) + builder.addField("codeContentHash", obj.codeContentHash) + builder.addField("extraHash", obj.extraHash) + builder.addField("cacheVersion", obj.cacheVersion) + builder.addField("str", obj.str) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband/spawn.contra b/util-cache/src/main/contraband/spawn.contra new file mode 100644 index 000000000..03cfaf2f5 --- /dev/null +++ b/util-cache/src/main/contraband/spawn.contra @@ -0,0 +1,21 @@ +package sbt.internal.util +@target(Scala) +@codecPackage("sbt.internal.util.codec") +@fullCodec("SpawnCodec") + +# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto +type SpawnInput { + digest: sbt.util.Digest! + codeContentHash: sbt.util.Digest! + extraHash: sbt.util.Digest! + cacheVersion: Long + str: String +} + +# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto +type SpawnExec { + input: sbt.internal.util.SpawnInput! + cacheHit: Boolean! + exitCode: Int + outputs: [xsbti.HashedVirtualFileRef] @since("0.2.0") +} diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 04c62f8f9..50cd15c44 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -8,10 +8,16 @@ package sbt.util -import java.io.{ File, IOException } +import java.io.{ File, IOException, PrintWriter } import java.nio.charset.StandardCharsets import java.nio.file.{ Files, Path, Paths, StandardCopyOption } -import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 } +import sbt.internal.util.{ + ActionCacheEvent, + CacheEventLog, + SpawnExec, + SpawnInput, + StringVirtualFile1 +} import sbt.io.syntax.* import sbt.io.IO import sbt.nio.file.{ **, FileTreeView } @@ -19,10 +25,10 @@ import sbt.nio.file.syntax.* import sbt.util.CacheImplicits import scala.reflect.ClassTag import scala.annotation.{ meta, StaticAnnotation } -import scala.util.control.{ Exception, NonFatal } +import scala.util.control.NonFatal import sjsonnew.{ HashWriter, JsonFormat } import sjsonnew.support.murmurhash.Hasher -import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter } import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes } import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef } @@ -31,6 +37,11 @@ object ActionCache: private[sbt] val manifestFileName = "sbtdir_manifest.json" private[sbt] val failureFileName = "failure.json" private[sbt] val failureExitCode = 1 + private[sbt] var execLog: Option[PrintWriter] = None + private[sbt] def setExecLog(log: Path): PrintWriter = + val writer = PrintWriter(log.toFile, "UTF-8") + execLog = Option(writer) + writer /** * This is a key function that drives remote caching. @@ -58,17 +69,19 @@ object ActionCache: ): O = import config.* + val inputDigest = mkInput(key, codeContentHash, extraHash, cacheVersion) + def cacheFailure(e: CompileFailed): Nothing = // Cache the failure so subsequent builds don't re-run failed compilation // This fixes https://github.com/sbt/sbt/issues/7662 // Use the same input digest as success, distinguished by exitCode cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val cachedFailure = CachedCompileFailure.fromException(e) val json = Converter.toJsonUnsafe(cachedFailure) + val valuePath = mkValuePath(inputDigest) val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json)) store.put( - UpdateActionResultRequest(input, Vector(failureFile), exitCode = failureExitCode) + UpdateActionResultRequest(inputDigest, Vector(failureFile), exitCode = failureExitCode) ) throw e @@ -81,49 +94,82 @@ object ActionCache: cacheFailure(e) case e: Exception => cacheEventLog.append(ActionCacheEvent.Error) + logExec( + SpawnExec(input = spawnInput, cacheHit = false, exitCode = 1) + ) throw e - val json = Converter.toJsonUnsafe(result) - val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() - val uncacheableOutputs = - outputs.filter(f => - f match - case vf if vf.id.endsWith(ActionCache.dirZipExt) => - false - case _ => - val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() - !outputPath.startsWith(normalizedOutputDir) - ) - if uncacheableOutputs.nonEmpty then - cacheEventLog.append(ActionCacheEvent.Error) - logger.error( - s"Cannot cache task because its output files are outside the output directory: \n" + - uncacheableOutputs.mkString(" - ", "\n - ", "") - ) - result - else - cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) - val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) - val newOutputs = Vector(valueFile) ++ outputs.toVector - try - store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match + try + val json = Converter.toJsonUnsafe(result) + val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() + val uncacheableOutputs = + outputs.filter(f => + f match + case vf if vf.id.endsWith(ActionCache.dirZipExt) => + false + case _ => + val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() + !outputPath.startsWith(normalizedOutputDir) + ) + if uncacheableOutputs.nonEmpty then + cacheEventLog.append(ActionCacheEvent.Error) + logger.error( + s"Cannot cache task because its output files are outside the output directory: \n" + + uncacheableOutputs.mkString(" - ", "\n - ", "") + ) + result + else + cacheEventLog.append(ActionCacheEvent.OnsiteTask) + val valuePath = mkValuePath(inputDigest) + val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) + val newOutputs = Vector(valueFile) ++ outputs.toVector + store.put(UpdateActionResultRequest(inputDigest, newOutputs, exitCode = 0)) match case Right(cachedResult) => store.syncBlobs(cachedResult.outputFiles, outputDirectory) + logExec( + SpawnExec( + input = spawnInput, + cacheHit = false, + exitCode = 0, + outputs = cachedResult.outputFiles, + ) + ) result case Left(e) => throw e - catch - case e: IOException => - logger.debug(s"Skipping cache storage due to error: ${e.getMessage}") - cacheEventLog.append(ActionCacheEvent.Error) - result + catch + case e: IOException => + logger.debug(s"Skipping cache storage due to error: ${e.getMessage}") + cacheEventLog.append(ActionCacheEvent.Error) + result + def spawnInput = SpawnInput( + digest = inputDigest, + codeContentHash = codeContentHash, + extraHash = extraHash, + cacheVersion = if cacheVersion != 0 then Some(cacheVersion) else None, + str = Some(key.toString()), + ) + inline def logExec(inline event: SpawnExec): Unit = + execLog.foreach: log => + logEvent(event, log) // Single cache lookup - use exitCode to distinguish success from failure - getWithFailure(key, codeContentHash, extraHash, tags, config) match - case Right(value) => value + getWithFailure(inputDigest, tags, config) match + case Right((value, result)) => + logExec( + SpawnExec( + input = spawnInput, + cacheHit = true, + exitCode = result.exitCode, + outputs = result.outputFiles, + ) + ) + value case Left(Some(failure)) => config.cacheEventLog.append(ActionCacheEvent.Found("cached-failure")) // Replay problems to the logger so users see the cached errors/warnings failure.replay(config.logger) + logExec( + SpawnExec(input = spawnInput, cacheHit = true, exitCode = 1) + ) throw failure.toException case Left(None) => organicTask end cache @@ -133,13 +179,11 @@ object ActionCache: * Returns Right(value) for cached success, Left(Some(failure)) for cached failure, * or Left(None) for cache miss. */ - private def getWithFailure[I: HashWriter, O: JsonFormat]( - key: I, - codeContentHash: Digest, - extraHash: Digest, + private def getWithFailure[O: JsonFormat]( + inputDigest: Digest, tags: List[CacheLevelTag], config: BuildWideCacheConfiguration, - ): Either[Option[CachedCompileFailure], O] = + ): Either[Option[CachedCompileFailure], (O, ActionResult)] = import config.store def valueFromStr(str: String, origin: Option[String]): O = config.cacheEventLog.append(ActionCacheEvent.Found(origin.getOrElse("unknown"))) @@ -152,59 +196,36 @@ object ActionCache: def parseCachedValue( str: String, - origin: Option[String], + result: ActionResult, isFailure: Boolean, - ): Option[Either[Option[CachedCompileFailure], O]] = + ): Option[Either[Option[CachedCompileFailure], (O, ActionResult)]] = try if isFailure then Some(Left(Some(failureFromStr(str)))) - else Some(Right(valueFromStr(str, origin))) + else Some(Right((valueFromStr(str, result.origin), result))) catch case _: Exception => None - // Optimization: Check if we can read directly from symlinked value file - val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) - val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath)) - - def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] = - if java.nio.file.Files.isSymbolicLink(resolvedValuePath) && java.nio.file.Files - .exists(resolvedValuePath) - then - Exception.nonFatalCatch - .opt(IO.read(resolvedValuePath.toFile(), StandardCharsets.UTF_8)) - .flatMap: str => - findActionResult(key, codeContentHash, extraHash, config) match - case Right(result) => - try - store.syncBlobs(result.outputFiles, config.outputDirectory) - parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode)) - catch case NonFatal(_) => None - case Left(_) => None - else None - - readFromSymlink() match - case Some(result) => result - case None => - findActionResult(key, codeContentHash, extraHash, config) match - case Right(result) => - try - val isFailure = result.exitCode.contains(failureExitCode) - result.contents.headOption match - case Some(head) => - store.syncBlobs(result.outputFiles, config.outputDirectory) - val str = String(head.array(), StandardCharsets.UTF_8) - parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) - case _ => - val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) - if paths.isEmpty then Left(None) - else - val str = IO.read(paths.head.toFile()) - parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) - catch - case NonFatal(e) => - config.logger.debug( - s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}" - ) - Left(None) - case Left(_) => Left(None) + findActionResult(inputDigest, config) match + case Right(result) => + try + val isFailure = result.exitCode.contains(failureExitCode) + result.contents.headOption match + case Some(head) => + store.syncBlobs(result.outputFiles, config.outputDirectory) + val str = String(head.array(), StandardCharsets.UTF_8) + parseCachedValue(str, result, isFailure).getOrElse(Left(None)) + case _ => + val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) + if paths.isEmpty then Left(None) + else + val str = IO.read(paths.head.toFile()) + parseCachedValue(str, result, isFailure).getOrElse(Left(None)) + catch + case NonFatal(e) => + config.logger.debug( + s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}" + ) + Left(None) + case Left(_) => Left(None) /** * Retrieves the cached value. @@ -216,8 +237,9 @@ object ActionCache: tags: List[CacheLevelTag], config: BuildWideCacheConfiguration, ): Option[O] = - getWithFailure(key, codeContentHash, extraHash, tags, config) match - case Right(value) => Some(value) + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) + getWithFailure(inputDigest, tags, config) match + case Right(value) => Some(value._1) case Left(_) => None /** @@ -229,10 +251,26 @@ object ActionCache: extraHash: Digest, config: BuildWideCacheConfiguration, ): Boolean = - findActionResult(key, codeContentHash, extraHash, config) match + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) + findActionResult(inputDigest, config) match case Right(_) => true case Left(_) => false + inline private[sbt] def findActionResult( + inputDigest: Digest, + config: BuildWideCacheConfiguration, + ): Either[Throwable, ActionResult] = + // val logger = config.logger + CacheImplicits.setCacheSize(config.localDigestCacheByteSize) + val getRequest = + GetActionResultRequest( + inputDigest, + inlineStdout = false, + inlineStderr = false, + Vector(mkValuePath(inputDigest)) + ) + config.store.get(getRequest) + inline private[sbt] def findActionResult[I: HashWriter]( key: I, codeContentHash: Digest, @@ -241,9 +279,14 @@ object ActionCache: ): Either[Throwable, ActionResult] = // val logger = config.logger CacheImplicits.setCacheSize(config.localDigestCacheByteSize) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion) + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val getRequest = - GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath)) + GetActionResultRequest( + inputDigest, + inlineStdout = false, + inlineStderr = false, + Vector(mkValuePath(inputDigest)) + ) config.store.get(getRequest) private inline def mkInput[I: HashWriter]( @@ -251,17 +294,20 @@ object ActionCache: codeContentHash: Digest, extraHash: Digest, cacheVersion: Long, - ): (Digest, String) = - val effectiveExtraHash = - if cacheVersion != 0L then Digest.sha256Hash(extraHash, Digest.dummy(cacheVersion)) - else extraHash - val input = - Digest.sha256Hash( + ): Digest = + Digest.sha256Hash( + (Vector( codeContentHash, - effectiveExtraHash, - Digest.dummy(Hasher.hashUnsafe[I](key)) - ) - (input, s"$${OUT}/value/$input.json") + Digest.dummy(Hasher.hashUnsafe[I](key)), + extraHash + ) ++ { + if cacheVersion == 0 then Vector.empty + else Vector(Digest.dummy(cacheVersion)) + })* + ) + + private inline def mkValuePath(inputDigest: Digest): String = + s"$${OUT}/value/${inputDigest}.json" def manifestFromFile(manifest: Path): Manifest = import sbt.internal.util.codec.ManifestCodec.given @@ -332,6 +378,14 @@ object ActionCache: private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] = Some(r.value, r.outputs) end InternalActionResult + + private[sbt] def logEvent(event: SpawnExec, log: PrintWriter): Unit = + import sbt.internal.util.codec.SpawnCodec.given + val json = Converter.toJsonUnsafe(event) + val s = PrettyPrinter(json) + log.println(s) + log.flush() + end ActionCache class BuildWideCacheConfiguration( @@ -357,7 +411,7 @@ class BuildWideCacheConfiguration( logger, cacheEventLog, CacheImplicits.defaultLocalDigestCacheByteSize, - 0L + 0L, ) def this( @@ -368,7 +422,15 @@ class BuildWideCacheConfiguration( cacheEventLog: CacheEventLog, localDigestCacheByteSize: Long, ) = - this(store, outputDirectory, fileConverter, logger, cacheEventLog, localDigestCacheByteSize, 0L) + this( + store, + outputDirectory, + fileConverter, + logger, + cacheEventLog, + localDigestCacheByteSize, + 0L, + ) override def toString(): String = s"BuildWideCacheConfiguration(store = $store, outputDirectory = $outputDirectory)" diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index e551900f1..cb6c78e47 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -83,7 +83,8 @@ end AbstractActionCacheStore /** * An aggregate ActionCacheStore. */ -class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) extends AbstractActionCacheStore: +case class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) + extends AbstractActionCacheStore: extension [A1](xs: Seq[A1]) // unlike collectFirst this accepts A1 => Seq[A2] inline def collectFirst2[A2](f: A1 => Seq[A2], size: Int): Seq[A2] = @@ -176,7 +177,8 @@ class InMemoryActionCacheStore extends AbstractActionCacheStore: underlying.toString() end InMemoryActionCacheStore -class DiskActionCacheStore(base: Path, converter: FileConverter) extends AbstractActionCacheStore: +case class DiskActionCacheStore(base: Path, converter: FileConverter) + extends AbstractActionCacheStore: lazy val casBase: Path = { val dir = base.resolve("cas") IO.createDirectory(dir.toFile)