mirror of https://github.com/sbt/sbt.git
[2.0.x] feat: Execution log (#9203)
**Problem** We need some tooling to debug caching issues. **Solution** This adds an exeprimental execution log support, which shows input and output of cached tasks.
This commit is contained in:
eugene yokota
Eugene Yokota
parent
217e0091f9
commit
fc15e03c33
|
|
@ -11,7 +11,7 @@ package sbt
|
|||
import java.io.{ File, IOException }
|
||||
import java.net.URI
|
||||
import java.nio.channels.ClosedChannelException
|
||||
import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files }
|
||||
import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files, Paths }
|
||||
import java.util.Properties
|
||||
import java.util.concurrent.ForkJoinPool
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
|
@ -32,7 +32,7 @@ import sbt.internal.util.complete.Parser
|
|||
import sbt.internal.util.{ RunningProcesses, Terminal as ITerminal, * }
|
||||
import sbt.io.*
|
||||
import sbt.io.syntax.*
|
||||
import sbt.util.{ Level, Logger, Show }
|
||||
import sbt.util.{ ActionCache, Level, Logger, Show }
|
||||
import xsbti.AppProvider
|
||||
|
||||
import scala.annotation.{ nowarn, tailrec }
|
||||
|
|
@ -260,12 +260,21 @@ object StandardMain {
|
|||
args.flatMap(levelFromArg).headOption.getOrElse(Level.Info)
|
||||
|
||||
private def initialGlobalLogging(file: Option[File], initialLevel: Level.Value): GlobalLogging =
|
||||
def createTemp(attempt: Int = 0): File = Retry:
|
||||
def createTemp(prefix: String)(attempt: Int = 0): File = Retry:
|
||||
file.foreach(f => if (!f.exists()) IO.createDirectory(f))
|
||||
File.createTempFile("sbt-global-log", ".log", file.orNull)
|
||||
File.createTempFile(prefix, ".log", file.orNull)
|
||||
|
||||
// Call this experimental since we don't want to commit to the log format for now
|
||||
val execLogProp = "sbt.experimental_execution_log"
|
||||
val execLog =
|
||||
if java.lang.Boolean.getBoolean(execLogProp) then
|
||||
Option(ActionCache.setExecLog(createTemp("exec-log")().toPath()))
|
||||
else sys.props.get(execLogProp).map(Paths.get(_)).map(ActionCache.setExecLog)
|
||||
execLog.foreach: log =>
|
||||
ShutdownHooks.add(() => log.close())
|
||||
GlobalLogging.initial(
|
||||
MainAppender.globalDefault(ConsoleOut.globalProxy),
|
||||
createTemp(),
|
||||
createTemp("sbt-global-log")(),
|
||||
ConsoleOut.globalProxy,
|
||||
initialLevel
|
||||
)
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ object DatatypeConfig {
|
|||
case "Map" | "Tuple2" | "scala.Tuple2" => { tpe =>
|
||||
twoArgs(tpe).flatMap(getFormats)
|
||||
}
|
||||
case "Int" | "Long" => { _ =>
|
||||
case "Int" | "Long" | "sbt.util.Digest" => { _ =>
|
||||
Nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util
|
||||
final class SpawnExec private (
|
||||
val input: sbt.internal.util.SpawnInput,
|
||||
val cacheHit: Boolean,
|
||||
val exitCode: Option[Int],
|
||||
val outputs: Vector[xsbti.HashedVirtualFileRef]) extends Serializable {
|
||||
|
||||
private def this(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]) = this(input, cacheHit, exitCode, Vector())
|
||||
|
||||
override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
|
||||
case x: SpawnExec => (this.input == x.input) && (this.cacheHit == x.cacheHit) && (this.exitCode == x.exitCode) && (this.outputs == x.outputs)
|
||||
case _ => false
|
||||
})
|
||||
override def hashCode: Int = {
|
||||
37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnExec".##) + input.##) + cacheHit.##) + exitCode.##) + outputs.##)
|
||||
}
|
||||
override def toString: String = {
|
||||
"SpawnExec(" + input + ", " + cacheHit + ", " + exitCode + ", " + outputs + ")"
|
||||
}
|
||||
private def copy(input: sbt.internal.util.SpawnInput = input, cacheHit: Boolean = cacheHit, exitCode: Option[Int] = exitCode, outputs: Vector[xsbti.HashedVirtualFileRef] = outputs): SpawnExec = {
|
||||
new SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
}
|
||||
def withInput(input: sbt.internal.util.SpawnInput): SpawnExec = {
|
||||
copy(input = input)
|
||||
}
|
||||
def withCacheHit(cacheHit: Boolean): SpawnExec = {
|
||||
copy(cacheHit = cacheHit)
|
||||
}
|
||||
def withExitCode(exitCode: Option[Int]): SpawnExec = {
|
||||
copy(exitCode = exitCode)
|
||||
}
|
||||
def withExitCode(exitCode: Int): SpawnExec = {
|
||||
copy(exitCode = Option(exitCode))
|
||||
}
|
||||
def withOutputs(outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = {
|
||||
copy(outputs = outputs)
|
||||
}
|
||||
}
|
||||
object SpawnExec {
|
||||
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]): SpawnExec = new SpawnExec(input, cacheHit, exitCode)
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode))
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int], outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int, outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode), outputs)
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util
|
||||
final class SpawnInput private (
|
||||
val digest: sbt.util.Digest,
|
||||
val codeContentHash: sbt.util.Digest,
|
||||
val extraHash: sbt.util.Digest,
|
||||
val cacheVersion: Option[Long],
|
||||
val str: Option[String]) extends Serializable {
|
||||
|
||||
|
||||
|
||||
override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
|
||||
case x: SpawnInput => (this.digest == x.digest) && (this.codeContentHash == x.codeContentHash) && (this.extraHash == x.extraHash) && (this.cacheVersion == x.cacheVersion) && (this.str == x.str)
|
||||
case _ => false
|
||||
})
|
||||
override def hashCode: Int = {
|
||||
37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnInput".##) + digest.##) + codeContentHash.##) + extraHash.##) + cacheVersion.##) + str.##)
|
||||
}
|
||||
override def toString: String = {
|
||||
"SpawnInput(" + digest + ", " + codeContentHash + ", " + extraHash + ", " + cacheVersion + ", " + str + ")"
|
||||
}
|
||||
private def copy(digest: sbt.util.Digest = digest, codeContentHash: sbt.util.Digest = codeContentHash, extraHash: sbt.util.Digest = extraHash, cacheVersion: Option[Long] = cacheVersion, str: Option[String] = str): SpawnInput = {
|
||||
new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
}
|
||||
def withDigest(digest: sbt.util.Digest): SpawnInput = {
|
||||
copy(digest = digest)
|
||||
}
|
||||
def withCodeContentHash(codeContentHash: sbt.util.Digest): SpawnInput = {
|
||||
copy(codeContentHash = codeContentHash)
|
||||
}
|
||||
def withExtraHash(extraHash: sbt.util.Digest): SpawnInput = {
|
||||
copy(extraHash = extraHash)
|
||||
}
|
||||
def withCacheVersion(cacheVersion: Option[Long]): SpawnInput = {
|
||||
copy(cacheVersion = cacheVersion)
|
||||
}
|
||||
def withCacheVersion(cacheVersion: Long): SpawnInput = {
|
||||
copy(cacheVersion = Option(cacheVersion))
|
||||
}
|
||||
def withStr(str: Option[String]): SpawnInput = {
|
||||
copy(str = str)
|
||||
}
|
||||
def withStr(str: String): SpawnInput = {
|
||||
copy(str = Option(str))
|
||||
}
|
||||
}
|
||||
object SpawnInput {
|
||||
|
||||
def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Option[Long], str: Option[String]): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Long, str: String): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, Option(cacheVersion), Option(str))
|
||||
}
|
||||
11
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala
generated
Normal file
11
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala
generated
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
trait SpawnCodec extends sjsonnew.BasicJsonProtocol
|
||||
with sbt.internal.util.codec.SpawnInputFormats
|
||||
with sbt.internal.util.codec.HashedVirtualFileRefFormats
|
||||
with sbt.internal.util.codec.SpawnExecFormats
|
||||
object SpawnCodec extends SpawnCodec
|
||||
33
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala
generated
Normal file
33
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala
generated
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
|
||||
trait SpawnExecFormats { self: sbt.internal.util.codec.SpawnInputFormats & sjsonnew.BasicJsonProtocol & sbt.internal.util.codec.HashedVirtualFileRefFormats =>
|
||||
given SpawnExecFormat: JsonFormat[sbt.internal.util.SpawnExec] = new JsonFormat[sbt.internal.util.SpawnExec] {
|
||||
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnExec = {
|
||||
__jsOpt match {
|
||||
case Some(__js) =>
|
||||
unbuilder.beginObject(__js)
|
||||
val input = unbuilder.readField[sbt.internal.util.SpawnInput]("input")
|
||||
val cacheHit = unbuilder.readField[Boolean]("cacheHit")
|
||||
val exitCode = unbuilder.readField[Option[Int]]("exitCode")
|
||||
val outputs = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputs")
|
||||
unbuilder.endObject()
|
||||
sbt.internal.util.SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
case None =>
|
||||
deserializationError("Expected JsObject but found None")
|
||||
}
|
||||
}
|
||||
override def write[J](obj: sbt.internal.util.SpawnExec, builder: Builder[J]): Unit = {
|
||||
builder.beginObject()
|
||||
builder.addField("input", obj.input)
|
||||
builder.addField("cacheHit", obj.cacheHit)
|
||||
builder.addField("exitCode", obj.exitCode)
|
||||
builder.addField("outputs", obj.outputs)
|
||||
builder.endObject()
|
||||
}
|
||||
}
|
||||
}
|
||||
35
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala
generated
Normal file
35
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala
generated
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
|
||||
trait SpawnInputFormats { self: sjsonnew.BasicJsonProtocol =>
|
||||
given SpawnInputFormat: JsonFormat[sbt.internal.util.SpawnInput] = new JsonFormat[sbt.internal.util.SpawnInput] {
|
||||
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnInput = {
|
||||
__jsOpt match {
|
||||
case Some(__js) =>
|
||||
unbuilder.beginObject(__js)
|
||||
val digest = unbuilder.readField[sbt.util.Digest]("digest")
|
||||
val codeContentHash = unbuilder.readField[sbt.util.Digest]("codeContentHash")
|
||||
val extraHash = unbuilder.readField[sbt.util.Digest]("extraHash")
|
||||
val cacheVersion = unbuilder.readField[Option[Long]]("cacheVersion")
|
||||
val str = unbuilder.readField[Option[String]]("str")
|
||||
unbuilder.endObject()
|
||||
sbt.internal.util.SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
case None =>
|
||||
deserializationError("Expected JsObject but found None")
|
||||
}
|
||||
}
|
||||
override def write[J](obj: sbt.internal.util.SpawnInput, builder: Builder[J]): Unit = {
|
||||
builder.beginObject()
|
||||
builder.addField("digest", obj.digest)
|
||||
builder.addField("codeContentHash", obj.codeContentHash)
|
||||
builder.addField("extraHash", obj.extraHash)
|
||||
builder.addField("cacheVersion", obj.cacheVersion)
|
||||
builder.addField("str", obj.str)
|
||||
builder.endObject()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package sbt.internal.util
|
||||
@target(Scala)
|
||||
@codecPackage("sbt.internal.util.codec")
|
||||
@fullCodec("SpawnCodec")
|
||||
|
||||
# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto
|
||||
type SpawnInput {
|
||||
digest: sbt.util.Digest!
|
||||
codeContentHash: sbt.util.Digest!
|
||||
extraHash: sbt.util.Digest!
|
||||
cacheVersion: Long
|
||||
str: String
|
||||
}
|
||||
|
||||
# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto
|
||||
type SpawnExec {
|
||||
input: sbt.internal.util.SpawnInput!
|
||||
cacheHit: Boolean!
|
||||
exitCode: Int
|
||||
outputs: [xsbti.HashedVirtualFileRef] @since("0.2.0")
|
||||
}
|
||||
|
|
@ -8,10 +8,16 @@
|
|||
|
||||
package sbt.util
|
||||
|
||||
import java.io.{ File, IOException }
|
||||
import java.io.{ File, IOException, PrintWriter }
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.{ Files, Path, Paths, StandardCopyOption }
|
||||
import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 }
|
||||
import sbt.internal.util.{
|
||||
ActionCacheEvent,
|
||||
CacheEventLog,
|
||||
SpawnExec,
|
||||
SpawnInput,
|
||||
StringVirtualFile1
|
||||
}
|
||||
import sbt.io.syntax.*
|
||||
import sbt.io.IO
|
||||
import sbt.nio.file.{ **, FileTreeView }
|
||||
|
|
@ -19,10 +25,10 @@ import sbt.nio.file.syntax.*
|
|||
import sbt.util.CacheImplicits
|
||||
import scala.reflect.ClassTag
|
||||
import scala.annotation.{ meta, StaticAnnotation }
|
||||
import scala.util.control.{ Exception, NonFatal }
|
||||
import scala.util.control.NonFatal
|
||||
import sjsonnew.{ HashWriter, JsonFormat }
|
||||
import sjsonnew.support.murmurhash.Hasher
|
||||
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
|
||||
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter }
|
||||
import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
|
||||
import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
|
||||
|
||||
|
|
@ -31,6 +37,11 @@ object ActionCache:
|
|||
private[sbt] val manifestFileName = "sbtdir_manifest.json"
|
||||
private[sbt] val failureFileName = "failure.json"
|
||||
private[sbt] val failureExitCode = 1
|
||||
private[sbt] var execLog: Option[PrintWriter] = None
|
||||
private[sbt] def setExecLog(log: Path): PrintWriter =
|
||||
val writer = PrintWriter(log.toFile, "UTF-8")
|
||||
execLog = Option(writer)
|
||||
writer
|
||||
|
||||
/**
|
||||
* This is a key function that drives remote caching.
|
||||
|
|
@ -58,17 +69,19 @@ object ActionCache:
|
|||
): O =
|
||||
import config.*
|
||||
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, cacheVersion)
|
||||
|
||||
def cacheFailure(e: CompileFailed): Nothing =
|
||||
// Cache the failure so subsequent builds don't re-run failed compilation
|
||||
// This fixes https://github.com/sbt/sbt/issues/7662
|
||||
// Use the same input digest as success, distinguished by exitCode
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val cachedFailure = CachedCompileFailure.fromException(e)
|
||||
val json = Converter.toJsonUnsafe(cachedFailure)
|
||||
val valuePath = mkValuePath(inputDigest)
|
||||
val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
store.put(
|
||||
UpdateActionResultRequest(input, Vector(failureFile), exitCode = failureExitCode)
|
||||
UpdateActionResultRequest(inputDigest, Vector(failureFile), exitCode = failureExitCode)
|
||||
)
|
||||
throw e
|
||||
|
||||
|
|
@ -81,49 +94,82 @@ object ActionCache:
|
|||
cacheFailure(e)
|
||||
case e: Exception =>
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logExec(
|
||||
SpawnExec(input = spawnInput, cacheHit = false, exitCode = 1)
|
||||
)
|
||||
throw e
|
||||
val json = Converter.toJsonUnsafe(result)
|
||||
val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize()
|
||||
val uncacheableOutputs =
|
||||
outputs.filter(f =>
|
||||
f match
|
||||
case vf if vf.id.endsWith(ActionCache.dirZipExt) =>
|
||||
false
|
||||
case _ =>
|
||||
val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize()
|
||||
!outputPath.startsWith(normalizedOutputDir)
|
||||
)
|
||||
if uncacheableOutputs.nonEmpty then
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logger.error(
|
||||
s"Cannot cache task because its output files are outside the output directory: \n" +
|
||||
uncacheableOutputs.mkString(" - ", "\n - ", "")
|
||||
)
|
||||
result
|
||||
else
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
val newOutputs = Vector(valueFile) ++ outputs.toVector
|
||||
try
|
||||
store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match
|
||||
try
|
||||
val json = Converter.toJsonUnsafe(result)
|
||||
val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize()
|
||||
val uncacheableOutputs =
|
||||
outputs.filter(f =>
|
||||
f match
|
||||
case vf if vf.id.endsWith(ActionCache.dirZipExt) =>
|
||||
false
|
||||
case _ =>
|
||||
val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize()
|
||||
!outputPath.startsWith(normalizedOutputDir)
|
||||
)
|
||||
if uncacheableOutputs.nonEmpty then
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logger.error(
|
||||
s"Cannot cache task because its output files are outside the output directory: \n" +
|
||||
uncacheableOutputs.mkString(" - ", "\n - ", "")
|
||||
)
|
||||
result
|
||||
else
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val valuePath = mkValuePath(inputDigest)
|
||||
val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
val newOutputs = Vector(valueFile) ++ outputs.toVector
|
||||
store.put(UpdateActionResultRequest(inputDigest, newOutputs, exitCode = 0)) match
|
||||
case Right(cachedResult) =>
|
||||
store.syncBlobs(cachedResult.outputFiles, outputDirectory)
|
||||
logExec(
|
||||
SpawnExec(
|
||||
input = spawnInput,
|
||||
cacheHit = false,
|
||||
exitCode = 0,
|
||||
outputs = cachedResult.outputFiles,
|
||||
)
|
||||
)
|
||||
result
|
||||
case Left(e) => throw e
|
||||
catch
|
||||
case e: IOException =>
|
||||
logger.debug(s"Skipping cache storage due to error: ${e.getMessage}")
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
result
|
||||
catch
|
||||
case e: IOException =>
|
||||
logger.debug(s"Skipping cache storage due to error: ${e.getMessage}")
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
result
|
||||
|
||||
def spawnInput = SpawnInput(
|
||||
digest = inputDigest,
|
||||
codeContentHash = codeContentHash,
|
||||
extraHash = extraHash,
|
||||
cacheVersion = if cacheVersion != 0 then Some(cacheVersion) else None,
|
||||
str = Some(key.toString()),
|
||||
)
|
||||
inline def logExec(inline event: SpawnExec): Unit =
|
||||
execLog.foreach: log =>
|
||||
logEvent(event, log)
|
||||
// Single cache lookup - use exitCode to distinguish success from failure
|
||||
getWithFailure(key, codeContentHash, extraHash, tags, config) match
|
||||
case Right(value) => value
|
||||
getWithFailure(inputDigest, tags, config) match
|
||||
case Right((value, result)) =>
|
||||
logExec(
|
||||
SpawnExec(
|
||||
input = spawnInput,
|
||||
cacheHit = true,
|
||||
exitCode = result.exitCode,
|
||||
outputs = result.outputFiles,
|
||||
)
|
||||
)
|
||||
value
|
||||
case Left(Some(failure)) =>
|
||||
config.cacheEventLog.append(ActionCacheEvent.Found("cached-failure"))
|
||||
// Replay problems to the logger so users see the cached errors/warnings
|
||||
failure.replay(config.logger)
|
||||
logExec(
|
||||
SpawnExec(input = spawnInput, cacheHit = true, exitCode = 1)
|
||||
)
|
||||
throw failure.toException
|
||||
case Left(None) => organicTask
|
||||
end cache
|
||||
|
|
@ -133,13 +179,11 @@ object ActionCache:
|
|||
* Returns Right(value) for cached success, Left(Some(failure)) for cached failure,
|
||||
* or Left(None) for cache miss.
|
||||
*/
|
||||
private def getWithFailure[I: HashWriter, O: JsonFormat](
|
||||
key: I,
|
||||
codeContentHash: Digest,
|
||||
extraHash: Digest,
|
||||
private def getWithFailure[O: JsonFormat](
|
||||
inputDigest: Digest,
|
||||
tags: List[CacheLevelTag],
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Either[Option[CachedCompileFailure], O] =
|
||||
): Either[Option[CachedCompileFailure], (O, ActionResult)] =
|
||||
import config.store
|
||||
def valueFromStr(str: String, origin: Option[String]): O =
|
||||
config.cacheEventLog.append(ActionCacheEvent.Found(origin.getOrElse("unknown")))
|
||||
|
|
@ -152,59 +196,36 @@ object ActionCache:
|
|||
|
||||
def parseCachedValue(
|
||||
str: String,
|
||||
origin: Option[String],
|
||||
result: ActionResult,
|
||||
isFailure: Boolean,
|
||||
): Option[Either[Option[CachedCompileFailure], O]] =
|
||||
): Option[Either[Option[CachedCompileFailure], (O, ActionResult)]] =
|
||||
try
|
||||
if isFailure then Some(Left(Some(failureFromStr(str))))
|
||||
else Some(Right(valueFromStr(str, origin)))
|
||||
else Some(Right((valueFromStr(str, result.origin), result)))
|
||||
catch case _: Exception => None
|
||||
|
||||
// Optimization: Check if we can read directly from symlinked value file
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath))
|
||||
|
||||
def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] =
|
||||
if java.nio.file.Files.isSymbolicLink(resolvedValuePath) && java.nio.file.Files
|
||||
.exists(resolvedValuePath)
|
||||
then
|
||||
Exception.nonFatalCatch
|
||||
.opt(IO.read(resolvedValuePath.toFile(), StandardCharsets.UTF_8))
|
||||
.flatMap: str =>
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode))
|
||||
catch case NonFatal(_) => None
|
||||
case Left(_) => None
|
||||
else None
|
||||
|
||||
readFromSymlink() match
|
||||
case Some(result) => result
|
||||
case None =>
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
val isFailure = result.exitCode.contains(failureExitCode)
|
||||
result.contents.headOption match
|
||||
case Some(head) =>
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
val str = String(head.array(), StandardCharsets.UTF_8)
|
||||
parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None))
|
||||
case _ =>
|
||||
val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
if paths.isEmpty then Left(None)
|
||||
else
|
||||
val str = IO.read(paths.head.toFile())
|
||||
parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None))
|
||||
catch
|
||||
case NonFatal(e) =>
|
||||
config.logger.debug(
|
||||
s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}"
|
||||
)
|
||||
Left(None)
|
||||
case Left(_) => Left(None)
|
||||
findActionResult(inputDigest, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
val isFailure = result.exitCode.contains(failureExitCode)
|
||||
result.contents.headOption match
|
||||
case Some(head) =>
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
val str = String(head.array(), StandardCharsets.UTF_8)
|
||||
parseCachedValue(str, result, isFailure).getOrElse(Left(None))
|
||||
case _ =>
|
||||
val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
if paths.isEmpty then Left(None)
|
||||
else
|
||||
val str = IO.read(paths.head.toFile())
|
||||
parseCachedValue(str, result, isFailure).getOrElse(Left(None))
|
||||
catch
|
||||
case NonFatal(e) =>
|
||||
config.logger.debug(
|
||||
s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}"
|
||||
)
|
||||
Left(None)
|
||||
case Left(_) => Left(None)
|
||||
|
||||
/**
|
||||
* Retrieves the cached value.
|
||||
|
|
@ -216,8 +237,9 @@ object ActionCache:
|
|||
tags: List[CacheLevelTag],
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Option[O] =
|
||||
getWithFailure(key, codeContentHash, extraHash, tags, config) match
|
||||
case Right(value) => Some(value)
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
getWithFailure(inputDigest, tags, config) match
|
||||
case Right(value) => Some(value._1)
|
||||
case Left(_) => None
|
||||
|
||||
/**
|
||||
|
|
@ -229,10 +251,26 @@ object ActionCache:
|
|||
extraHash: Digest,
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Boolean =
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
findActionResult(inputDigest, config) match
|
||||
case Right(_) => true
|
||||
case Left(_) => false
|
||||
|
||||
inline private[sbt] def findActionResult(
|
||||
inputDigest: Digest,
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Either[Throwable, ActionResult] =
|
||||
// val logger = config.logger
|
||||
CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
|
||||
val getRequest =
|
||||
GetActionResultRequest(
|
||||
inputDigest,
|
||||
inlineStdout = false,
|
||||
inlineStderr = false,
|
||||
Vector(mkValuePath(inputDigest))
|
||||
)
|
||||
config.store.get(getRequest)
|
||||
|
||||
inline private[sbt] def findActionResult[I: HashWriter](
|
||||
key: I,
|
||||
codeContentHash: Digest,
|
||||
|
|
@ -241,9 +279,14 @@ object ActionCache:
|
|||
): Either[Throwable, ActionResult] =
|
||||
// val logger = config.logger
|
||||
CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val getRequest =
|
||||
GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
|
||||
GetActionResultRequest(
|
||||
inputDigest,
|
||||
inlineStdout = false,
|
||||
inlineStderr = false,
|
||||
Vector(mkValuePath(inputDigest))
|
||||
)
|
||||
config.store.get(getRequest)
|
||||
|
||||
private inline def mkInput[I: HashWriter](
|
||||
|
|
@ -251,17 +294,20 @@ object ActionCache:
|
|||
codeContentHash: Digest,
|
||||
extraHash: Digest,
|
||||
cacheVersion: Long,
|
||||
): (Digest, String) =
|
||||
val effectiveExtraHash =
|
||||
if cacheVersion != 0L then Digest.sha256Hash(extraHash, Digest.dummy(cacheVersion))
|
||||
else extraHash
|
||||
val input =
|
||||
Digest.sha256Hash(
|
||||
): Digest =
|
||||
Digest.sha256Hash(
|
||||
(Vector(
|
||||
codeContentHash,
|
||||
effectiveExtraHash,
|
||||
Digest.dummy(Hasher.hashUnsafe[I](key))
|
||||
)
|
||||
(input, s"$${OUT}/value/$input.json")
|
||||
Digest.dummy(Hasher.hashUnsafe[I](key)),
|
||||
extraHash
|
||||
) ++ {
|
||||
if cacheVersion == 0 then Vector.empty
|
||||
else Vector(Digest.dummy(cacheVersion))
|
||||
})*
|
||||
)
|
||||
|
||||
private inline def mkValuePath(inputDigest: Digest): String =
|
||||
s"$${OUT}/value/${inputDigest}.json"
|
||||
|
||||
def manifestFromFile(manifest: Path): Manifest =
|
||||
import sbt.internal.util.codec.ManifestCodec.given
|
||||
|
|
@ -332,6 +378,14 @@ object ActionCache:
|
|||
private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] =
|
||||
Some(r.value, r.outputs)
|
||||
end InternalActionResult
|
||||
|
||||
private[sbt] def logEvent(event: SpawnExec, log: PrintWriter): Unit =
|
||||
import sbt.internal.util.codec.SpawnCodec.given
|
||||
val json = Converter.toJsonUnsafe(event)
|
||||
val s = PrettyPrinter(json)
|
||||
log.println(s)
|
||||
log.flush()
|
||||
|
||||
end ActionCache
|
||||
|
||||
class BuildWideCacheConfiguration(
|
||||
|
|
@ -357,7 +411,7 @@ class BuildWideCacheConfiguration(
|
|||
logger,
|
||||
cacheEventLog,
|
||||
CacheImplicits.defaultLocalDigestCacheByteSize,
|
||||
0L
|
||||
0L,
|
||||
)
|
||||
|
||||
def this(
|
||||
|
|
@ -368,7 +422,15 @@ class BuildWideCacheConfiguration(
|
|||
cacheEventLog: CacheEventLog,
|
||||
localDigestCacheByteSize: Long,
|
||||
) =
|
||||
this(store, outputDirectory, fileConverter, logger, cacheEventLog, localDigestCacheByteSize, 0L)
|
||||
this(
|
||||
store,
|
||||
outputDirectory,
|
||||
fileConverter,
|
||||
logger,
|
||||
cacheEventLog,
|
||||
localDigestCacheByteSize,
|
||||
0L,
|
||||
)
|
||||
|
||||
override def toString(): String =
|
||||
s"BuildWideCacheConfiguration(store = $store, outputDirectory = $outputDirectory)"
|
||||
|
|
|
|||
|
|
@ -83,7 +83,8 @@ end AbstractActionCacheStore
|
|||
/**
|
||||
* An aggregate ActionCacheStore.
|
||||
*/
|
||||
class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) extends AbstractActionCacheStore:
|
||||
case class AggregateActionCacheStore(stores: Seq[ActionCacheStore])
|
||||
extends AbstractActionCacheStore:
|
||||
extension [A1](xs: Seq[A1])
|
||||
// unlike collectFirst this accepts A1 => Seq[A2]
|
||||
inline def collectFirst2[A2](f: A1 => Seq[A2], size: Int): Seq[A2] =
|
||||
|
|
@ -176,7 +177,8 @@ class InMemoryActionCacheStore extends AbstractActionCacheStore:
|
|||
underlying.toString()
|
||||
end InMemoryActionCacheStore
|
||||
|
||||
class DiskActionCacheStore(base: Path, converter: FileConverter) extends AbstractActionCacheStore:
|
||||
case class DiskActionCacheStore(base: Path, converter: FileConverter)
|
||||
extends AbstractActionCacheStore:
|
||||
lazy val casBase: Path = {
|
||||
val dir = base.resolve("cas")
|
||||
IO.createDirectory(dir.toFile)
|
||||
|
|
|
|||
Loading…
Reference in New Issue