mirror of https://github.com/sbt/sbt.git
Merge pull request #9238 from eed3si9n/bport/exec_log
[2.0.x] bport: Backport execution log and cache version
This commit is contained in:
commit
37ccb83b71
|
|
@ -84,11 +84,18 @@ object GlobalLogging {
|
|||
newAppender: (ManagedLogger, PrintWriter, GlobalLogBacking, LoggerContext) => GlobalLogging,
|
||||
newBackingFile: => File,
|
||||
console: ConsoleOut
|
||||
): GlobalLogging = {
|
||||
): GlobalLogging =
|
||||
initial(newAppender, newBackingFile, console, Level.Info)
|
||||
|
||||
def initial(
|
||||
newAppender: (ManagedLogger, PrintWriter, GlobalLogBacking, LoggerContext) => GlobalLogging,
|
||||
newBackingFile: => File,
|
||||
console: ConsoleOut,
|
||||
initialLevel: Level.Value,
|
||||
): GlobalLogging =
|
||||
val loggerName = generateName
|
||||
val log = LoggerContext.globalContext.logger(loggerName, None, None)
|
||||
val appender = ConsoleAppender(ConsoleAppender.generateName(), console)
|
||||
LoggerContext.globalContext.addAppender(loggerName, appender -> Level.Info)
|
||||
LoggerContext.globalContext.addAppender(loggerName, appender -> initialLevel)
|
||||
GlobalLogging(log, console, appender, GlobalLogBacking(newBackingFile), newAppender)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -330,4 +330,8 @@ object RunnerScriptTest extends verify.BasicTestSuite with ShellScriptUtil:
|
|||
s"Should not have errors from comment character, but found: ${errorMessages.mkString(", ")}"
|
||||
)
|
||||
|
||||
testOutput("sbt --experimental_execution_log=true")("--experimental_execution_log=true", "-v"):
|
||||
(out: List[String]) =>
|
||||
assert(out.contains[String]("-Dsbt.experimental_execution_log=true"))
|
||||
|
||||
end RunnerScriptTest
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ set sbt_args_mem=
|
|||
set sbt_args_client=
|
||||
set sbt_args_jvm_client=
|
||||
set sbt_args_no_server=
|
||||
set sbt_args_experimental_execution_log=
|
||||
set is_this_dir_sbt=0
|
||||
|
||||
rem users can set SBT_OPTS via .sbtopts
|
||||
|
|
@ -419,6 +420,21 @@ if defined _color_arg (
|
|||
goto args_loop
|
||||
)
|
||||
|
||||
if "%~0" == "--experimental_execution_log" set _experimental_execution_log_arg=true
|
||||
|
||||
if defined _experimental_execution_log_arg (
|
||||
set _experimental_execution_log_arg=
|
||||
if not "%~1" == "" (
|
||||
set sbt_args_experimental_execution_log=%~1
|
||||
shift
|
||||
goto args_loop
|
||||
) else (
|
||||
echo "%~0" is missing a value
|
||||
goto error
|
||||
)
|
||||
goto args_loop
|
||||
)
|
||||
|
||||
if "%~0" == "--no-share" set _no_share_arg=true
|
||||
if "%~0" == "-no-share" set _no_share_arg=true
|
||||
|
||||
|
|
@ -724,6 +740,10 @@ if not defined sbt_args_no_hide_jdk_warnings (
|
|||
)
|
||||
)
|
||||
|
||||
if defined sbt_args_experimental_execution_log (
|
||||
set _SBT_OPTS=-Dsbt.experimental_execution_log=!sbt_args_experimental_execution_log! !_SBT_OPTS!
|
||||
)
|
||||
|
||||
rem TODO: _SBT_OPTS needs to be processed as args and diffed against SBT_ARGS
|
||||
|
||||
if !sbt_args_print_sbt_version! equ 1 (
|
||||
|
|
@ -1137,6 +1157,8 @@ echo --script-version print the version of sbt script
|
|||
echo -d ^| --debug set sbt log level to debug
|
||||
echo -debug-inc ^| --debug-inc
|
||||
echo enable extra debugging for the incremental compiler
|
||||
echo --experimental_execution_log=true^|^<path^>
|
||||
echo enable experimental execution log
|
||||
rem echo -J-X pass option -X directly to the java runtime
|
||||
rem echo ^(-J is stripped^)
|
||||
rem echo -S-X add -X to sbt's scalacOptions ^(-S is stripped^)
|
||||
|
|
|
|||
|
|
@ -129,6 +129,12 @@ object BasicKeys {
|
|||
10000
|
||||
)
|
||||
|
||||
val cacheVersion = AttributeKey[Long](
|
||||
"cacheVersion",
|
||||
"A version number that invalidates all task caches when changed.",
|
||||
10000
|
||||
)
|
||||
|
||||
// Unlike other BasicKeys, this is not used directly as a setting key,
|
||||
// and severLog / logLevel is used instead.
|
||||
private[sbt] val serverLogLevel =
|
||||
|
|
|
|||
|
|
@ -151,6 +151,8 @@ class NetworkClient(
|
|||
private lazy val noStdErr = arguments.completionArguments.contains("--no-stderr") &&
|
||||
!sys.env.contains("SBTN_AUTO_COMPLETE") && !sys.env.contains("SBTC_AUTO_COMPLETE")
|
||||
private def shutdownOnly = arguments.commandArguments == Seq(Shutdown)
|
||||
private lazy val serverAutoStart: Boolean =
|
||||
sys.props.get("sbt.server.autostart").forall(_.toLowerCase == "true")
|
||||
|
||||
private def mkSocket(file: File): (Socket, Option[String]) = ClientSocket.socket(file, useJNI)
|
||||
|
||||
|
|
@ -190,6 +192,9 @@ class NetworkClient(
|
|||
if (shutdownOnly) {
|
||||
console.appendLog(Level.Info, "no sbt server is running. ciao")
|
||||
System.exit(0)
|
||||
} else if (!serverAutoStart) {
|
||||
console.appendLog(Level.Error, "no sbt server is running (sbt.server.autostart=false)")
|
||||
System.exit(1)
|
||||
} else if (promptCompleteUsers) {
|
||||
val msg = if (noTab) "" else "No sbt server is running. Press <tab> to start one..."
|
||||
errorStream.print(s"\n$msg")
|
||||
|
|
@ -1198,7 +1203,7 @@ object NetworkClient {
|
|||
completionArguments,
|
||||
sbtScript,
|
||||
bsp,
|
||||
sbtLaunchJar
|
||||
sbtLaunchJar,
|
||||
)
|
||||
}
|
||||
private[client] val completions = "--completions"
|
||||
|
|
@ -1257,8 +1262,6 @@ object NetworkClient {
|
|||
"--timings",
|
||||
"-traces",
|
||||
"--traces",
|
||||
"-no-server",
|
||||
"--no-server",
|
||||
"-no-share",
|
||||
"--no-share",
|
||||
"-no-global",
|
||||
|
|
@ -1275,6 +1278,8 @@ object NetworkClient {
|
|||
"-supershell=",
|
||||
"--color=",
|
||||
"-color=",
|
||||
"--autostart=",
|
||||
"-autostart=",
|
||||
)
|
||||
private[client] def parseArgs(args: Array[String]): Arguments = {
|
||||
val defaultSbtScript = if (Properties.isWin) "sbt.bat" else "sbt"
|
||||
|
|
@ -1346,7 +1351,7 @@ object NetworkClient {
|
|||
completionArguments.toSeq,
|
||||
sbtScript.getOrElse(defaultSbtScript).replace("%20", " "),
|
||||
bsp,
|
||||
launchJar
|
||||
launchJar,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,38 @@ object NetworkClientParseArgsTest extends BasicTestSuite:
|
|||
val result = parse("-bsp")
|
||||
assert(result.bsp)
|
||||
|
||||
// -- --no-server and --autostart= set sbt.server.autostart --
|
||||
|
||||
test("--no-server sets sbt.server.autostart=false"):
|
||||
try
|
||||
System.clearProperty("sbt.server.autostart")
|
||||
parse("--no-server", "compile")
|
||||
assert(System.getProperty("sbt.server.autostart") == "false")
|
||||
finally System.clearProperty("sbt.server.autostart")
|
||||
|
||||
test("-no-server sets sbt.server.autostart=false"):
|
||||
try
|
||||
System.clearProperty("sbt.server.autostart")
|
||||
parse("-no-server", "compile")
|
||||
assert(System.getProperty("sbt.server.autostart") == "false")
|
||||
finally System.clearProperty("sbt.server.autostart")
|
||||
|
||||
test("--autostart=false sets sbt.server.autostart=false"):
|
||||
try
|
||||
System.clearProperty("sbt.server.autostart")
|
||||
val result = parse("--autostart=false", "compile")
|
||||
assert(System.getProperty("sbt.server.autostart") == "false")
|
||||
assert(!result.sbtArguments.exists(_.contains("autostart")))
|
||||
assert(result.commandArguments.contains("compile"))
|
||||
finally System.clearProperty("sbt.server.autostart")
|
||||
|
||||
test("--autostart=true sets sbt.server.autostart=true"):
|
||||
try
|
||||
System.clearProperty("sbt.server.autostart")
|
||||
parse("--autostart=true", "compile")
|
||||
assert(System.getProperty("sbt.server.autostart") == "true")
|
||||
finally System.clearProperty("sbt.server.autostart")
|
||||
|
||||
test("--sbt-launch-jar is preserved"):
|
||||
val result = parse("--sbt-launch-jar", "/path/to/sbt-launch.jar", "compile")
|
||||
assert(result.sbtLaunchJar.contains("/path/to/sbt-launch.jar"))
|
||||
|
|
|
|||
|
|
@ -283,6 +283,8 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
|
|||
private[sbt] val cacheEventLog: CacheEventLog = CacheEventLog()
|
||||
private[sbt] val localDigestCacheByteSizeKey =
|
||||
SettingKey[Long](BasicKeys.localDigestCacheByteSize)
|
||||
private[sbt] val cacheVersionKey =
|
||||
SettingKey[Long](BasicKeys.cacheVersion)
|
||||
@cacheLevel(include = Array.empty)
|
||||
val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task {
|
||||
val state = stateKey.value
|
||||
|
|
@ -298,13 +300,15 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
|
|||
DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter)
|
||||
)
|
||||
val cacheByteSize = localDigestCacheByteSizeKey.value
|
||||
val cv = cacheVersionKey.value
|
||||
BuildWideCacheConfiguration(
|
||||
cacheStore,
|
||||
outputDirectory,
|
||||
fileConverter,
|
||||
state.log,
|
||||
cacheEventLog,
|
||||
cacheByteSize
|
||||
cacheByteSize,
|
||||
cv,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -460,6 +460,7 @@ object Keys {
|
|||
val allowZombieClassLoaders = settingKey[Boolean]("Allow a classloader that has previously been closed by `run` or `test` to continue loading classes.")
|
||||
val localCacheDirectory = settingKey[File]("Operating system specific cache directory.")
|
||||
val localDigestCacheByteSize = SettingKey[Long](BasicKeys.localDigestCacheByteSize).withRank(DSetting)
|
||||
val cacheVersion = SettingKey[Long](BasicKeys.cacheVersion).withRank(BSetting)
|
||||
val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting)
|
||||
val exportPipelining = settingKey[Boolean]("Produce early output so downstream subprojects can do pipelining.").withRank(BSetting)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ package sbt
|
|||
import java.io.{ File, IOException }
|
||||
import java.net.URI
|
||||
import java.nio.channels.ClosedChannelException
|
||||
import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files }
|
||||
import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files, Paths }
|
||||
import java.util.Properties
|
||||
import java.util.concurrent.ForkJoinPool
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
|
@ -32,7 +32,7 @@ import sbt.internal.util.complete.Parser
|
|||
import sbt.internal.util.{ RunningProcesses, Terminal as ITerminal, * }
|
||||
import sbt.io.*
|
||||
import sbt.io.syntax.*
|
||||
import sbt.util.{ Level, Logger, Show }
|
||||
import sbt.util.{ ActionCache, Level, Logger, Show }
|
||||
import xsbti.AppProvider
|
||||
|
||||
import scala.annotation.{ nowarn, tailrec }
|
||||
|
|
@ -246,17 +246,42 @@ object StandardMain {
|
|||
ConsoleOut.systemOutOverwrite(ConsoleOut.overwriteContaining("Resolving "))
|
||||
ConsoleOut.setGlobalProxy(console)
|
||||
|
||||
private def initialGlobalLogging(file: Option[File]): GlobalLogging = {
|
||||
def createTemp(attempt: Int = 0): File = Retry {
|
||||
private[sbt] def logLevelFromArguments(args: Seq[String]): Level.Value =
|
||||
val earlyCmd = BasicCommandStrings.EarlyCommand
|
||||
val levelOptions = Level.values.toSeq.flatMap: v =>
|
||||
List("-" + v.toString, "--" + v.toString)
|
||||
def levelFromArg(arg: String): Option[Level.Value] =
|
||||
if arg.startsWith(earlyCmd + "(") && arg.endsWith(")") then
|
||||
val inner = arg.slice(earlyCmd.length + 1, arg.length - 1).trim
|
||||
Level.values.find(_.toString == inner)
|
||||
else if levelOptions.contains(arg) then
|
||||
Level.values.find(v => arg == "-" + v.toString || arg == "--" + v.toString)
|
||||
else None
|
||||
args.flatMap(levelFromArg).headOption.getOrElse(Level.Info)
|
||||
|
||||
private def initialGlobalLogging(file: Option[File], initialLevel: Level.Value): GlobalLogging =
|
||||
def createTemp(prefix: String)(attempt: Int = 0): File = Retry:
|
||||
file.foreach(f => if (!f.exists()) IO.createDirectory(f))
|
||||
File.createTempFile("sbt-global-log", ".log", file.orNull)
|
||||
}
|
||||
File.createTempFile(prefix, ".log", file.orNull)
|
||||
|
||||
// Call this experimental since we don't want to commit to the log format for now
|
||||
val execLogProp = "sbt.experimental_execution_log"
|
||||
val execLog =
|
||||
if java.lang.Boolean.getBoolean(execLogProp) then
|
||||
Option(ActionCache.setExecLog(createTemp("exec-log")().toPath()))
|
||||
else sys.props.get(execLogProp).map(Paths.get(_)).map(ActionCache.setExecLog)
|
||||
execLog.foreach: log =>
|
||||
ShutdownHooks.add(() => log.close())
|
||||
GlobalLogging.initial(
|
||||
MainAppender.globalDefault(ConsoleOut.globalProxy),
|
||||
createTemp(),
|
||||
ConsoleOut.globalProxy
|
||||
createTemp("sbt-global-log")(),
|
||||
ConsoleOut.globalProxy,
|
||||
initialLevel
|
||||
)
|
||||
}
|
||||
|
||||
private def initialGlobalLogging(file: Option[File]): GlobalLogging =
|
||||
initialGlobalLogging(file, Level.Info)
|
||||
|
||||
def initialGlobalLogging(file: File): GlobalLogging = initialGlobalLogging(Option(file))
|
||||
@deprecated("use version that takes file argument", "1.4.0")
|
||||
def initialGlobalLogging: GlobalLogging = initialGlobalLogging(None)
|
||||
|
|
@ -265,8 +290,7 @@ object StandardMain {
|
|||
configuration: xsbti.AppConfiguration,
|
||||
initialDefinitions: Seq[Command],
|
||||
preCommands: Seq[String]
|
||||
): State = {
|
||||
// This is to workaround https://github.com/sbt/io/issues/110
|
||||
): State =
|
||||
if (!sys.props.contains("jna.nosys")) sys.props.put("jna.nosys", "true")
|
||||
|
||||
import BasicCommandStrings.{ DashDashDetachStdio, DashDashServer, isEarlyCommand }
|
||||
|
|
@ -274,11 +298,18 @@ object StandardMain {
|
|||
configuration.arguments
|
||||
.map(_.trim)
|
||||
.filterNot(c => c == DashDashDetachStdio || c == DashDashServer)
|
||||
.toSeq
|
||||
val initialLevel = logLevelFromArguments(userCommands)
|
||||
val (earlyCommands, normalCommands) = (preCommands ++ userCommands).partition(isEarlyCommand)
|
||||
val commands = (earlyCommands ++ normalCommands).toList map { x =>
|
||||
Exec(x, None)
|
||||
}
|
||||
val initAttrs = BuiltinCommands.initialAttributes
|
||||
val commands = (earlyCommands ++ normalCommands).toList.map(x => Exec(x, None))
|
||||
val baseAttrs = BuiltinCommands.initialAttributes
|
||||
val initAttrs =
|
||||
if initialLevel == Level.Info then baseAttrs
|
||||
else
|
||||
baseAttrs
|
||||
.put(Keys.logLevel.key, initialLevel)
|
||||
.put(BasicKeys.explicitGlobalLogLevels, true)
|
||||
val logDir = BuildPaths.globalLoggingStandard(configuration.baseDirectory)
|
||||
val s = State(
|
||||
configuration,
|
||||
initialDefinitions,
|
||||
|
|
@ -287,12 +318,12 @@ object StandardMain {
|
|||
commands,
|
||||
State.newHistory,
|
||||
initAttrs,
|
||||
initialGlobalLogging(BuildPaths.globalLoggingStandard(configuration.baseDirectory)),
|
||||
initialGlobalLogging(Option(logDir), initialLevel),
|
||||
None,
|
||||
State.Continue
|
||||
)
|
||||
s.initializeClassLoaderCache
|
||||
}
|
||||
end initialState
|
||||
}
|
||||
|
||||
import sbt.BasicCommandStrings.*
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ object RemoteCache:
|
|||
lazy val globalSettings: Seq[Def.Setting[?]] = Seq(
|
||||
localCacheDirectory :== defaultCacheLocation,
|
||||
localDigestCacheByteSize :== CacheImplicits.defaultLocalDigestCacheByteSize,
|
||||
cacheVersion :== sys.props.get("sbt.cacheversion").flatMap(_.toLongOption).getOrElse(0L),
|
||||
rootOutputDirectory := {
|
||||
appConfiguration.value.baseDirectory
|
||||
.toPath()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
package internal
|
||||
|
||||
import sbt.util.Level
|
||||
|
||||
object InitialLogLevelSpec extends verify.BasicTestSuite:
|
||||
|
||||
test("logLevelFromArguments returns Debug when -debug is in arguments"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq("-debug")) == Level.Debug)
|
||||
|
||||
test("logLevelFromArguments returns Debug when --debug is in arguments"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq("compile", "--debug")) == Level.Debug)
|
||||
|
||||
test("logLevelFromArguments returns Debug when early(debug) is in arguments"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq("early(debug)")) == Level.Debug)
|
||||
assert(StandardMain.logLevelFromArguments(Seq("compile", "early(debug)")) == Level.Debug)
|
||||
|
||||
test("logLevelFromArguments returns Info when no level option in arguments"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq()) == Level.Info)
|
||||
assert(StandardMain.logLevelFromArguments(Seq("compile", "run")) == Level.Info)
|
||||
|
||||
test("logLevelFromArguments uses first level option when multiple present"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq("-warn", "-debug")) == Level.Warn)
|
||||
assert(StandardMain.logLevelFromArguments(Seq("-error", "-info")) == Level.Error)
|
||||
|
||||
test("logLevelFromArguments supports all level options"):
|
||||
assert(StandardMain.logLevelFromArguments(Seq("-info")) == Level.Info)
|
||||
assert(StandardMain.logLevelFromArguments(Seq("--warn")) == Level.Warn)
|
||||
assert(StandardMain.logLevelFromArguments(Seq("--error")) == Level.Error)
|
||||
end InitialLogLevelSpec
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
## cacheVersion setting
|
||||
|
||||
sbt 2.x caches task results by default. `cacheVersion` provides an escape hatch to invalidate all caches when needed.
|
||||
|
||||
### Usage
|
||||
|
||||
In `build.sbt`:
|
||||
|
||||
```scala
|
||||
Global / cacheVersion := 1L
|
||||
```
|
||||
|
||||
Or via system property:
|
||||
|
||||
```
|
||||
sbt -Dsbt.cacheversion=1
|
||||
```
|
||||
|
||||
### Details
|
||||
|
||||
- Defaults to reading system property `sbt.cacheversion`, or else `0L`
|
||||
- When `cacheVersion` is `0L`, caching behaves identically to previous versions
|
||||
- Changing the value invalidates all task caches, forcing recomputation
|
||||
- The value is incorporated into every cache key via `BuildWideCacheConfiguration`
|
||||
|
|
@ -85,7 +85,7 @@ object DatatypeConfig {
|
|||
case "Map" | "Tuple2" | "scala.Tuple2" => { tpe =>
|
||||
twoArgs(tpe).flatMap(getFormats)
|
||||
}
|
||||
case "Int" | "Long" => { _ =>
|
||||
case "Int" | "Long" | "sbt.util.Digest" => { _ =>
|
||||
Nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
5
sbt
5
sbt
|
|
@ -668,6 +668,8 @@ Usage: `basename "$0"` [options]
|
|||
-d | --debug set sbt log level to debug
|
||||
-debug-inc | --debug-inc
|
||||
enable extra debugging for the incremental compiler
|
||||
--experimental_execution_log=true|<path>
|
||||
enable experimental execution log
|
||||
|
||||
# sbt version (default: from project/build.properties if present, else latest release)
|
||||
--sbt-version <version> use the specified version of sbt
|
||||
|
|
@ -721,8 +723,11 @@ map_args () {
|
|||
--supershell=*) options=( "${options[@]}" "-Dsbt.supershell=${1:13}" ) && shift ;;
|
||||
-supershell=*) options=( "${options[@]}" "-Dsbt.supershell=${1:12}" ) && shift ;;
|
||||
-no-server|--no-server) options=( "${options[@]}" "-Dsbt.io.virtual=false" "-Dsbt.server.autostart=false" ) && shift ;;
|
||||
--autostart=*) options=( "${options[@]}" "-Dsbt.server.autostart=${1:13}" ) && shift ;;
|
||||
-autostart=*) options=( "${options[@]}" "-Dsbt.server.autostart=${1:12}" ) && shift ;;
|
||||
--color=*) options=( "${options[@]}" "-Dsbt.color=${1:8}" ) && shift ;;
|
||||
-color=*) options=( "${options[@]}" "-Dsbt.color=${1:7}" ) && shift ;;
|
||||
--experimental_execution_log=*) options=( "${options[@]}" "-Dsbt.experimental_execution_log=${1:29}" ) && shift ;;
|
||||
-no-share|--no-share) options=( "${options[@]}" "${noshare_opts[@]}" ) && shift ;;
|
||||
-no-global|--no-global) options=( "${options[@]}" "-Dsbt.global.base=$(pwd)/project/.sbtboot" ) && shift ;;
|
||||
-ivy|--ivy) require_arg path "$1" "$2" && options=( "${options[@]}" "-Dsbt.ivy.home=$2" ) && shift 2 ;;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
scalaVersion := "3.7.4"
|
||||
|
||||
lazy val checkCounter = inputKey[Unit]("assert counter file value")
|
||||
|
||||
checkCounter := {
|
||||
val expected = complete.DefaultParsers.spaceDelimited("<arg>").parsed.head.toInt
|
||||
val f = baseDirectory.value / "target" / "counter.txt"
|
||||
val actual = if (f.exists()) IO.read(f).trim.toInt else 0
|
||||
assert(actual == expected, s"Expected counter=$expected but got $actual")
|
||||
}
|
||||
|
||||
lazy val bumpCounter = taskKey[Int]("cached task that bumps a counter file on each execution")
|
||||
|
||||
bumpCounter := {
|
||||
val f = baseDirectory.value / "target" / "counter.txt"
|
||||
val n = if (f.exists()) IO.read(f).trim.toInt else 0
|
||||
val next = n + 1
|
||||
IO.write(f, next.toString)
|
||||
next
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# bumpCounter is a cached task that increments a file counter on each execution.
|
||||
# When the cache is hit, the task body is NOT re-executed, so the counter stays the same.
|
||||
|
||||
# First call: body runs, counter goes to 1
|
||||
> bumpCounter
|
||||
> checkCounter 1
|
||||
|
||||
# Second call: cache hit, body does NOT run, counter stays at 1
|
||||
> bumpCounter
|
||||
> checkCounter 1
|
||||
|
||||
# Change cacheVersion: invalidates all caches
|
||||
> set Global / cacheVersion := 1L
|
||||
|
||||
# Third call: cache miss (version changed), body runs, counter goes to 2
|
||||
> bumpCounter
|
||||
> checkCounter 2
|
||||
|
||||
# Fourth call: cache hit again, counter stays at 2
|
||||
> bumpCounter
|
||||
> checkCounter 2
|
||||
|
|
@ -41,8 +41,12 @@ object ArgParser:
|
|||
opt[Int]("mem").action((x, c) => c.copy(mem = Some(x))),
|
||||
opt[String]("supershell").action((x, c) => c.copy(supershell = Some(x))),
|
||||
opt[String]("color").action((x, c) => c.copy(color = Some(x))),
|
||||
opt[String]("autostart").action((x, c) => c.copy(autostart = Some(x))),
|
||||
opt[Int]("jvm-debug").action((x, c) => c.copy(jvmDebug = Some(x))),
|
||||
opt[String]("java-home").action((x, c) => c.copy(javaHome = Some(x))),
|
||||
opt[String]("experimental_execution_log").action((x, c) =>
|
||||
c.copy(experimentalExecutionLog = Some(x))
|
||||
),
|
||||
arg[String]("<arg>")
|
||||
.unbounded()
|
||||
.optional()
|
||||
|
|
|
|||
|
|
@ -29,11 +29,13 @@ case class LauncherOptions(
|
|||
mem: Option[Int] = None,
|
||||
supershell: Option[String] = None,
|
||||
color: Option[String] = None,
|
||||
autostart: Option[String] = None,
|
||||
jvmDebug: Option[Int] = None,
|
||||
javaHome: Option[String] = None,
|
||||
server: Boolean = false,
|
||||
residual: Seq[String] = Nil,
|
||||
sbtNew: Boolean = false,
|
||||
experimentalExecutionLog: Option[String] = None,
|
||||
)
|
||||
|
||||
object LauncherOptions:
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ object Runner:
|
|||
opts.sbtCache.foreach(v => s = s :+ s"-Dsbt.global.localcache=$v")
|
||||
opts.ivy.foreach(v => s = s :+ s"-Dsbt.ivy.home=$v")
|
||||
opts.color.foreach(v => s = s :+ s"-Dsbt.color=$v")
|
||||
opts.autostart.foreach(v => s = s :+ s"-Dsbt.server.autostart=$v")
|
||||
opts.experimentalExecutionLog.foreach(v => s = s :+ s"-Dsbt.experimental_execution_log=$v")
|
||||
if opts.timings then
|
||||
s = s ++ Seq("-Dsbt.task.timings=true", "-Dsbt.task.timings.on.shutdown=true")
|
||||
if opts.traces then s = s :+ "-Dsbt.traces=true"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util
|
||||
final class SpawnExec private (
|
||||
val input: sbt.internal.util.SpawnInput,
|
||||
val cacheHit: Boolean,
|
||||
val exitCode: Option[Int],
|
||||
val outputs: Vector[xsbti.HashedVirtualFileRef]) extends Serializable {
|
||||
|
||||
private def this(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]) = this(input, cacheHit, exitCode, Vector())
|
||||
|
||||
override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
|
||||
case x: SpawnExec => (this.input == x.input) && (this.cacheHit == x.cacheHit) && (this.exitCode == x.exitCode) && (this.outputs == x.outputs)
|
||||
case _ => false
|
||||
})
|
||||
override def hashCode: Int = {
|
||||
37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnExec".##) + input.##) + cacheHit.##) + exitCode.##) + outputs.##)
|
||||
}
|
||||
override def toString: String = {
|
||||
"SpawnExec(" + input + ", " + cacheHit + ", " + exitCode + ", " + outputs + ")"
|
||||
}
|
||||
private def copy(input: sbt.internal.util.SpawnInput = input, cacheHit: Boolean = cacheHit, exitCode: Option[Int] = exitCode, outputs: Vector[xsbti.HashedVirtualFileRef] = outputs): SpawnExec = {
|
||||
new SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
}
|
||||
def withInput(input: sbt.internal.util.SpawnInput): SpawnExec = {
|
||||
copy(input = input)
|
||||
}
|
||||
def withCacheHit(cacheHit: Boolean): SpawnExec = {
|
||||
copy(cacheHit = cacheHit)
|
||||
}
|
||||
def withExitCode(exitCode: Option[Int]): SpawnExec = {
|
||||
copy(exitCode = exitCode)
|
||||
}
|
||||
def withExitCode(exitCode: Int): SpawnExec = {
|
||||
copy(exitCode = Option(exitCode))
|
||||
}
|
||||
def withOutputs(outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = {
|
||||
copy(outputs = outputs)
|
||||
}
|
||||
}
|
||||
object SpawnExec {
|
||||
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]): SpawnExec = new SpawnExec(input, cacheHit, exitCode)
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode))
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int], outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int, outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode), outputs)
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util
|
||||
final class SpawnInput private (
|
||||
val digest: sbt.util.Digest,
|
||||
val codeContentHash: sbt.util.Digest,
|
||||
val extraHash: sbt.util.Digest,
|
||||
val cacheVersion: Option[Long],
|
||||
val str: Option[String]) extends Serializable {
|
||||
|
||||
|
||||
|
||||
override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
|
||||
case x: SpawnInput => (this.digest == x.digest) && (this.codeContentHash == x.codeContentHash) && (this.extraHash == x.extraHash) && (this.cacheVersion == x.cacheVersion) && (this.str == x.str)
|
||||
case _ => false
|
||||
})
|
||||
override def hashCode: Int = {
|
||||
37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnInput".##) + digest.##) + codeContentHash.##) + extraHash.##) + cacheVersion.##) + str.##)
|
||||
}
|
||||
override def toString: String = {
|
||||
"SpawnInput(" + digest + ", " + codeContentHash + ", " + extraHash + ", " + cacheVersion + ", " + str + ")"
|
||||
}
|
||||
private def copy(digest: sbt.util.Digest = digest, codeContentHash: sbt.util.Digest = codeContentHash, extraHash: sbt.util.Digest = extraHash, cacheVersion: Option[Long] = cacheVersion, str: Option[String] = str): SpawnInput = {
|
||||
new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
}
|
||||
def withDigest(digest: sbt.util.Digest): SpawnInput = {
|
||||
copy(digest = digest)
|
||||
}
|
||||
def withCodeContentHash(codeContentHash: sbt.util.Digest): SpawnInput = {
|
||||
copy(codeContentHash = codeContentHash)
|
||||
}
|
||||
def withExtraHash(extraHash: sbt.util.Digest): SpawnInput = {
|
||||
copy(extraHash = extraHash)
|
||||
}
|
||||
def withCacheVersion(cacheVersion: Option[Long]): SpawnInput = {
|
||||
copy(cacheVersion = cacheVersion)
|
||||
}
|
||||
def withCacheVersion(cacheVersion: Long): SpawnInput = {
|
||||
copy(cacheVersion = Option(cacheVersion))
|
||||
}
|
||||
def withStr(str: Option[String]): SpawnInput = {
|
||||
copy(str = str)
|
||||
}
|
||||
def withStr(str: String): SpawnInput = {
|
||||
copy(str = Option(str))
|
||||
}
|
||||
}
|
||||
object SpawnInput {
|
||||
|
||||
def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Option[Long], str: Option[String]): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Long, str: String): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, Option(cacheVersion), Option(str))
|
||||
}
|
||||
11
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala
generated
Normal file
11
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala
generated
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
trait SpawnCodec extends sjsonnew.BasicJsonProtocol
|
||||
with sbt.internal.util.codec.SpawnInputFormats
|
||||
with sbt.internal.util.codec.HashedVirtualFileRefFormats
|
||||
with sbt.internal.util.codec.SpawnExecFormats
|
||||
object SpawnCodec extends SpawnCodec
|
||||
33
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala
generated
Normal file
33
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala
generated
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
|
||||
trait SpawnExecFormats { self: sbt.internal.util.codec.SpawnInputFormats & sjsonnew.BasicJsonProtocol & sbt.internal.util.codec.HashedVirtualFileRefFormats =>
|
||||
given SpawnExecFormat: JsonFormat[sbt.internal.util.SpawnExec] = new JsonFormat[sbt.internal.util.SpawnExec] {
|
||||
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnExec = {
|
||||
__jsOpt match {
|
||||
case Some(__js) =>
|
||||
unbuilder.beginObject(__js)
|
||||
val input = unbuilder.readField[sbt.internal.util.SpawnInput]("input")
|
||||
val cacheHit = unbuilder.readField[Boolean]("cacheHit")
|
||||
val exitCode = unbuilder.readField[Option[Int]]("exitCode")
|
||||
val outputs = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputs")
|
||||
unbuilder.endObject()
|
||||
sbt.internal.util.SpawnExec(input, cacheHit, exitCode, outputs)
|
||||
case None =>
|
||||
deserializationError("Expected JsObject but found None")
|
||||
}
|
||||
}
|
||||
override def write[J](obj: sbt.internal.util.SpawnExec, builder: Builder[J]): Unit = {
|
||||
builder.beginObject()
|
||||
builder.addField("input", obj.input)
|
||||
builder.addField("cacheHit", obj.cacheHit)
|
||||
builder.addField("exitCode", obj.exitCode)
|
||||
builder.addField("outputs", obj.outputs)
|
||||
builder.endObject()
|
||||
}
|
||||
}
|
||||
}
|
||||
35
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala
generated
Normal file
35
util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala
generated
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* This code is generated using [[https://www.scala-sbt.org/contraband]].
|
||||
*/
|
||||
|
||||
// DO NOT EDIT MANUALLY
|
||||
package sbt.internal.util.codec
|
||||
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
|
||||
trait SpawnInputFormats { self: sjsonnew.BasicJsonProtocol =>
|
||||
given SpawnInputFormat: JsonFormat[sbt.internal.util.SpawnInput] = new JsonFormat[sbt.internal.util.SpawnInput] {
|
||||
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnInput = {
|
||||
__jsOpt match {
|
||||
case Some(__js) =>
|
||||
unbuilder.beginObject(__js)
|
||||
val digest = unbuilder.readField[sbt.util.Digest]("digest")
|
||||
val codeContentHash = unbuilder.readField[sbt.util.Digest]("codeContentHash")
|
||||
val extraHash = unbuilder.readField[sbt.util.Digest]("extraHash")
|
||||
val cacheVersion = unbuilder.readField[Option[Long]]("cacheVersion")
|
||||
val str = unbuilder.readField[Option[String]]("str")
|
||||
unbuilder.endObject()
|
||||
sbt.internal.util.SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str)
|
||||
case None =>
|
||||
deserializationError("Expected JsObject but found None")
|
||||
}
|
||||
}
|
||||
override def write[J](obj: sbt.internal.util.SpawnInput, builder: Builder[J]): Unit = {
|
||||
builder.beginObject()
|
||||
builder.addField("digest", obj.digest)
|
||||
builder.addField("codeContentHash", obj.codeContentHash)
|
||||
builder.addField("extraHash", obj.extraHash)
|
||||
builder.addField("cacheVersion", obj.cacheVersion)
|
||||
builder.addField("str", obj.str)
|
||||
builder.endObject()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package sbt.internal.util
|
||||
@target(Scala)
|
||||
@codecPackage("sbt.internal.util.codec")
|
||||
@fullCodec("SpawnCodec")
|
||||
|
||||
# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto
|
||||
type SpawnInput {
|
||||
digest: sbt.util.Digest!
|
||||
codeContentHash: sbt.util.Digest!
|
||||
extraHash: sbt.util.Digest!
|
||||
cacheVersion: Long
|
||||
str: String
|
||||
}
|
||||
|
||||
# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto
|
||||
type SpawnExec {
|
||||
input: sbt.internal.util.SpawnInput!
|
||||
cacheHit: Boolean!
|
||||
exitCode: Int
|
||||
outputs: [xsbti.HashedVirtualFileRef] @since("0.2.0")
|
||||
}
|
||||
|
|
@ -8,10 +8,16 @@
|
|||
|
||||
package sbt.util
|
||||
|
||||
import java.io.{ File, IOException }
|
||||
import java.io.{ File, IOException, PrintWriter }
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.{ Files, Path, Paths, StandardCopyOption }
|
||||
import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 }
|
||||
import sbt.internal.util.{
|
||||
ActionCacheEvent,
|
||||
CacheEventLog,
|
||||
SpawnExec,
|
||||
SpawnInput,
|
||||
StringVirtualFile1
|
||||
}
|
||||
import sbt.io.syntax.*
|
||||
import sbt.io.IO
|
||||
import sbt.nio.file.{ **, FileTreeView }
|
||||
|
|
@ -19,10 +25,10 @@ import sbt.nio.file.syntax.*
|
|||
import sbt.util.CacheImplicits
|
||||
import scala.reflect.ClassTag
|
||||
import scala.annotation.{ meta, StaticAnnotation }
|
||||
import scala.util.control.{ Exception, NonFatal }
|
||||
import scala.util.control.NonFatal
|
||||
import sjsonnew.{ HashWriter, JsonFormat }
|
||||
import sjsonnew.support.murmurhash.Hasher
|
||||
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
|
||||
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter }
|
||||
import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
|
||||
import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
|
||||
|
||||
|
|
@ -31,6 +37,11 @@ object ActionCache:
|
|||
private[sbt] val manifestFileName = "sbtdir_manifest.json"
|
||||
private[sbt] val failureFileName = "failure.json"
|
||||
private[sbt] val failureExitCode = 1
|
||||
private[sbt] var execLog: Option[PrintWriter] = None
|
||||
private[sbt] def setExecLog(log: Path): PrintWriter =
|
||||
val writer = PrintWriter(log.toFile, "UTF-8")
|
||||
execLog = Option(writer)
|
||||
writer
|
||||
|
||||
/**
|
||||
* This is a key function that drives remote caching.
|
||||
|
|
@ -41,9 +52,10 @@ object ActionCache:
|
|||
* all we need from the input is to generate some hash value.
|
||||
* - codeContentHash: This hash represents the Scala code of the task.
|
||||
* Even if the input tasks are the same, the code part needs to be tracked.
|
||||
* - extraHash: Reserved for later, which we might use to invalidate the cache.
|
||||
* - extraHash: Extra hash for cache invalidation (combined with config.cacheVersion).
|
||||
* - tags: Tags to track cache level.
|
||||
* - config: The configuration that's used to store where the cache backends are.
|
||||
* config.cacheVersion is incorporated into the cache key to allow global invalidation.
|
||||
* - action: The actual action to be cached.
|
||||
*/
|
||||
def cache[I: HashWriter, O: JsonFormat](
|
||||
|
|
@ -57,17 +69,19 @@ object ActionCache:
|
|||
): O =
|
||||
import config.*
|
||||
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, cacheVersion)
|
||||
|
||||
def cacheFailure(e: CompileFailed): Nothing =
|
||||
// Cache the failure so subsequent builds don't re-run failed compilation
|
||||
// This fixes https://github.com/sbt/sbt/issues/7662
|
||||
// Use the same input digest as success, distinguished by exitCode
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
|
||||
val cachedFailure = CachedCompileFailure.fromException(e)
|
||||
val json = Converter.toJsonUnsafe(cachedFailure)
|
||||
val valuePath = mkValuePath(inputDigest)
|
||||
val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
store.put(
|
||||
UpdateActionResultRequest(input, Vector(failureFile), exitCode = failureExitCode)
|
||||
UpdateActionResultRequest(inputDigest, Vector(failureFile), exitCode = failureExitCode)
|
||||
)
|
||||
throw e
|
||||
|
||||
|
|
@ -80,49 +94,82 @@ object ActionCache:
|
|||
cacheFailure(e)
|
||||
case e: Exception =>
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logExec(
|
||||
SpawnExec(input = spawnInput, cacheHit = false, exitCode = 1)
|
||||
)
|
||||
throw e
|
||||
val json = Converter.toJsonUnsafe(result)
|
||||
val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize()
|
||||
val uncacheableOutputs =
|
||||
outputs.filter(f =>
|
||||
f match
|
||||
case vf if vf.id.endsWith(ActionCache.dirZipExt) =>
|
||||
false
|
||||
case _ =>
|
||||
val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize()
|
||||
!outputPath.startsWith(normalizedOutputDir)
|
||||
)
|
||||
if uncacheableOutputs.nonEmpty then
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logger.error(
|
||||
s"Cannot cache task because its output files are outside the output directory: \n" +
|
||||
uncacheableOutputs.mkString(" - ", "\n - ", "")
|
||||
)
|
||||
result
|
||||
else
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
|
||||
val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
val newOutputs = Vector(valueFile) ++ outputs.toVector
|
||||
try
|
||||
store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match
|
||||
try
|
||||
val json = Converter.toJsonUnsafe(result)
|
||||
val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize()
|
||||
val uncacheableOutputs =
|
||||
outputs.filter(f =>
|
||||
f match
|
||||
case vf if vf.id.endsWith(ActionCache.dirZipExt) =>
|
||||
false
|
||||
case _ =>
|
||||
val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize()
|
||||
!outputPath.startsWith(normalizedOutputDir)
|
||||
)
|
||||
if uncacheableOutputs.nonEmpty then
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
logger.error(
|
||||
s"Cannot cache task because its output files are outside the output directory: \n" +
|
||||
uncacheableOutputs.mkString(" - ", "\n - ", "")
|
||||
)
|
||||
result
|
||||
else
|
||||
cacheEventLog.append(ActionCacheEvent.OnsiteTask)
|
||||
val valuePath = mkValuePath(inputDigest)
|
||||
val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json))
|
||||
val newOutputs = Vector(valueFile) ++ outputs.toVector
|
||||
store.put(UpdateActionResultRequest(inputDigest, newOutputs, exitCode = 0)) match
|
||||
case Right(cachedResult) =>
|
||||
store.syncBlobs(cachedResult.outputFiles, outputDirectory)
|
||||
logExec(
|
||||
SpawnExec(
|
||||
input = spawnInput,
|
||||
cacheHit = false,
|
||||
exitCode = 0,
|
||||
outputs = cachedResult.outputFiles,
|
||||
)
|
||||
)
|
||||
result
|
||||
case Left(e) => throw e
|
||||
catch
|
||||
case e: IOException =>
|
||||
logger.debug(s"Skipping cache storage due to error: ${e.getMessage}")
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
result
|
||||
catch
|
||||
case e: IOException =>
|
||||
logger.debug(s"Skipping cache storage due to error: ${e.getMessage}")
|
||||
cacheEventLog.append(ActionCacheEvent.Error)
|
||||
result
|
||||
|
||||
def spawnInput = SpawnInput(
|
||||
digest = inputDigest,
|
||||
codeContentHash = codeContentHash,
|
||||
extraHash = extraHash,
|
||||
cacheVersion = if cacheVersion != 0 then Some(cacheVersion) else None,
|
||||
str = Some(key.toString()),
|
||||
)
|
||||
inline def logExec(inline event: SpawnExec): Unit =
|
||||
execLog.foreach: log =>
|
||||
logEvent(event, log)
|
||||
// Single cache lookup - use exitCode to distinguish success from failure
|
||||
getWithFailure(key, codeContentHash, extraHash, tags, config) match
|
||||
case Right(value) => value
|
||||
getWithFailure(inputDigest, tags, config) match
|
||||
case Right((value, result)) =>
|
||||
logExec(
|
||||
SpawnExec(
|
||||
input = spawnInput,
|
||||
cacheHit = true,
|
||||
exitCode = result.exitCode,
|
||||
outputs = result.outputFiles,
|
||||
)
|
||||
)
|
||||
value
|
||||
case Left(Some(failure)) =>
|
||||
config.cacheEventLog.append(ActionCacheEvent.Found("cached-failure"))
|
||||
// Replay problems to the logger so users see the cached errors/warnings
|
||||
failure.replay(config.logger)
|
||||
logExec(
|
||||
SpawnExec(input = spawnInput, cacheHit = true, exitCode = 1)
|
||||
)
|
||||
throw failure.toException
|
||||
case Left(None) => organicTask
|
||||
end cache
|
||||
|
|
@ -132,13 +179,11 @@ object ActionCache:
|
|||
* Returns Right(value) for cached success, Left(Some(failure)) for cached failure,
|
||||
* or Left(None) for cache miss.
|
||||
*/
|
||||
private def getWithFailure[I: HashWriter, O: JsonFormat](
|
||||
key: I,
|
||||
codeContentHash: Digest,
|
||||
extraHash: Digest,
|
||||
private def getWithFailure[O: JsonFormat](
|
||||
inputDigest: Digest,
|
||||
tags: List[CacheLevelTag],
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Either[Option[CachedCompileFailure], O] =
|
||||
): Either[Option[CachedCompileFailure], (O, ActionResult)] =
|
||||
import config.store
|
||||
def valueFromStr(str: String, origin: Option[String]): O =
|
||||
config.cacheEventLog.append(ActionCacheEvent.Found(origin.getOrElse("unknown")))
|
||||
|
|
@ -151,59 +196,36 @@ object ActionCache:
|
|||
|
||||
def parseCachedValue(
|
||||
str: String,
|
||||
origin: Option[String],
|
||||
result: ActionResult,
|
||||
isFailure: Boolean,
|
||||
): Option[Either[Option[CachedCompileFailure], O]] =
|
||||
): Option[Either[Option[CachedCompileFailure], (O, ActionResult)]] =
|
||||
try
|
||||
if isFailure then Some(Left(Some(failureFromStr(str))))
|
||||
else Some(Right(valueFromStr(str, origin)))
|
||||
else Some(Right((valueFromStr(str, result.origin), result)))
|
||||
catch case _: Exception => None
|
||||
|
||||
// Optimization: Check if we can read directly from symlinked value file
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
|
||||
val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath))
|
||||
|
||||
def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] =
|
||||
if java.nio.file.Files.isSymbolicLink(resolvedValuePath) && java.nio.file.Files
|
||||
.exists(resolvedValuePath)
|
||||
then
|
||||
Exception.nonFatalCatch
|
||||
.opt(IO.read(resolvedValuePath.toFile(), StandardCharsets.UTF_8))
|
||||
.flatMap: str =>
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode))
|
||||
catch case NonFatal(_) => None
|
||||
case Left(_) => None
|
||||
else None
|
||||
|
||||
readFromSymlink() match
|
||||
case Some(result) => result
|
||||
case None =>
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
val isFailure = result.exitCode.contains(failureExitCode)
|
||||
result.contents.headOption match
|
||||
case Some(head) =>
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
val str = String(head.array(), StandardCharsets.UTF_8)
|
||||
parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None))
|
||||
case _ =>
|
||||
val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
if paths.isEmpty then Left(None)
|
||||
else
|
||||
val str = IO.read(paths.head.toFile())
|
||||
parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None))
|
||||
catch
|
||||
case NonFatal(e) =>
|
||||
config.logger.debug(
|
||||
s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}"
|
||||
)
|
||||
Left(None)
|
||||
case Left(_) => Left(None)
|
||||
findActionResult(inputDigest, config) match
|
||||
case Right(result) =>
|
||||
try
|
||||
val isFailure = result.exitCode.contains(failureExitCode)
|
||||
result.contents.headOption match
|
||||
case Some(head) =>
|
||||
store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
val str = String(head.array(), StandardCharsets.UTF_8)
|
||||
parseCachedValue(str, result, isFailure).getOrElse(Left(None))
|
||||
case _ =>
|
||||
val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
|
||||
if paths.isEmpty then Left(None)
|
||||
else
|
||||
val str = IO.read(paths.head.toFile())
|
||||
parseCachedValue(str, result, isFailure).getOrElse(Left(None))
|
||||
catch
|
||||
case NonFatal(e) =>
|
||||
config.logger.debug(
|
||||
s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}"
|
||||
)
|
||||
Left(None)
|
||||
case Left(_) => Left(None)
|
||||
|
||||
/**
|
||||
* Retrieves the cached value.
|
||||
|
|
@ -215,8 +237,9 @@ object ActionCache:
|
|||
tags: List[CacheLevelTag],
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Option[O] =
|
||||
getWithFailure(key, codeContentHash, extraHash, tags, config) match
|
||||
case Right(value) => Some(value)
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
getWithFailure(inputDigest, tags, config) match
|
||||
case Right(value) => Some(value._1)
|
||||
case Left(_) => None
|
||||
|
||||
/**
|
||||
|
|
@ -228,10 +251,26 @@ object ActionCache:
|
|||
extraHash: Digest,
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Boolean =
|
||||
findActionResult(key, codeContentHash, extraHash, config) match
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
findActionResult(inputDigest, config) match
|
||||
case Right(_) => true
|
||||
case Left(_) => false
|
||||
|
||||
inline private[sbt] def findActionResult(
|
||||
inputDigest: Digest,
|
||||
config: BuildWideCacheConfiguration,
|
||||
): Either[Throwable, ActionResult] =
|
||||
// val logger = config.logger
|
||||
CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
|
||||
val getRequest =
|
||||
GetActionResultRequest(
|
||||
inputDigest,
|
||||
inlineStdout = false,
|
||||
inlineStderr = false,
|
||||
Vector(mkValuePath(inputDigest))
|
||||
)
|
||||
config.store.get(getRequest)
|
||||
|
||||
inline private[sbt] def findActionResult[I: HashWriter](
|
||||
key: I,
|
||||
codeContentHash: Digest,
|
||||
|
|
@ -240,19 +279,35 @@ object ActionCache:
|
|||
): Either[Throwable, ActionResult] =
|
||||
// val logger = config.logger
|
||||
CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
|
||||
val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
|
||||
val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion)
|
||||
val getRequest =
|
||||
GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
|
||||
GetActionResultRequest(
|
||||
inputDigest,
|
||||
inlineStdout = false,
|
||||
inlineStderr = false,
|
||||
Vector(mkValuePath(inputDigest))
|
||||
)
|
||||
config.store.get(getRequest)
|
||||
|
||||
private inline def mkInput[I: HashWriter](
|
||||
key: I,
|
||||
codeContentHash: Digest,
|
||||
extraHash: Digest
|
||||
): (Digest, String) =
|
||||
val input =
|
||||
Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key)))
|
||||
(input, s"$${OUT}/value/$input.json")
|
||||
extraHash: Digest,
|
||||
cacheVersion: Long,
|
||||
): Digest =
|
||||
Digest.sha256Hash(
|
||||
(Vector(
|
||||
codeContentHash,
|
||||
Digest.dummy(Hasher.hashUnsafe[I](key)),
|
||||
extraHash
|
||||
) ++ {
|
||||
if cacheVersion == 0 then Vector.empty
|
||||
else Vector(Digest.dummy(cacheVersion))
|
||||
})*
|
||||
)
|
||||
|
||||
private inline def mkValuePath(inputDigest: Digest): String =
|
||||
s"$${OUT}/value/${inputDigest}.json"
|
||||
|
||||
def manifestFromFile(manifest: Path): Manifest =
|
||||
import sbt.internal.util.codec.ManifestCodec.given
|
||||
|
|
@ -323,6 +378,14 @@ object ActionCache:
|
|||
private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] =
|
||||
Some(r.value, r.outputs)
|
||||
end InternalActionResult
|
||||
|
||||
private[sbt] def logEvent(event: SpawnExec, log: PrintWriter): Unit =
|
||||
import sbt.internal.util.codec.SpawnCodec.given
|
||||
val json = Converter.toJsonUnsafe(event)
|
||||
val s = PrettyPrinter(json)
|
||||
log.println(s)
|
||||
log.flush()
|
||||
|
||||
end ActionCache
|
||||
|
||||
class BuildWideCacheConfiguration(
|
||||
|
|
@ -332,13 +395,14 @@ class BuildWideCacheConfiguration(
|
|||
val logger: Logger,
|
||||
val cacheEventLog: CacheEventLog,
|
||||
val localDigestCacheByteSize: Long,
|
||||
val cacheVersion: Long,
|
||||
):
|
||||
def this(
|
||||
store: ActionCacheStore,
|
||||
outputDirectory: Path,
|
||||
fileConverter: FileConverter,
|
||||
logger: Logger,
|
||||
cacheEventLog: CacheEventLog
|
||||
cacheEventLog: CacheEventLog,
|
||||
) =
|
||||
this(
|
||||
store,
|
||||
|
|
@ -346,7 +410,26 @@ class BuildWideCacheConfiguration(
|
|||
fileConverter,
|
||||
logger,
|
||||
cacheEventLog,
|
||||
CacheImplicits.defaultLocalDigestCacheByteSize
|
||||
CacheImplicits.defaultLocalDigestCacheByteSize,
|
||||
0L,
|
||||
)
|
||||
|
||||
def this(
|
||||
store: ActionCacheStore,
|
||||
outputDirectory: Path,
|
||||
fileConverter: FileConverter,
|
||||
logger: Logger,
|
||||
cacheEventLog: CacheEventLog,
|
||||
localDigestCacheByteSize: Long,
|
||||
) =
|
||||
this(
|
||||
store,
|
||||
outputDirectory,
|
||||
fileConverter,
|
||||
logger,
|
||||
cacheEventLog,
|
||||
localDigestCacheByteSize,
|
||||
0L,
|
||||
)
|
||||
|
||||
override def toString(): String =
|
||||
|
|
|
|||
|
|
@ -83,7 +83,8 @@ end AbstractActionCacheStore
|
|||
/**
|
||||
* An aggregate ActionCacheStore.
|
||||
*/
|
||||
class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) extends AbstractActionCacheStore:
|
||||
case class AggregateActionCacheStore(stores: Seq[ActionCacheStore])
|
||||
extends AbstractActionCacheStore:
|
||||
extension [A1](xs: Seq[A1])
|
||||
// unlike collectFirst this accepts A1 => Seq[A2]
|
||||
inline def collectFirst2[A2](f: A1 => Seq[A2], size: Int): Seq[A2] =
|
||||
|
|
@ -176,7 +177,8 @@ class InMemoryActionCacheStore extends AbstractActionCacheStore:
|
|||
underlying.toString()
|
||||
end InMemoryActionCacheStore
|
||||
|
||||
class DiskActionCacheStore(base: Path, converter: FileConverter) extends AbstractActionCacheStore:
|
||||
case class DiskActionCacheStore(base: Path, converter: FileConverter)
|
||||
extends AbstractActionCacheStore:
|
||||
lazy val casBase: Path = {
|
||||
val dir = base.resolve("cas")
|
||||
IO.createDirectory(dir.toFile)
|
||||
|
|
|
|||
|
|
@ -298,6 +298,36 @@ object ActionCacheTest extends BasicTestSuite:
|
|||
assert(v2 == 42)
|
||||
assert(called == 2)
|
||||
|
||||
test("Changing cacheVersion invalidates the cache"):
|
||||
withDiskCache(testCacheVersionInvalidation)
|
||||
|
||||
def testCacheVersionInvalidation(cache: ActionCacheStore): Unit =
|
||||
import sjsonnew.BasicJsonProtocol.*
|
||||
var called = 0
|
||||
val action: ((Int, Int)) => InternalActionResult[Int] = { (a, b) =>
|
||||
called += 1
|
||||
InternalActionResult(a + b, Nil)
|
||||
}
|
||||
IO.withTemporaryDirectory: tempDir =>
|
||||
val config0 = getCacheConfig(cache, tempDir)
|
||||
// First call: computes the result
|
||||
val v1 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action)
|
||||
assert(v1 == 2)
|
||||
assert(called == 1)
|
||||
// Second call with same config: hits cache
|
||||
val v2 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action)
|
||||
assert(v2 == 2)
|
||||
assert(called == 1)
|
||||
// Third call with different cacheVersion: cache miss, recomputes
|
||||
val config1 = getCacheConfig(cache, tempDir, cacheVersion = 1L)
|
||||
val v3 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action)
|
||||
assert(v3 == 2)
|
||||
assert(called == 2)
|
||||
// Fourth call with same cacheVersion=1: hits cache again
|
||||
val v4 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action)
|
||||
assert(v4 == 2)
|
||||
assert(called == 2)
|
||||
|
||||
def withInMemoryCache(f: InMemoryActionCacheStore => Unit): Unit =
|
||||
val cache = InMemoryActionCacheStore()
|
||||
f(cache)
|
||||
|
|
@ -312,12 +342,24 @@ object ActionCacheTest extends BasicTestSuite:
|
|||
keepDirectory = false
|
||||
)
|
||||
|
||||
def getCacheConfig(cache: ActionCacheStore, outputDir: File): BuildWideCacheConfiguration =
|
||||
def getCacheConfig(
|
||||
cache: ActionCacheStore,
|
||||
outputDir: File,
|
||||
cacheVersion: Long = 0L,
|
||||
): BuildWideCacheConfiguration =
|
||||
val logger = new Logger:
|
||||
override def trace(t: => Throwable): Unit = ()
|
||||
override def success(message: => String): Unit = ()
|
||||
override def log(level: Level.Value, message: => String): Unit = ()
|
||||
BuildWideCacheConfiguration(cache, outputDir.toPath(), fileConverter, logger, CacheEventLog())
|
||||
BuildWideCacheConfiguration(
|
||||
cache,
|
||||
outputDir.toPath(),
|
||||
fileConverter,
|
||||
logger,
|
||||
CacheEventLog(),
|
||||
CacheImplicits.defaultLocalDigestCacheByteSize,
|
||||
cacheVersion,
|
||||
)
|
||||
|
||||
def fileConverter = new FileConverter:
|
||||
override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id)
|
||||
|
|
|
|||
Loading…
Reference in New Issue