diff --git a/internal/util-logging/src/main/scala/sbt/internal/util/GlobalLogging.scala b/internal/util-logging/src/main/scala/sbt/internal/util/GlobalLogging.scala index 1c09acd23..3d737a071 100644 --- a/internal/util-logging/src/main/scala/sbt/internal/util/GlobalLogging.scala +++ b/internal/util-logging/src/main/scala/sbt/internal/util/GlobalLogging.scala @@ -84,11 +84,18 @@ object GlobalLogging { newAppender: (ManagedLogger, PrintWriter, GlobalLogBacking, LoggerContext) => GlobalLogging, newBackingFile: => File, console: ConsoleOut - ): GlobalLogging = { + ): GlobalLogging = + initial(newAppender, newBackingFile, console, Level.Info) + + def initial( + newAppender: (ManagedLogger, PrintWriter, GlobalLogBacking, LoggerContext) => GlobalLogging, + newBackingFile: => File, + console: ConsoleOut, + initialLevel: Level.Value, + ): GlobalLogging = val loggerName = generateName val log = LoggerContext.globalContext.logger(loggerName, None, None) val appender = ConsoleAppender(ConsoleAppender.generateName(), console) - LoggerContext.globalContext.addAppender(loggerName, appender -> Level.Info) + LoggerContext.globalContext.addAppender(loggerName, appender -> initialLevel) GlobalLogging(log, console, appender, GlobalLogBacking(newBackingFile), newAppender) - } } diff --git a/launcher-package/integration-test/src/test/scala/RunnerScriptTest.scala b/launcher-package/integration-test/src/test/scala/RunnerScriptTest.scala index dfc728f84..3abe411df 100644 --- a/launcher-package/integration-test/src/test/scala/RunnerScriptTest.scala +++ b/launcher-package/integration-test/src/test/scala/RunnerScriptTest.scala @@ -330,4 +330,8 @@ object RunnerScriptTest extends verify.BasicTestSuite with ShellScriptUtil: s"Should not have errors from comment character, but found: ${errorMessages.mkString(", ")}" ) + testOutput("sbt --experimental_execution_log=true")("--experimental_execution_log=true", "-v"): + (out: List[String]) => + assert(out.contains[String]("-Dsbt.experimental_execution_log=true")) + end RunnerScriptTest diff --git a/launcher-package/src/universal/bin/sbt.bat b/launcher-package/src/universal/bin/sbt.bat index 323b263a8..8cb79e2ef 100755 --- a/launcher-package/src/universal/bin/sbt.bat +++ b/launcher-package/src/universal/bin/sbt.bat @@ -54,6 +54,7 @@ set sbt_args_mem= set sbt_args_client= set sbt_args_jvm_client= set sbt_args_no_server= +set sbt_args_experimental_execution_log= set is_this_dir_sbt=0 rem users can set SBT_OPTS via .sbtopts @@ -419,6 +420,21 @@ if defined _color_arg ( goto args_loop ) +if "%~0" == "--experimental_execution_log" set _experimental_execution_log_arg=true + +if defined _experimental_execution_log_arg ( + set _experimental_execution_log_arg= + if not "%~1" == "" ( + set sbt_args_experimental_execution_log=%~1 + shift + goto args_loop + ) else ( + echo "%~0" is missing a value + goto error + ) + goto args_loop +) + if "%~0" == "--no-share" set _no_share_arg=true if "%~0" == "-no-share" set _no_share_arg=true @@ -724,6 +740,10 @@ if not defined sbt_args_no_hide_jdk_warnings ( ) ) +if defined sbt_args_experimental_execution_log ( + set _SBT_OPTS=-Dsbt.experimental_execution_log=!sbt_args_experimental_execution_log! !_SBT_OPTS! +) + rem TODO: _SBT_OPTS needs to be processed as args and diffed against SBT_ARGS if !sbt_args_print_sbt_version! equ 1 ( @@ -1137,6 +1157,8 @@ echo --script-version print the version of sbt script echo -d ^| --debug set sbt log level to debug echo -debug-inc ^| --debug-inc echo enable extra debugging for the incremental compiler +echo --experimental_execution_log=true^|^ +echo enable experimental execution log rem echo -J-X pass option -X directly to the java runtime rem echo ^(-J is stripped^) rem echo -S-X add -X to sbt's scalacOptions ^(-S is stripped^) diff --git a/main-command/src/main/scala/sbt/BasicKeys.scala b/main-command/src/main/scala/sbt/BasicKeys.scala index 5b4decfff..d2a6972ea 100644 --- a/main-command/src/main/scala/sbt/BasicKeys.scala +++ b/main-command/src/main/scala/sbt/BasicKeys.scala @@ -129,6 +129,12 @@ object BasicKeys { 10000 ) + val cacheVersion = AttributeKey[Long]( + "cacheVersion", + "A version number that invalidates all task caches when changed.", + 10000 + ) + // Unlike other BasicKeys, this is not used directly as a setting key, // and severLog / logLevel is used instead. private[sbt] val serverLogLevel = diff --git a/main-command/src/main/scala/sbt/internal/client/NetworkClient.scala b/main-command/src/main/scala/sbt/internal/client/NetworkClient.scala index b0eb7e99c..694dd5c80 100644 --- a/main-command/src/main/scala/sbt/internal/client/NetworkClient.scala +++ b/main-command/src/main/scala/sbt/internal/client/NetworkClient.scala @@ -151,6 +151,8 @@ class NetworkClient( private lazy val noStdErr = arguments.completionArguments.contains("--no-stderr") && !sys.env.contains("SBTN_AUTO_COMPLETE") && !sys.env.contains("SBTC_AUTO_COMPLETE") private def shutdownOnly = arguments.commandArguments == Seq(Shutdown) + private lazy val serverAutoStart: Boolean = + sys.props.get("sbt.server.autostart").forall(_.toLowerCase == "true") private def mkSocket(file: File): (Socket, Option[String]) = ClientSocket.socket(file, useJNI) @@ -190,6 +192,9 @@ class NetworkClient( if (shutdownOnly) { console.appendLog(Level.Info, "no sbt server is running. ciao") System.exit(0) + } else if (!serverAutoStart) { + console.appendLog(Level.Error, "no sbt server is running (sbt.server.autostart=false)") + System.exit(1) } else if (promptCompleteUsers) { val msg = if (noTab) "" else "No sbt server is running. Press to start one..." errorStream.print(s"\n$msg") @@ -1198,7 +1203,7 @@ object NetworkClient { completionArguments, sbtScript, bsp, - sbtLaunchJar + sbtLaunchJar, ) } private[client] val completions = "--completions" @@ -1257,8 +1262,6 @@ object NetworkClient { "--timings", "-traces", "--traces", - "-no-server", - "--no-server", "-no-share", "--no-share", "-no-global", @@ -1275,6 +1278,8 @@ object NetworkClient { "-supershell=", "--color=", "-color=", + "--autostart=", + "-autostart=", ) private[client] def parseArgs(args: Array[String]): Arguments = { val defaultSbtScript = if (Properties.isWin) "sbt.bat" else "sbt" @@ -1346,7 +1351,7 @@ object NetworkClient { completionArguments.toSeq, sbtScript.getOrElse(defaultSbtScript).replace("%20", " "), bsp, - launchJar + launchJar, ) } diff --git a/main-command/src/test/scala/sbt/internal/client/NetworkClientParseArgsTest.scala b/main-command/src/test/scala/sbt/internal/client/NetworkClientParseArgsTest.scala index 40f174e21..67ef43217 100644 --- a/main-command/src/test/scala/sbt/internal/client/NetworkClientParseArgsTest.scala +++ b/main-command/src/test/scala/sbt/internal/client/NetworkClientParseArgsTest.scala @@ -111,6 +111,38 @@ object NetworkClientParseArgsTest extends BasicTestSuite: val result = parse("-bsp") assert(result.bsp) + // -- --no-server and --autostart= set sbt.server.autostart -- + + test("--no-server sets sbt.server.autostart=false"): + try + System.clearProperty("sbt.server.autostart") + parse("--no-server", "compile") + assert(System.getProperty("sbt.server.autostart") == "false") + finally System.clearProperty("sbt.server.autostart") + + test("-no-server sets sbt.server.autostart=false"): + try + System.clearProperty("sbt.server.autostart") + parse("-no-server", "compile") + assert(System.getProperty("sbt.server.autostart") == "false") + finally System.clearProperty("sbt.server.autostart") + + test("--autostart=false sets sbt.server.autostart=false"): + try + System.clearProperty("sbt.server.autostart") + val result = parse("--autostart=false", "compile") + assert(System.getProperty("sbt.server.autostart") == "false") + assert(!result.sbtArguments.exists(_.contains("autostart"))) + assert(result.commandArguments.contains("compile")) + finally System.clearProperty("sbt.server.autostart") + + test("--autostart=true sets sbt.server.autostart=true"): + try + System.clearProperty("sbt.server.autostart") + parse("--autostart=true", "compile") + assert(System.getProperty("sbt.server.autostart") == "true") + finally System.clearProperty("sbt.server.autostart") + test("--sbt-launch-jar is preserved"): val result = parse("--sbt-launch-jar", "/path/to/sbt-launch.jar", "compile") assert(result.sbtLaunchJar.contains("/path/to/sbt-launch.jar")) diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala index 6a49cd985..f510ecb86 100644 --- a/main-settings/src/main/scala/sbt/Def.scala +++ b/main-settings/src/main/scala/sbt/Def.scala @@ -283,6 +283,8 @@ object Def extends BuildSyntax with Init with InitializeImplicits: private[sbt] val cacheEventLog: CacheEventLog = CacheEventLog() private[sbt] val localDigestCacheByteSizeKey = SettingKey[Long](BasicKeys.localDigestCacheByteSize) + private[sbt] val cacheVersionKey = + SettingKey[Long](BasicKeys.cacheVersion) @cacheLevel(include = Array.empty) val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task { val state = stateKey.value @@ -298,13 +300,15 @@ object Def extends BuildSyntax with Init with InitializeImplicits: DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter) ) val cacheByteSize = localDigestCacheByteSizeKey.value + val cv = cacheVersionKey.value BuildWideCacheConfiguration( cacheStore, outputDirectory, fileConverter, state.log, cacheEventLog, - cacheByteSize + cacheByteSize, + cv, ) } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index e1d546ff1..e6fa4de25 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -460,6 +460,7 @@ object Keys { val allowZombieClassLoaders = settingKey[Boolean]("Allow a classloader that has previously been closed by `run` or `test` to continue loading classes.") val localCacheDirectory = settingKey[File]("Operating system specific cache directory.") val localDigestCacheByteSize = SettingKey[Long](BasicKeys.localDigestCacheByteSize).withRank(DSetting) + val cacheVersion = SettingKey[Long](BasicKeys.cacheVersion).withRank(BSetting) val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting) val exportPipelining = settingKey[Boolean]("Produce early output so downstream subprojects can do pipelining.").withRank(BSetting) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 2f87ff5bc..fea308de3 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -11,7 +11,7 @@ package sbt import java.io.{ File, IOException } import java.net.URI import java.nio.channels.ClosedChannelException -import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files } +import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files, Paths } import java.util.Properties import java.util.concurrent.ForkJoinPool import java.util.concurrent.atomic.AtomicBoolean @@ -32,7 +32,7 @@ import sbt.internal.util.complete.Parser import sbt.internal.util.{ RunningProcesses, Terminal as ITerminal, * } import sbt.io.* import sbt.io.syntax.* -import sbt.util.{ Level, Logger, Show } +import sbt.util.{ ActionCache, Level, Logger, Show } import xsbti.AppProvider import scala.annotation.{ nowarn, tailrec } @@ -246,17 +246,42 @@ object StandardMain { ConsoleOut.systemOutOverwrite(ConsoleOut.overwriteContaining("Resolving ")) ConsoleOut.setGlobalProxy(console) - private def initialGlobalLogging(file: Option[File]): GlobalLogging = { - def createTemp(attempt: Int = 0): File = Retry { + private[sbt] def logLevelFromArguments(args: Seq[String]): Level.Value = + val earlyCmd = BasicCommandStrings.EarlyCommand + val levelOptions = Level.values.toSeq.flatMap: v => + List("-" + v.toString, "--" + v.toString) + def levelFromArg(arg: String): Option[Level.Value] = + if arg.startsWith(earlyCmd + "(") && arg.endsWith(")") then + val inner = arg.slice(earlyCmd.length + 1, arg.length - 1).trim + Level.values.find(_.toString == inner) + else if levelOptions.contains(arg) then + Level.values.find(v => arg == "-" + v.toString || arg == "--" + v.toString) + else None + args.flatMap(levelFromArg).headOption.getOrElse(Level.Info) + + private def initialGlobalLogging(file: Option[File], initialLevel: Level.Value): GlobalLogging = + def createTemp(prefix: String)(attempt: Int = 0): File = Retry: file.foreach(f => if (!f.exists()) IO.createDirectory(f)) - File.createTempFile("sbt-global-log", ".log", file.orNull) - } + File.createTempFile(prefix, ".log", file.orNull) + + // Call this experimental since we don't want to commit to the log format for now + val execLogProp = "sbt.experimental_execution_log" + val execLog = + if java.lang.Boolean.getBoolean(execLogProp) then + Option(ActionCache.setExecLog(createTemp("exec-log")().toPath())) + else sys.props.get(execLogProp).map(Paths.get(_)).map(ActionCache.setExecLog) + execLog.foreach: log => + ShutdownHooks.add(() => log.close()) GlobalLogging.initial( MainAppender.globalDefault(ConsoleOut.globalProxy), - createTemp(), - ConsoleOut.globalProxy + createTemp("sbt-global-log")(), + ConsoleOut.globalProxy, + initialLevel ) - } + + private def initialGlobalLogging(file: Option[File]): GlobalLogging = + initialGlobalLogging(file, Level.Info) + def initialGlobalLogging(file: File): GlobalLogging = initialGlobalLogging(Option(file)) @deprecated("use version that takes file argument", "1.4.0") def initialGlobalLogging: GlobalLogging = initialGlobalLogging(None) @@ -265,8 +290,7 @@ object StandardMain { configuration: xsbti.AppConfiguration, initialDefinitions: Seq[Command], preCommands: Seq[String] - ): State = { - // This is to workaround https://github.com/sbt/io/issues/110 + ): State = if (!sys.props.contains("jna.nosys")) sys.props.put("jna.nosys", "true") import BasicCommandStrings.{ DashDashDetachStdio, DashDashServer, isEarlyCommand } @@ -274,11 +298,18 @@ object StandardMain { configuration.arguments .map(_.trim) .filterNot(c => c == DashDashDetachStdio || c == DashDashServer) + .toSeq + val initialLevel = logLevelFromArguments(userCommands) val (earlyCommands, normalCommands) = (preCommands ++ userCommands).partition(isEarlyCommand) - val commands = (earlyCommands ++ normalCommands).toList map { x => - Exec(x, None) - } - val initAttrs = BuiltinCommands.initialAttributes + val commands = (earlyCommands ++ normalCommands).toList.map(x => Exec(x, None)) + val baseAttrs = BuiltinCommands.initialAttributes + val initAttrs = + if initialLevel == Level.Info then baseAttrs + else + baseAttrs + .put(Keys.logLevel.key, initialLevel) + .put(BasicKeys.explicitGlobalLogLevels, true) + val logDir = BuildPaths.globalLoggingStandard(configuration.baseDirectory) val s = State( configuration, initialDefinitions, @@ -287,12 +318,12 @@ object StandardMain { commands, State.newHistory, initAttrs, - initialGlobalLogging(BuildPaths.globalLoggingStandard(configuration.baseDirectory)), + initialGlobalLogging(Option(logDir), initialLevel), None, State.Continue ) s.initializeClassLoaderCache - } + end initialState } import sbt.BasicCommandStrings.* diff --git a/main/src/main/scala/sbt/RemoteCache.scala b/main/src/main/scala/sbt/RemoteCache.scala index 91f5b616c..ab2f50de1 100644 --- a/main/src/main/scala/sbt/RemoteCache.scala +++ b/main/src/main/scala/sbt/RemoteCache.scala @@ -30,6 +30,7 @@ object RemoteCache: lazy val globalSettings: Seq[Def.Setting[?]] = Seq( localCacheDirectory :== defaultCacheLocation, localDigestCacheByteSize :== CacheImplicits.defaultLocalDigestCacheByteSize, + cacheVersion :== sys.props.get("sbt.cacheversion").flatMap(_.toLongOption).getOrElse(0L), rootOutputDirectory := { appConfiguration.value.baseDirectory .toPath() diff --git a/main/src/test/scala/sbt/internal/InitialLogLevelSpec.scala b/main/src/test/scala/sbt/internal/InitialLogLevelSpec.scala new file mode 100644 index 000000000..4296bbe99 --- /dev/null +++ b/main/src/test/scala/sbt/internal/InitialLogLevelSpec.scala @@ -0,0 +1,38 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import sbt.util.Level + +object InitialLogLevelSpec extends verify.BasicTestSuite: + + test("logLevelFromArguments returns Debug when -debug is in arguments"): + assert(StandardMain.logLevelFromArguments(Seq("-debug")) == Level.Debug) + + test("logLevelFromArguments returns Debug when --debug is in arguments"): + assert(StandardMain.logLevelFromArguments(Seq("compile", "--debug")) == Level.Debug) + + test("logLevelFromArguments returns Debug when early(debug) is in arguments"): + assert(StandardMain.logLevelFromArguments(Seq("early(debug)")) == Level.Debug) + assert(StandardMain.logLevelFromArguments(Seq("compile", "early(debug)")) == Level.Debug) + + test("logLevelFromArguments returns Info when no level option in arguments"): + assert(StandardMain.logLevelFromArguments(Seq()) == Level.Info) + assert(StandardMain.logLevelFromArguments(Seq("compile", "run")) == Level.Info) + + test("logLevelFromArguments uses first level option when multiple present"): + assert(StandardMain.logLevelFromArguments(Seq("-warn", "-debug")) == Level.Warn) + assert(StandardMain.logLevelFromArguments(Seq("-error", "-info")) == Level.Error) + + test("logLevelFromArguments supports all level options"): + assert(StandardMain.logLevelFromArguments(Seq("-info")) == Level.Info) + assert(StandardMain.logLevelFromArguments(Seq("--warn")) == Level.Warn) + assert(StandardMain.logLevelFromArguments(Seq("--error")) == Level.Error) +end InitialLogLevelSpec diff --git a/notes/2.0.0/cache-version.md b/notes/2.0.0/cache-version.md new file mode 100644 index 000000000..4dad2b536 --- /dev/null +++ b/notes/2.0.0/cache-version.md @@ -0,0 +1,24 @@ +## cacheVersion setting + +sbt 2.x caches task results by default. `cacheVersion` provides an escape hatch to invalidate all caches when needed. + +### Usage + +In `build.sbt`: + +```scala +Global / cacheVersion := 1L +``` + +Or via system property: + +``` +sbt -Dsbt.cacheversion=1 +``` + +### Details + +- Defaults to reading system property `sbt.cacheversion`, or else `0L` +- When `cacheVersion` is `0L`, caching behaves identically to previous versions +- Changing the value invalidates all task caches, forcing recomputation +- The value is incorporated into every cache key via `BuildWideCacheConfiguration` diff --git a/project/DatatypeConfig.scala b/project/DatatypeConfig.scala index 6327fdd27..ecc6d7e20 100644 --- a/project/DatatypeConfig.scala +++ b/project/DatatypeConfig.scala @@ -85,7 +85,7 @@ object DatatypeConfig { case "Map" | "Tuple2" | "scala.Tuple2" => { tpe => twoArgs(tpe).flatMap(getFormats) } - case "Int" | "Long" => { _ => + case "Int" | "Long" | "sbt.util.Digest" => { _ => Nil } } diff --git a/sbt b/sbt index 849a2c7ef..2ced6ea5f 100755 --- a/sbt +++ b/sbt @@ -668,6 +668,8 @@ Usage: `basename "$0"` [options] -d | --debug set sbt log level to debug -debug-inc | --debug-inc enable extra debugging for the incremental compiler + --experimental_execution_log=true| + enable experimental execution log # sbt version (default: from project/build.properties if present, else latest release) --sbt-version use the specified version of sbt @@ -721,8 +723,11 @@ map_args () { --supershell=*) options=( "${options[@]}" "-Dsbt.supershell=${1:13}" ) && shift ;; -supershell=*) options=( "${options[@]}" "-Dsbt.supershell=${1:12}" ) && shift ;; -no-server|--no-server) options=( "${options[@]}" "-Dsbt.io.virtual=false" "-Dsbt.server.autostart=false" ) && shift ;; + --autostart=*) options=( "${options[@]}" "-Dsbt.server.autostart=${1:13}" ) && shift ;; + -autostart=*) options=( "${options[@]}" "-Dsbt.server.autostart=${1:12}" ) && shift ;; --color=*) options=( "${options[@]}" "-Dsbt.color=${1:8}" ) && shift ;; -color=*) options=( "${options[@]}" "-Dsbt.color=${1:7}" ) && shift ;; + --experimental_execution_log=*) options=( "${options[@]}" "-Dsbt.experimental_execution_log=${1:29}" ) && shift ;; -no-share|--no-share) options=( "${options[@]}" "${noshare_opts[@]}" ) && shift ;; -no-global|--no-global) options=( "${options[@]}" "-Dsbt.global.base=$(pwd)/project/.sbtboot" ) && shift ;; -ivy|--ivy) require_arg path "$1" "$2" && options=( "${options[@]}" "-Dsbt.ivy.home=$2" ) && shift 2 ;; diff --git a/sbt-app/src/sbt-test/actions/cache-version/build.sbt b/sbt-app/src/sbt-test/actions/cache-version/build.sbt new file mode 100644 index 000000000..2a8cc00f3 --- /dev/null +++ b/sbt-app/src/sbt-test/actions/cache-version/build.sbt @@ -0,0 +1,20 @@ +scalaVersion := "3.7.4" + +lazy val checkCounter = inputKey[Unit]("assert counter file value") + +checkCounter := { + val expected = complete.DefaultParsers.spaceDelimited("").parsed.head.toInt + val f = baseDirectory.value / "target" / "counter.txt" + val actual = if (f.exists()) IO.read(f).trim.toInt else 0 + assert(actual == expected, s"Expected counter=$expected but got $actual") +} + +lazy val bumpCounter = taskKey[Int]("cached task that bumps a counter file on each execution") + +bumpCounter := { + val f = baseDirectory.value / "target" / "counter.txt" + val n = if (f.exists()) IO.read(f).trim.toInt else 0 + val next = n + 1 + IO.write(f, next.toString) + next +} diff --git a/sbt-app/src/sbt-test/actions/cache-version/test b/sbt-app/src/sbt-test/actions/cache-version/test new file mode 100644 index 000000000..3830e2fc0 --- /dev/null +++ b/sbt-app/src/sbt-test/actions/cache-version/test @@ -0,0 +1,21 @@ +# bumpCounter is a cached task that increments a file counter on each execution. +# When the cache is hit, the task body is NOT re-executed, so the counter stays the same. + +# First call: body runs, counter goes to 1 +> bumpCounter +> checkCounter 1 + +# Second call: cache hit, body does NOT run, counter stays at 1 +> bumpCounter +> checkCounter 1 + +# Change cacheVersion: invalidates all caches +> set Global / cacheVersion := 1L + +# Third call: cache miss (version changed), body runs, counter goes to 2 +> bumpCounter +> checkCounter 2 + +# Fourth call: cache hit again, counter stays at 2 +> bumpCounter +> checkCounter 2 diff --git a/sbtw/src/main/scala/sbtw/ArgParser.scala b/sbtw/src/main/scala/sbtw/ArgParser.scala index 719a5b9a4..5536bd563 100644 --- a/sbtw/src/main/scala/sbtw/ArgParser.scala +++ b/sbtw/src/main/scala/sbtw/ArgParser.scala @@ -41,8 +41,12 @@ object ArgParser: opt[Int]("mem").action((x, c) => c.copy(mem = Some(x))), opt[String]("supershell").action((x, c) => c.copy(supershell = Some(x))), opt[String]("color").action((x, c) => c.copy(color = Some(x))), + opt[String]("autostart").action((x, c) => c.copy(autostart = Some(x))), opt[Int]("jvm-debug").action((x, c) => c.copy(jvmDebug = Some(x))), opt[String]("java-home").action((x, c) => c.copy(javaHome = Some(x))), + opt[String]("experimental_execution_log").action((x, c) => + c.copy(experimentalExecutionLog = Some(x)) + ), arg[String]("") .unbounded() .optional() diff --git a/sbtw/src/main/scala/sbtw/LauncherOptions.scala b/sbtw/src/main/scala/sbtw/LauncherOptions.scala index bc95e8f55..e6527eca3 100644 --- a/sbtw/src/main/scala/sbtw/LauncherOptions.scala +++ b/sbtw/src/main/scala/sbtw/LauncherOptions.scala @@ -29,11 +29,13 @@ case class LauncherOptions( mem: Option[Int] = None, supershell: Option[String] = None, color: Option[String] = None, + autostart: Option[String] = None, jvmDebug: Option[Int] = None, javaHome: Option[String] = None, server: Boolean = false, residual: Seq[String] = Nil, sbtNew: Boolean = false, + experimentalExecutionLog: Option[String] = None, ) object LauncherOptions: diff --git a/sbtw/src/main/scala/sbtw/Runner.scala b/sbtw/src/main/scala/sbtw/Runner.scala index 48bdc328c..9df81705c 100644 --- a/sbtw/src/main/scala/sbtw/Runner.scala +++ b/sbtw/src/main/scala/sbtw/Runner.scala @@ -62,6 +62,8 @@ object Runner: opts.sbtCache.foreach(v => s = s :+ s"-Dsbt.global.localcache=$v") opts.ivy.foreach(v => s = s :+ s"-Dsbt.ivy.home=$v") opts.color.foreach(v => s = s :+ s"-Dsbt.color=$v") + opts.autostart.foreach(v => s = s :+ s"-Dsbt.server.autostart=$v") + opts.experimentalExecutionLog.foreach(v => s = s :+ s"-Dsbt.experimental_execution_log=$v") if opts.timings then s = s ++ Seq("-Dsbt.task.timings=true", "-Dsbt.task.timings.on.shutdown=true") if opts.traces then s = s :+ "-Dsbt.traces=true" diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala new file mode 100644 index 000000000..92a5348cb --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnExec.scala @@ -0,0 +1,50 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util +final class SpawnExec private ( + val input: sbt.internal.util.SpawnInput, + val cacheHit: Boolean, + val exitCode: Option[Int], + val outputs: Vector[xsbti.HashedVirtualFileRef]) extends Serializable { + + private def this(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]) = this(input, cacheHit, exitCode, Vector()) + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: SpawnExec => (this.input == x.input) && (this.cacheHit == x.cacheHit) && (this.exitCode == x.exitCode) && (this.outputs == x.outputs) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnExec".##) + input.##) + cacheHit.##) + exitCode.##) + outputs.##) + } + override def toString: String = { + "SpawnExec(" + input + ", " + cacheHit + ", " + exitCode + ", " + outputs + ")" + } + private def copy(input: sbt.internal.util.SpawnInput = input, cacheHit: Boolean = cacheHit, exitCode: Option[Int] = exitCode, outputs: Vector[xsbti.HashedVirtualFileRef] = outputs): SpawnExec = { + new SpawnExec(input, cacheHit, exitCode, outputs) + } + def withInput(input: sbt.internal.util.SpawnInput): SpawnExec = { + copy(input = input) + } + def withCacheHit(cacheHit: Boolean): SpawnExec = { + copy(cacheHit = cacheHit) + } + def withExitCode(exitCode: Option[Int]): SpawnExec = { + copy(exitCode = exitCode) + } + def withExitCode(exitCode: Int): SpawnExec = { + copy(exitCode = Option(exitCode)) + } + def withOutputs(outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = { + copy(outputs = outputs) + } +} +object SpawnExec { + + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int]): SpawnExec = new SpawnExec(input, cacheHit, exitCode) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode)) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Option[Int], outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, exitCode, outputs) + def apply(input: sbt.internal.util.SpawnInput, cacheHit: Boolean, exitCode: Int, outputs: Vector[xsbti.HashedVirtualFileRef]): SpawnExec = new SpawnExec(input, cacheHit, Option(exitCode), outputs) +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala new file mode 100644 index 000000000..97561e582 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/SpawnInput.scala @@ -0,0 +1,55 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util +final class SpawnInput private ( + val digest: sbt.util.Digest, + val codeContentHash: sbt.util.Digest, + val extraHash: sbt.util.Digest, + val cacheVersion: Option[Long], + val str: Option[String]) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: SpawnInput => (this.digest == x.digest) && (this.codeContentHash == x.codeContentHash) && (this.extraHash == x.extraHash) && (this.cacheVersion == x.cacheVersion) && (this.str == x.str) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbt.internal.util.SpawnInput".##) + digest.##) + codeContentHash.##) + extraHash.##) + cacheVersion.##) + str.##) + } + override def toString: String = { + "SpawnInput(" + digest + ", " + codeContentHash + ", " + extraHash + ", " + cacheVersion + ", " + str + ")" + } + private def copy(digest: sbt.util.Digest = digest, codeContentHash: sbt.util.Digest = codeContentHash, extraHash: sbt.util.Digest = extraHash, cacheVersion: Option[Long] = cacheVersion, str: Option[String] = str): SpawnInput = { + new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + } + def withDigest(digest: sbt.util.Digest): SpawnInput = { + copy(digest = digest) + } + def withCodeContentHash(codeContentHash: sbt.util.Digest): SpawnInput = { + copy(codeContentHash = codeContentHash) + } + def withExtraHash(extraHash: sbt.util.Digest): SpawnInput = { + copy(extraHash = extraHash) + } + def withCacheVersion(cacheVersion: Option[Long]): SpawnInput = { + copy(cacheVersion = cacheVersion) + } + def withCacheVersion(cacheVersion: Long): SpawnInput = { + copy(cacheVersion = Option(cacheVersion)) + } + def withStr(str: Option[String]): SpawnInput = { + copy(str = str) + } + def withStr(str: String): SpawnInput = { + copy(str = Option(str)) + } +} +object SpawnInput { + + def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Option[Long], str: Option[String]): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + def apply(digest: sbt.util.Digest, codeContentHash: sbt.util.Digest, extraHash: sbt.util.Digest, cacheVersion: Long, str: String): SpawnInput = new SpawnInput(digest, codeContentHash, extraHash, Option(cacheVersion), Option(str)) +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala new file mode 100644 index 000000000..143e9a077 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnCodec.scala @@ -0,0 +1,11 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +trait SpawnCodec extends sjsonnew.BasicJsonProtocol + with sbt.internal.util.codec.SpawnInputFormats + with sbt.internal.util.codec.HashedVirtualFileRefFormats + with sbt.internal.util.codec.SpawnExecFormats +object SpawnCodec extends SpawnCodec \ No newline at end of file diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala new file mode 100644 index 000000000..b086518b7 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnExecFormats.scala @@ -0,0 +1,33 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait SpawnExecFormats { self: sbt.internal.util.codec.SpawnInputFormats & sjsonnew.BasicJsonProtocol & sbt.internal.util.codec.HashedVirtualFileRefFormats => +given SpawnExecFormat: JsonFormat[sbt.internal.util.SpawnExec] = new JsonFormat[sbt.internal.util.SpawnExec] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnExec = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val input = unbuilder.readField[sbt.internal.util.SpawnInput]("input") + val cacheHit = unbuilder.readField[Boolean]("cacheHit") + val exitCode = unbuilder.readField[Option[Int]]("exitCode") + val outputs = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputs") + unbuilder.endObject() + sbt.internal.util.SpawnExec(input, cacheHit, exitCode, outputs) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.internal.util.SpawnExec, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("input", obj.input) + builder.addField("cacheHit", obj.cacheHit) + builder.addField("exitCode", obj.exitCode) + builder.addField("outputs", obj.outputs) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala new file mode 100644 index 000000000..ec3878999 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/SpawnInputFormats.scala @@ -0,0 +1,35 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait SpawnInputFormats { self: sjsonnew.BasicJsonProtocol => +given SpawnInputFormat: JsonFormat[sbt.internal.util.SpawnInput] = new JsonFormat[sbt.internal.util.SpawnInput] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.internal.util.SpawnInput = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val digest = unbuilder.readField[sbt.util.Digest]("digest") + val codeContentHash = unbuilder.readField[sbt.util.Digest]("codeContentHash") + val extraHash = unbuilder.readField[sbt.util.Digest]("extraHash") + val cacheVersion = unbuilder.readField[Option[Long]]("cacheVersion") + val str = unbuilder.readField[Option[String]]("str") + unbuilder.endObject() + sbt.internal.util.SpawnInput(digest, codeContentHash, extraHash, cacheVersion, str) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.internal.util.SpawnInput, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("digest", obj.digest) + builder.addField("codeContentHash", obj.codeContentHash) + builder.addField("extraHash", obj.extraHash) + builder.addField("cacheVersion", obj.cacheVersion) + builder.addField("str", obj.str) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband/spawn.contra b/util-cache/src/main/contraband/spawn.contra new file mode 100644 index 000000000..03cfaf2f5 --- /dev/null +++ b/util-cache/src/main/contraband/spawn.contra @@ -0,0 +1,21 @@ +package sbt.internal.util +@target(Scala) +@codecPackage("sbt.internal.util.codec") +@fullCodec("SpawnCodec") + +# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto +type SpawnInput { + digest: sbt.util.Digest! + codeContentHash: sbt.util.Digest! + extraHash: sbt.util.Digest! + cacheVersion: Long + str: String +} + +# https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto +type SpawnExec { + input: sbt.internal.util.SpawnInput! + cacheHit: Boolean! + exitCode: Int + outputs: [xsbti.HashedVirtualFileRef] @since("0.2.0") +} diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index e08cfd1d1..50cd15c44 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -8,10 +8,16 @@ package sbt.util -import java.io.{ File, IOException } +import java.io.{ File, IOException, PrintWriter } import java.nio.charset.StandardCharsets import java.nio.file.{ Files, Path, Paths, StandardCopyOption } -import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 } +import sbt.internal.util.{ + ActionCacheEvent, + CacheEventLog, + SpawnExec, + SpawnInput, + StringVirtualFile1 +} import sbt.io.syntax.* import sbt.io.IO import sbt.nio.file.{ **, FileTreeView } @@ -19,10 +25,10 @@ import sbt.nio.file.syntax.* import sbt.util.CacheImplicits import scala.reflect.ClassTag import scala.annotation.{ meta, StaticAnnotation } -import scala.util.control.{ Exception, NonFatal } +import scala.util.control.NonFatal import sjsonnew.{ HashWriter, JsonFormat } import sjsonnew.support.murmurhash.Hasher -import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter } import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes } import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef } @@ -31,6 +37,11 @@ object ActionCache: private[sbt] val manifestFileName = "sbtdir_manifest.json" private[sbt] val failureFileName = "failure.json" private[sbt] val failureExitCode = 1 + private[sbt] var execLog: Option[PrintWriter] = None + private[sbt] def setExecLog(log: Path): PrintWriter = + val writer = PrintWriter(log.toFile, "UTF-8") + execLog = Option(writer) + writer /** * This is a key function that drives remote caching. @@ -41,9 +52,10 @@ object ActionCache: * all we need from the input is to generate some hash value. * - codeContentHash: This hash represents the Scala code of the task. * Even if the input tasks are the same, the code part needs to be tracked. - * - extraHash: Reserved for later, which we might use to invalidate the cache. + * - extraHash: Extra hash for cache invalidation (combined with config.cacheVersion). * - tags: Tags to track cache level. * - config: The configuration that's used to store where the cache backends are. + * config.cacheVersion is incorporated into the cache key to allow global invalidation. * - action: The actual action to be cached. */ def cache[I: HashWriter, O: JsonFormat]( @@ -57,17 +69,19 @@ object ActionCache: ): O = import config.* + val inputDigest = mkInput(key, codeContentHash, extraHash, cacheVersion) + def cacheFailure(e: CompileFailed): Nothing = // Cache the failure so subsequent builds don't re-run failed compilation // This fixes https://github.com/sbt/sbt/issues/7662 // Use the same input digest as success, distinguished by exitCode cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) val cachedFailure = CachedCompileFailure.fromException(e) val json = Converter.toJsonUnsafe(cachedFailure) + val valuePath = mkValuePath(inputDigest) val failureFile = StringVirtualFile1(valuePath, CompactPrinter(json)) store.put( - UpdateActionResultRequest(input, Vector(failureFile), exitCode = failureExitCode) + UpdateActionResultRequest(inputDigest, Vector(failureFile), exitCode = failureExitCode) ) throw e @@ -80,49 +94,82 @@ object ActionCache: cacheFailure(e) case e: Exception => cacheEventLog.append(ActionCacheEvent.Error) + logExec( + SpawnExec(input = spawnInput, cacheHit = false, exitCode = 1) + ) throw e - val json = Converter.toJsonUnsafe(result) - val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() - val uncacheableOutputs = - outputs.filter(f => - f match - case vf if vf.id.endsWith(ActionCache.dirZipExt) => - false - case _ => - val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() - !outputPath.startsWith(normalizedOutputDir) - ) - if uncacheableOutputs.nonEmpty then - cacheEventLog.append(ActionCacheEvent.Error) - logger.error( - s"Cannot cache task because its output files are outside the output directory: \n" + - uncacheableOutputs.mkString(" - ", "\n - ", "") - ) - result - else - cacheEventLog.append(ActionCacheEvent.OnsiteTask) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) - val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) - val newOutputs = Vector(valueFile) ++ outputs.toVector - try - store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match + try + val json = Converter.toJsonUnsafe(result) + val normalizedOutputDir = outputDirectory.toAbsolutePath.normalize() + val uncacheableOutputs = + outputs.filter(f => + f match + case vf if vf.id.endsWith(ActionCache.dirZipExt) => + false + case _ => + val outputPath = fileConverter.toPath(f).toAbsolutePath.normalize() + !outputPath.startsWith(normalizedOutputDir) + ) + if uncacheableOutputs.nonEmpty then + cacheEventLog.append(ActionCacheEvent.Error) + logger.error( + s"Cannot cache task because its output files are outside the output directory: \n" + + uncacheableOutputs.mkString(" - ", "\n - ", "") + ) + result + else + cacheEventLog.append(ActionCacheEvent.OnsiteTask) + val valuePath = mkValuePath(inputDigest) + val valueFile = StringVirtualFile1(valuePath, CompactPrinter(json)) + val newOutputs = Vector(valueFile) ++ outputs.toVector + store.put(UpdateActionResultRequest(inputDigest, newOutputs, exitCode = 0)) match case Right(cachedResult) => store.syncBlobs(cachedResult.outputFiles, outputDirectory) + logExec( + SpawnExec( + input = spawnInput, + cacheHit = false, + exitCode = 0, + outputs = cachedResult.outputFiles, + ) + ) result case Left(e) => throw e - catch - case e: IOException => - logger.debug(s"Skipping cache storage due to error: ${e.getMessage}") - cacheEventLog.append(ActionCacheEvent.Error) - result + catch + case e: IOException => + logger.debug(s"Skipping cache storage due to error: ${e.getMessage}") + cacheEventLog.append(ActionCacheEvent.Error) + result + def spawnInput = SpawnInput( + digest = inputDigest, + codeContentHash = codeContentHash, + extraHash = extraHash, + cacheVersion = if cacheVersion != 0 then Some(cacheVersion) else None, + str = Some(key.toString()), + ) + inline def logExec(inline event: SpawnExec): Unit = + execLog.foreach: log => + logEvent(event, log) // Single cache lookup - use exitCode to distinguish success from failure - getWithFailure(key, codeContentHash, extraHash, tags, config) match - case Right(value) => value + getWithFailure(inputDigest, tags, config) match + case Right((value, result)) => + logExec( + SpawnExec( + input = spawnInput, + cacheHit = true, + exitCode = result.exitCode, + outputs = result.outputFiles, + ) + ) + value case Left(Some(failure)) => config.cacheEventLog.append(ActionCacheEvent.Found("cached-failure")) // Replay problems to the logger so users see the cached errors/warnings failure.replay(config.logger) + logExec( + SpawnExec(input = spawnInput, cacheHit = true, exitCode = 1) + ) throw failure.toException case Left(None) => organicTask end cache @@ -132,13 +179,11 @@ object ActionCache: * Returns Right(value) for cached success, Left(Some(failure)) for cached failure, * or Left(None) for cache miss. */ - private def getWithFailure[I: HashWriter, O: JsonFormat]( - key: I, - codeContentHash: Digest, - extraHash: Digest, + private def getWithFailure[O: JsonFormat]( + inputDigest: Digest, tags: List[CacheLevelTag], config: BuildWideCacheConfiguration, - ): Either[Option[CachedCompileFailure], O] = + ): Either[Option[CachedCompileFailure], (O, ActionResult)] = import config.store def valueFromStr(str: String, origin: Option[String]): O = config.cacheEventLog.append(ActionCacheEvent.Found(origin.getOrElse("unknown"))) @@ -151,59 +196,36 @@ object ActionCache: def parseCachedValue( str: String, - origin: Option[String], + result: ActionResult, isFailure: Boolean, - ): Option[Either[Option[CachedCompileFailure], O]] = + ): Option[Either[Option[CachedCompileFailure], (O, ActionResult)]] = try if isFailure then Some(Left(Some(failureFromStr(str)))) - else Some(Right(valueFromStr(str, origin))) + else Some(Right((valueFromStr(str, result.origin), result))) catch case _: Exception => None - // Optimization: Check if we can read directly from symlinked value file - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) - val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath)) - - def readFromSymlink(): Option[Either[Option[CachedCompileFailure], O]] = - if java.nio.file.Files.isSymbolicLink(resolvedValuePath) && java.nio.file.Files - .exists(resolvedValuePath) - then - Exception.nonFatalCatch - .opt(IO.read(resolvedValuePath.toFile(), StandardCharsets.UTF_8)) - .flatMap: str => - findActionResult(key, codeContentHash, extraHash, config) match - case Right(result) => - try - store.syncBlobs(result.outputFiles, config.outputDirectory) - parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode)) - catch case NonFatal(_) => None - case Left(_) => None - else None - - readFromSymlink() match - case Some(result) => result - case None => - findActionResult(key, codeContentHash, extraHash, config) match - case Right(result) => - try - val isFailure = result.exitCode.contains(failureExitCode) - result.contents.headOption match - case Some(head) => - store.syncBlobs(result.outputFiles, config.outputDirectory) - val str = String(head.array(), StandardCharsets.UTF_8) - parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) - case _ => - val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) - if paths.isEmpty then Left(None) - else - val str = IO.read(paths.head.toFile()) - parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) - catch - case NonFatal(e) => - config.logger.debug( - s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}" - ) - Left(None) - case Left(_) => Left(None) + findActionResult(inputDigest, config) match + case Right(result) => + try + val isFailure = result.exitCode.contains(failureExitCode) + result.contents.headOption match + case Some(head) => + store.syncBlobs(result.outputFiles, config.outputDirectory) + val str = String(head.array(), StandardCharsets.UTF_8) + parseCachedValue(str, result, isFailure).getOrElse(Left(None)) + case _ => + val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) + if paths.isEmpty then Left(None) + else + val str = IO.read(paths.head.toFile()) + parseCachedValue(str, result, isFailure).getOrElse(Left(None)) + catch + case NonFatal(e) => + config.logger.debug( + s"Ignoring cache retrieval failure, will recompute: ${e.getMessage}" + ) + Left(None) + case Left(_) => Left(None) /** * Retrieves the cached value. @@ -215,8 +237,9 @@ object ActionCache: tags: List[CacheLevelTag], config: BuildWideCacheConfiguration, ): Option[O] = - getWithFailure(key, codeContentHash, extraHash, tags, config) match - case Right(value) => Some(value) + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) + getWithFailure(inputDigest, tags, config) match + case Right(value) => Some(value._1) case Left(_) => None /** @@ -228,10 +251,26 @@ object ActionCache: extraHash: Digest, config: BuildWideCacheConfiguration, ): Boolean = - findActionResult(key, codeContentHash, extraHash, config) match + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) + findActionResult(inputDigest, config) match case Right(_) => true case Left(_) => false + inline private[sbt] def findActionResult( + inputDigest: Digest, + config: BuildWideCacheConfiguration, + ): Either[Throwable, ActionResult] = + // val logger = config.logger + CacheImplicits.setCacheSize(config.localDigestCacheByteSize) + val getRequest = + GetActionResultRequest( + inputDigest, + inlineStdout = false, + inlineStderr = false, + Vector(mkValuePath(inputDigest)) + ) + config.store.get(getRequest) + inline private[sbt] def findActionResult[I: HashWriter]( key: I, codeContentHash: Digest, @@ -240,19 +279,35 @@ object ActionCache: ): Either[Throwable, ActionResult] = // val logger = config.logger CacheImplicits.setCacheSize(config.localDigestCacheByteSize) - val (input, valuePath) = mkInput(key, codeContentHash, extraHash) + val inputDigest = mkInput(key, codeContentHash, extraHash, config.cacheVersion) val getRequest = - GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath)) + GetActionResultRequest( + inputDigest, + inlineStdout = false, + inlineStderr = false, + Vector(mkValuePath(inputDigest)) + ) config.store.get(getRequest) private inline def mkInput[I: HashWriter]( key: I, codeContentHash: Digest, - extraHash: Digest - ): (Digest, String) = - val input = - Digest.sha256Hash(codeContentHash, extraHash, Digest.dummy(Hasher.hashUnsafe[I](key))) - (input, s"$${OUT}/value/$input.json") + extraHash: Digest, + cacheVersion: Long, + ): Digest = + Digest.sha256Hash( + (Vector( + codeContentHash, + Digest.dummy(Hasher.hashUnsafe[I](key)), + extraHash + ) ++ { + if cacheVersion == 0 then Vector.empty + else Vector(Digest.dummy(cacheVersion)) + })* + ) + + private inline def mkValuePath(inputDigest: Digest): String = + s"$${OUT}/value/${inputDigest}.json" def manifestFromFile(manifest: Path): Manifest = import sbt.internal.util.codec.ManifestCodec.given @@ -323,6 +378,14 @@ object ActionCache: private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] = Some(r.value, r.outputs) end InternalActionResult + + private[sbt] def logEvent(event: SpawnExec, log: PrintWriter): Unit = + import sbt.internal.util.codec.SpawnCodec.given + val json = Converter.toJsonUnsafe(event) + val s = PrettyPrinter(json) + log.println(s) + log.flush() + end ActionCache class BuildWideCacheConfiguration( @@ -332,13 +395,14 @@ class BuildWideCacheConfiguration( val logger: Logger, val cacheEventLog: CacheEventLog, val localDigestCacheByteSize: Long, + val cacheVersion: Long, ): def this( store: ActionCacheStore, outputDirectory: Path, fileConverter: FileConverter, logger: Logger, - cacheEventLog: CacheEventLog + cacheEventLog: CacheEventLog, ) = this( store, @@ -346,7 +410,26 @@ class BuildWideCacheConfiguration( fileConverter, logger, cacheEventLog, - CacheImplicits.defaultLocalDigestCacheByteSize + CacheImplicits.defaultLocalDigestCacheByteSize, + 0L, + ) + + def this( + store: ActionCacheStore, + outputDirectory: Path, + fileConverter: FileConverter, + logger: Logger, + cacheEventLog: CacheEventLog, + localDigestCacheByteSize: Long, + ) = + this( + store, + outputDirectory, + fileConverter, + logger, + cacheEventLog, + localDigestCacheByteSize, + 0L, ) override def toString(): String = diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index e551900f1..cb6c78e47 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -83,7 +83,8 @@ end AbstractActionCacheStore /** * An aggregate ActionCacheStore. */ -class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) extends AbstractActionCacheStore: +case class AggregateActionCacheStore(stores: Seq[ActionCacheStore]) + extends AbstractActionCacheStore: extension [A1](xs: Seq[A1]) // unlike collectFirst this accepts A1 => Seq[A2] inline def collectFirst2[A2](f: A1 => Seq[A2], size: Int): Seq[A2] = @@ -176,7 +177,8 @@ class InMemoryActionCacheStore extends AbstractActionCacheStore: underlying.toString() end InMemoryActionCacheStore -class DiskActionCacheStore(base: Path, converter: FileConverter) extends AbstractActionCacheStore: +case class DiskActionCacheStore(base: Path, converter: FileConverter) + extends AbstractActionCacheStore: lazy val casBase: Path = { val dir = base.resolve("cas") IO.createDirectory(dir.toFile) diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index 79bcc5831..05c79b48f 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -298,6 +298,36 @@ object ActionCacheTest extends BasicTestSuite: assert(v2 == 42) assert(called == 2) + test("Changing cacheVersion invalidates the cache"): + withDiskCache(testCacheVersionInvalidation) + + def testCacheVersionInvalidation(cache: ActionCacheStore): Unit = + import sjsonnew.BasicJsonProtocol.* + var called = 0 + val action: ((Int, Int)) => InternalActionResult[Int] = { (a, b) => + called += 1 + InternalActionResult(a + b, Nil) + } + IO.withTemporaryDirectory: tempDir => + val config0 = getCacheConfig(cache, tempDir) + // First call: computes the result + val v1 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action) + assert(v1 == 2) + assert(called == 1) + // Second call with same config: hits cache + val v2 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config0)(action) + assert(v2 == 2) + assert(called == 1) + // Third call with different cacheVersion: cache miss, recomputes + val config1 = getCacheConfig(cache, tempDir, cacheVersion = 1L) + val v3 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action) + assert(v3 == 2) + assert(called == 2) + // Fourth call with same cacheVersion=1: hits cache again + val v4 = ActionCache.cache((1, 1), Digest.zero, Digest.zero, tags, config1)(action) + assert(v4 == 2) + assert(called == 2) + def withInMemoryCache(f: InMemoryActionCacheStore => Unit): Unit = val cache = InMemoryActionCacheStore() f(cache) @@ -312,12 +342,24 @@ object ActionCacheTest extends BasicTestSuite: keepDirectory = false ) - def getCacheConfig(cache: ActionCacheStore, outputDir: File): BuildWideCacheConfiguration = + def getCacheConfig( + cache: ActionCacheStore, + outputDir: File, + cacheVersion: Long = 0L, + ): BuildWideCacheConfiguration = val logger = new Logger: override def trace(t: => Throwable): Unit = () override def success(message: => String): Unit = () override def log(level: Level.Value, message: => String): Unit = () - BuildWideCacheConfiguration(cache, outputDir.toPath(), fileConverter, logger, CacheEventLog()) + BuildWideCacheConfiguration( + cache, + outputDir.toPath(), + fileConverter, + logger, + CacheEventLog(), + CacheImplicits.defaultLocalDigestCacheByteSize, + cacheVersion, + ) def fileConverter = new FileConverter: override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id)