diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index ef40dc267..786034b87 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -14,14 +14,62 @@ jobs: JVM_OPTS: -Xms800M -Xmx2G -Xss6M -XX:ReservedCodeCacheSize=128M -server -Dsbt.io.virtual=false -Dfile.encoding=UTF-8 SBT_ETC_FILE: $HOME/etc/sbt/sbtopts steps: - - uses: actions/checkout@v6 - - name: Setup JDK - uses: actions/setup-java@v5 - with: - distribution: "zulu" - java-version: "17" - cache: sbt - - uses: sbt/setup-sbt@v1 - - name: Server test - shell: bash - run: sbt -v --client "serverTestProj/test" + - uses: actions/checkout@v7 + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: "zulu" + java-version: "17" + cache: sbt + - uses: sbt/setup-sbt@v1 + with: + disk-cache: false + - name: Set bazel-remote version + id: bazel-remote-version + run: echo "version=2.6.2" >> "$GITHUB_OUTPUT" + - name: Cache bazel-remote + id: cache-bazel-remote + uses: actions/cache@v4 + with: + path: $RUNNER_TOOL_CACHE/local/bazel-remote + key: bazel-remote-v${{ steps.bazel-remote-version.outputs.version }}-${{ runner.os }}-${{ runner.arch }} + - name: Download bazel-remote + if: steps.cache-bazel-remote.outputs.cache-hit != 'true' + shell: bash + env: + BAZEL_REMOTE_VERSION: ${{ steps.bazel-remote-version.outputs.version }} + run: | + mkdir -p "$RUNNER_TOOL_CACHE/local" + curl -sL -o "$RUNNER_TOOL_CACHE/local/bazel-remote" "https://github.com/buchgr/bazel-remote/releases/download/v${BAZEL_REMOTE_VERSION}/bazel-remote-${BAZEL_REMOTE_VERSION}-linux-amd64" + chmod +x "$RUNNER_TOOL_CACHE/local/bazel-remote" + - name: Add bazel-remote to PATH + run: echo "$RUNNER_TOOL_CACHE/local" >> "$GITHUB_PATH" + - name: Start bazel-remote + shell: bash + run: | + mkdir -p "$HOME/bazel-remote/temp" + nohup "$RUNNER_TOOL_CACHE/local/bazel-remote" --max_size 5 --dir "$HOME/bazel-remote/temp" \ + --http_address localhost:8000 --grpc_address localhost:2024 \ + > /tmp/bazel-remote.log 2>&1 & + echo $! > /tmp/bazel-remote.pid + for i in $(seq 1 30); do + curl -sf http://localhost:8000/status > /dev/null && exit 0 + sleep 1 + done + echo "bazel-remote did not start in time" + cat /tmp/bazel-remote.log + exit 1 + - name: Remote cache scripted test + shell: bash + run: | + sbt -v --client "doc; publishLocal" + sbt -v --client "scripted remote-cache/*" + - name: Server test + shell: bash + run: sbt -v --client "serverTestProj/test" + - name: Stop bazel-remote + if: always() + run: | + if [ -f /tmp/bazel-remote.pid ]; then + kill "$(cat /tmp/bazel-remote.pid)" || true + fi diff --git a/.gitignore b/.gitignore index 42f4ebf74..cee3b1f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ launcher-package/citest/freshly-baked sbt-launch.jar local-temp .jdk +*.sbt.semanticdb diff --git a/build.sbt b/build.sbt index 16befd471..7bd908fc3 100644 --- a/build.sbt +++ b/build.sbt @@ -15,9 +15,9 @@ ThisBuild / version := { nightlyVersion.getOrElse(v) } // update sbt.sh at root -ThisBuild / Utils.sbtnVersion := "2.0.0-b4d628dd" -ThisBuild / Utils.version2_13 := "2.0.0-SNAPSHOT" +ThisBuild / Utils.sbtnVersion := "2.0.0-a0c4773a" ThisBuild / versionScheme := Some("early-semver") +ThisBuild / Utils.version2_13 := "2.0.0-SNAPSHOT" ThisBuild / scalafmtOnCompile := !(Global / insideCI).value ThisBuild / Test / scalafmtOnCompile := !(Global / insideCI).value // ThisBuild / turbo := true @@ -391,6 +391,7 @@ lazy val utilCache = project exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.farmHash"), exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.farmHashStr"), exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.toFarmHashString"), + exclude[DirectMissingMethodProblem]("sbt.internal.util.CacheEventSummary#Data.*"), ), Test / fork := true, ) @@ -566,8 +567,11 @@ lazy val remoteCacheProj = (project in file("sbt-remote-cache")) name := "sbt-remote-cache", pluginCrossBuild / sbtVersion := version.value, publishMavenStyle := true, - mimaSettings, libraryDependencies ++= Seq(remoteapis, scalaVerify % Test), + mimaSettings, + mimaBinaryIssueFilters ++= Seq( + exclude[DirectMissingMethodProblem]("sbt.internal.GrpcActionCacheStore.this"), + ), ) // Implementation and support code for defining actions. diff --git a/lm-core/src/main/scala/sbt/librarymanagement/ConfigRef.scala b/lm-core/src/main/scala/sbt/librarymanagement/ConfigRef.scala index 5ef4bdb2d..9dceb8526 100644 --- a/lm-core/src/main/scala/sbt/librarymanagement/ConfigRef.scala +++ b/lm-core/src/main/scala/sbt/librarymanagement/ConfigRef.scala @@ -44,11 +44,13 @@ object ConfigRef extends sbt.librarymanagement.ConfigRefFunctions { private lazy val Pom = new ConfigRef("pom") private lazy val ScalaTool = new ConfigRef("scala-tool") private lazy val ScalaDocTool = new ConfigRef("scala-doc-tool") + private lazy val ScalaReplTool = new ConfigRef("scala-repl-tool") private lazy val CompilerPlugin = new ConfigRef("plugin") private lazy val Component = new ConfigRef("component") private lazy val RuntimeInternal = new ConfigRef("runtime-internal") private lazy val TestInternal = new ConfigRef("test-internal") private lazy val CompileInternal = new ConfigRef("compile-internal") + private lazy val ZincTool = new ConfigRef("zinc-tool") def apply(name: String): ConfigRef = name match { case "default" => Default @@ -61,11 +63,13 @@ object ConfigRef extends sbt.librarymanagement.ConfigRefFunctions { case "pom" => Pom case "scala-tool" => ScalaTool case "scala-doc-tool" => ScalaDocTool + case "scala-repl-tool" => ScalaReplTool case "plugin" => CompilerPlugin case "component" => Component case "runtime-internal" => RuntimeInternal case "test-internal" => TestInternal case "compile-internal" => CompileInternal + case "zinc-tool" => ZincTool case _ => cache.getOrElseUpdate(name, new ConfigRef(name)) } } diff --git a/lm-core/src/main/scala/sbt/librarymanagement/ConfigurationExtra.scala b/lm-core/src/main/scala/sbt/librarymanagement/ConfigurationExtra.scala index a50ea2c58..966df7c9d 100644 --- a/lm-core/src/main/scala/sbt/librarymanagement/ConfigurationExtra.scala +++ b/lm-core/src/main/scala/sbt/librarymanagement/ConfigurationExtra.scala @@ -50,6 +50,7 @@ object Configurations { lazy val ScalaDocTool = Configuration.of("ScalaDocTool", "scala-doc-tool").hide lazy val ScalaReplTool = Configuration.of("ScalaReplTool", "scala-repl-tool").hide lazy val CompilerPlugin = Configuration.of("CompilerPlugin", "plugin").hide + lazy val ZincTool = Configuration.of("ZincTool", "zinc-tool").hide lazy val Component = Configuration.of("Component", "component").hide private[sbt] val DefaultMavenConfiguration = defaultConfiguration(true) diff --git a/lm-core/src/main/scala/sbt/librarymanagement/ScalaArtifacts.scala b/lm-core/src/main/scala/sbt/librarymanagement/ScalaArtifacts.scala index 590c2fde4..4ccb7255d 100644 --- a/lm-core/src/main/scala/sbt/librarymanagement/ScalaArtifacts.scala +++ b/lm-core/src/main/scala/sbt/librarymanagement/ScalaArtifacts.scala @@ -18,6 +18,7 @@ object ScalaArtifacts { final val Scala3TastyInspectorID = "scala3-tasty-inspector" final val Scala3ReplID = "scala3-repl" final val Scala3_8Artifacts = Vector(LibraryID, Scala3LibraryID) + final val scala2SbtBridgeStart = "2.13.12" private[sbt] final val Scala3LibraryPrefix = Scala3LibraryID + "_" private[sbt] final val Scala3CompilerPrefix = Scala3CompilerID + "_" @@ -132,6 +133,29 @@ object ScalaArtifacts { Some(Configurations.ScalaTool.name + "->default,optional(default)") ) .platform(Platform.jvm) + + private[sbt] def hasScala2SbtBridge(sv: String): Boolean = + VersionNumber(sv).matchesSemVer( + SemanticSelector(s"=2.13 >=$scala2SbtBridgeStart") + ) + + private[sbt] def compilerBridgeDependencies( + org: String, + scalaVersion: String + ): Seq[ModuleID] = + if isScala3(scalaVersion) then + Vector( + ModuleID(org, "scala3-sbt-bridge", scalaVersion) + .withConfigurations(Some(s"${Configurations.ZincTool.name}->default,optional(default)")) + .platform(Platform.jvm) + ) + else if hasScala2SbtBridge(scalaVersion) then + Vector( + ModuleID(org, "scala2-sbt-bridge", scalaVersion) + .withConfigurations(Some(s"${Configurations.ZincTool.name}->default,optional(default)")) + .platform(Platform.jvm) + ) + else Nil } object SbtArtifacts { diff --git a/main-command/src/main/scala/sbt/internal/server/Server.scala b/main-command/src/main/scala/sbt/internal/server/Server.scala index d7ee44284..e71d6e643 100644 --- a/main-command/src/main/scala/sbt/internal/server/Server.scala +++ b/main-command/src/main/scala/sbt/internal/server/Server.scala @@ -159,7 +159,6 @@ private[sbt] object Server { } override def shutdown(): Unit = { - log.info("shutting down sbt server") if (portfile.exists) { IO.delete(portfile) } @@ -171,6 +170,7 @@ private[sbt] object Server { case null => case s => s.close() } + log.info("shutting down sbt server") } private def writeTokenfile(): Unit = { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index e704dd593..6cf443ff1 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -764,34 +764,14 @@ object Defaults extends BuildCommon { clean.value (ThisBuild / publish / clean).value }, - scalaCompilerBridgeBin := Def - .ifS(Def.task { + scalaCompilerBridgeBin := Def.uncached { + if { val sv = scalaVersion.value - val hasSbtBridge = ScalaArtifacts.isScala3(sv) || ZincLmUtil.hasScala2SbtBridge(sv) + val hasSbtBridge = ScalaArtifacts.isScala3(sv) || ScalaArtifacts.hasScala2SbtBridge(sv) hasSbtBridge - })(Def.cachedTask { - // Use scalaDynVersion to resolve dynamic versions (e.g., "3-latest.candidate" -> "3.8.1-RC1") - val sv = scalaDynVersion.value - val conv = fileConverter.value - val s = streams.value - val t = target.value - val r = dependencyResolution.value - val uc = updateConfiguration.value - val jar = ZincLmUtil.fetchDefaultBridgeModule( - scalaOrganization.value, - sv, - r, - uc, - (update / unresolvedWarningConfiguration).value, - s.log - ) - val out = t / "compiler-bridge" / jar.getName() - val outVf = conv.toVirtualFile(out.toPath()) - IO.copyFile(jar, out) - Def.declareOutput(outVf) - Vector(outVf: HashedVirtualFileRef) - })(Def.task(Vector.empty)) - .value, + } then Compiler.compilerBridgeFromUpdate.value + else Vector.empty + }, scalaCompilerBridgeJars := (Def.taskDyn { val s = streams.value val b = scalaCompilerBridgeBin.value @@ -1365,6 +1345,7 @@ object Defaults extends BuildCommon { ) ) } + def forkOptionsTask: Initialize[Task[ForkOptions]] = Def.task { val canUseArgumentsFile = sys.props @@ -1383,6 +1364,10 @@ object Defaults extends BuildCommon { ) } + /** Fork options for run-like tasks: the forked process inherits sbt's working directory. */ + private[sbt] def runForkOptionsTask: Initialize[Task[ForkOptions]] = + Def.task(forkOptionsTask.value.withWorkingDirectory(None)) + def testExecutionTask(task: Scoped): Initialize[Task[Tests.Execution]] = Def.task { new Tests.Execution( @@ -2611,7 +2596,7 @@ object Defaults extends BuildCommon { private lazy val newRunnerSettings: Seq[Setting[?]] = Seq( runner := Def.uncached(ClassLoaders.runner.value), - forkOptions := Def.uncached(forkOptionsTask.value) + forkOptions := Def.uncached(runForkOptionsTask.value) ) lazy val baseTasks: Seq[Setting[?]] = projectTasks ++ packageBase @@ -3349,7 +3334,7 @@ object Classpaths { ivyConfigurations ++= Configurations.auxiliary, ivyConfigurations ++= { if (managedScalaInstance.value && scalaHome.value.isEmpty) - Configurations.ScalaTool :: Configurations.ScalaDocTool :: Configurations.ScalaReplTool :: Nil + Configurations.ScalaTool :: Configurations.ScalaDocTool :: Configurations.ScalaReplTool :: Configurations.ZincTool :: Nil else Nil }, // Coursier needs these @@ -3586,6 +3571,7 @@ object Classpaths { then Nil else ScalaArtifacts.toolDependencies(scalaOrg, version) ++ + ScalaArtifacts.compilerBridgeDependencies(scalaOrg, version) ++ ScalaArtifacts.docToolDependencies(scalaOrg, version) ++ ScalaArtifacts.replToolDependencies(scalaOrg, version) allToolDeps.map(_.platform(Platform.jvm)) ++ pluginAdjust @@ -4969,7 +4955,7 @@ trait BuildExtra extends BuildCommon with DefExtra { } } }.evaluated - ) ++ inTask(scoped)((config / forkOptions) := Def.uncached(forkOptionsTask.value)) + ) ++ inTask(scoped)((config / forkOptions) := Def.uncached(runForkOptionsTask.value)) } // public API @@ -4991,7 +4977,7 @@ trait BuildExtra extends BuildCommon with DefExtra { r.run(mainClass, cp.files, arguments, s.log).get } }.value - ) ++ inTask(scoped)((config / forkOptions) := Def.uncached(forkOptionsTask.value)) + ) ++ inTask(scoped)((config / forkOptions) := Def.uncached(runForkOptionsTask.value)) def initScoped[T](sk: ScopedKey[?], i: Initialize[T]): Initialize[T] = initScope(fillTaskAxis(sk.scope, sk.key), i) diff --git a/main/src/main/scala/sbt/ProjectExtra.scala b/main/src/main/scala/sbt/ProjectExtra.scala index 4985203b4..de0a13e2b 100755 --- a/main/src/main/scala/sbt/ProjectExtra.scala +++ b/main/src/main/scala/sbt/ProjectExtra.scala @@ -376,6 +376,15 @@ trait ProjectExtra extends Scoped.Syntax: val srvLogLevel: Option[Level.Value] = (ref / serverLog / logLevel).get(structure.data) val hs: Option[Seq[ServerHandler]] = get(ThisBuild / fullServerHandlers) val caches: Option[Seq[ActionCacheStore]] = get(cacheStores) + // cacheStores is recomputed on every reload; close any store dropped from the new value. + s.attributes.get(cacheStores.key) match + case Some(oldCaches) => + val kept = caches.getOrElse(Nil) + oldCaches.foreach { + case store: AutoCloseable if !kept.exists(_ eq store) => store.close() + case _ => () + } + case None => () val rod: Option[NioPath] = get(rootOutputDirectory) val fileConverter: Option[FileConverter] = get(Keys.fileConverter) val commandDefs = allCommands.distinct.flatten[Command].map(_.tag(projectCommand, true)) diff --git a/main/src/main/scala/sbt/internal/CommandExchange.scala b/main/src/main/scala/sbt/internal/CommandExchange.scala index dc669a76a..75ded6960 100644 --- a/main/src/main/scala/sbt/internal/CommandExchange.scala +++ b/main/src/main/scala/sbt/internal/CommandExchange.scala @@ -319,9 +319,9 @@ private[sbt] final class CommandExchange { } procFile = None fastTrackThread.close() - channels foreach (_.shutdown(true)) + channels.foreach(c => Util.ignoreResult(Try(c.shutdown(true)))) // interrupt and kill the thread - server.foreach(_.shutdown()) + server.foreach(s => Util.ignoreResult(Try(s.shutdown()))) server = None EvaluateTask.onShutdown() } diff --git a/main/src/main/scala/sbt/internal/Compiler.scala b/main/src/main/scala/sbt/internal/Compiler.scala index ef898b095..5ecd5acdc 100644 --- a/main/src/main/scala/sbt/internal/Compiler.scala +++ b/main/src/main/scala/sbt/internal/Compiler.scala @@ -142,6 +142,17 @@ object Compiler: ) } + def compilerBridgeFromUpdate: Def.Initialize[Task[Seq[HashedVirtualFileRef]]] = + Def.task { + val fullReport = Keys.update.value + val report = fullReport.configuration(Configurations.ZincTool) + val allJars = report match + case Some(r) => r.modules.flatMap(_.artifacts.map(_._2)) + case None => Nil + val conv = Keys.fileConverter.value + allJars.map(x => (conv.toVirtualFile(x.toPath()): HashedVirtualFileRef)) + } + def scalaInstanceConfigFromUpdate( extraToolConf: Option[Configuration] ): Def.Initialize[Task[ScalaInstanceConfig]] = Def.task { diff --git a/main/src/main/scala/sbt/internal/server/BuildServerProtocol.scala b/main/src/main/scala/sbt/internal/server/BuildServerProtocol.scala index 6a55a5a9e..b9bcf85e9 100644 --- a/main/src/main/scala/sbt/internal/server/BuildServerProtocol.scala +++ b/main/src/main/scala/sbt/internal/server/BuildServerProtocol.scala @@ -327,7 +327,7 @@ object BuildServerProtocol { JavacOptionsItem(target, javacOptions, classpath, classDirectory.toURI) }, bspBuildTargetJVMRunEnvironment := bspInputTask { (_, filter) => - val items = bspBuildTargetJvmEnvironmentItem.result.all(filter).value + val items = (run / bspBuildTargetJvmEnvironmentItem).result.all(filter).value val successfulItems = anyOrThrow(items) val result = JvmRunEnvironmentResult(successfulItems.toVector, None) state.value.respondEvent(result) @@ -338,7 +338,8 @@ object BuildServerProtocol { val result = JvmTestEnvironmentResult(successfulItems.toVector, None) state.value.respondEvent(result) }.evaluated, - bspBuildTargetJvmEnvironmentItem := jvmEnvironmentItem().value, + bspBuildTargetJvmEnvironmentItem := jvmEnvironmentItem(forkOptions).value, + run / bspBuildTargetJvmEnvironmentItem := jvmEnvironmentItem(run / forkOptions).value, bspInternalDependencyConfigurations := internalDependencyConfigurationsSetting.value, bspScalaTestClassesItem := scalaTestClassesTask.value, bspScalaMainClassesItem := scalaMainClassesTask.value, @@ -770,7 +771,12 @@ object BuildServerProtocol { Def.task(taskImpl(workspace, filter)) } - private def jvmEnvironmentItem(): Initialize[Task[JvmEnvironmentItem]] = Def.task { + private def bspEnvironmentVariables(opts: ForkOptions): Vector[String] = + opts.envVars.map { (k, v) => s"$k=$v" }.toVector + + private def jvmEnvironmentItem( + forkOptions: Initialize[Task[ForkOptions]] + ): Initialize[Task[JvmEnvironmentItem]] = Def.task { val target = Keys.bspTargetIdentifier.value val converter = fileConverter.value val classpath = Keys.fullClasspath.value @@ -778,16 +784,17 @@ object BuildServerProtocol { .map(converter.toPath) .map(_.toFile.toURI) .toVector - val jvmOptions = Keys.javaOptions.value.toVector - val baseDir = Keys.baseDirectory.value.getAbsolutePath - val env = envVars.value + val opts = forkOptions.value + val workingDir = opts.workingDirectory + .getOrElse(new File(sys.props("user.dir"))) + .getAbsolutePath JvmEnvironmentItem( target, classpath, - jvmOptions, - baseDir, - env + opts.runJVMOptions, + workingDir, + opts.envVars ) } @@ -898,7 +905,8 @@ object BuildServerProtocol { val json = jsonParser.parsed val runParams = json.flatMap(Converter.fromJson[RunParams]).get val defaultClass = Keys.mainClass.value - val defaultJvmOptions = Keys.javaOptions.value + val defaultOpts = (run / forkOptions).value + val defaultEnv = bspEnvironmentVariables(defaultOpts) val mainClass = runParams.dataKind match { case Some("scala-main-class") => @@ -910,9 +918,7 @@ object BuildServerProtocol { e.getMessage ) case Success(value) => - value.withEnvironmentVariables( - envVars.value.map { (k, v) => s"$k=$v" }.toVector ++ value.environmentVariables - ) + value.withEnvironmentVariables(defaultEnv ++ value.environmentVariables) } case Some(dataKind) => @@ -930,8 +936,8 @@ object BuildServerProtocol { ) ), runParams.arguments, - defaultJvmOptions.toVector, - envVars.value.map { (k, v) => s"$k=$v" }.toVector + defaultOpts.runJVMOptions, + defaultEnv ) } runMainClassTask(mainClass, runParams.originId) @@ -989,21 +995,18 @@ object BuildServerProtocol { val state = Keys.state.value val logger = Keys.streams.value.log val classpath = Attributed.data(fullClasspath.value) - val forkOpts = ForkOptions( - javaHome = javaHome.value, - outputStrategy = outputStrategy.value, - // bootJars is empty by default because only jars on the user's classpath should be on the boot classpath - bootJars = Vector(), - workingDirectory = Some(baseDirectory.value), - runJVMOptions = mainClass.jvmOptions, - connectInput = connectInput.value, - envVars = mainClass.environmentVariables - .flatMap(_.split("=", 2).toList match { - case key :: value :: Nil => Some(key -> value) - case _ => None - }) - .toMap - ) + // connectInput is disabled so non-interactive BSP output is captured as log messages + val forkOpts = (run / forkOptions).value + .withConnectInput(false) + .withRunJVMOptions(mainClass.jvmOptions) + .withEnvVars( + mainClass.environmentVariables + .flatMap(_.split("=", 2).toList match { + case key :: value :: Nil => Some(key -> value) + case _ => None + }) + .toMap + ) val runner = new ForkRun(forkOpts) val converter = fileConverter.value val cp = classpath.map(converter.toPath) @@ -1073,13 +1076,14 @@ object BuildServerProtocol { } private def scalaMainClassesTask: Initialize[Task[ScalaMainClassesItem]] = Def.task { - val jvmOptions = Keys.javaOptions.value.toVector + val opts = (run / forkOptions).value + val env = bspEnvironmentVariables(opts) val mainClasses = Keys.discoveredMainClasses.value.map( ScalaMainClass( _, Vector(), - jvmOptions, - envVars.value.map { (k, v) => s"$k=$v" }.toVector + opts.runJVMOptions, + env ) ) ScalaMainClassesItem( diff --git a/notes/2.0.0/fork-working-directory.md b/notes/2.0.0/fork-working-directory.md new file mode 100644 index 000000000..b02151264 --- /dev/null +++ b/notes/2.0.0/fork-working-directory.md @@ -0,0 +1,25 @@ +### Forked run starts in sbt's working directory + +Previously, forked `run` set the forked JVM's working directory to the project's +`baseDirectory`, while non-forked `run` executed in the directory sbt itself was +started from. In a multi-project build, toggling `fork` silently changed the +directory that relative paths resolved against. + +sbt 2.x makes forked `run` (and forked `console`) inherit sbt's own working +directory by default, consistent with non-forked execution and with `sbtn` +expectations. Forked `test` is unchanged and keeps the project's `baseDirectory` +as its working directory. The working directory of any forked process can be +configured via `forkOptions`: + +```scala +Compile / run / forkOptions := Def.uncached( + (Compile / run / forkOptions).value.withWorkingDirectory(Some(baseDirectory.value)) +) +``` + +The BSP `buildTarget/jvmRunEnvironment` response reports the same working +directory that `run` uses. + +This addresses [#1032][i1032] for `run`. + +[i1032]: https://github.com/sbt/sbt/issues/1032 diff --git a/notes/2.0.0/migration.md b/notes/2.0.0/migration.md index 18dd31853..7d5d9956f 100644 --- a/notes/2.0.0/migration.md +++ b/notes/2.0.0/migration.md @@ -1,5 +1,17 @@ +## Forked run working directory + +Forked `run` no longer runs in the project's `baseDirectory`; it inherits sbt's +working directory, matching non-forked behavior. Forked `test` is unchanged. To +restore the sbt 1.x behavior: + +```scala +Compile / run / forkOptions := Def.uncached( + (Compile / run / forkOptions).value.withWorkingDirectory(Some(baseDirectory.value)) +) +``` + ## files extension on Classpath ```scala diff --git a/notes/2.0.0/reboot-reconnect.md b/notes/2.0.0/reboot-reconnect.md new file mode 100644 index 000000000..251be0527 --- /dev/null +++ b/notes/2.0.0/reboot-reconnect.md @@ -0,0 +1,15 @@ +### `reboot` works from the thin client again + +Running `reboot` in the sbt shell dropped to the OS shell ("sbt server connection +closed") instead of rebooting. The server's teardown began with a log line that +throws when the terminal in scope is the rebooting client's already-closed +virtual terminal, aborting teardown before the server socket was closed and the +portfile deleted; the relaunched instance then mistook the leaked socket for +another running sbt and never started its server, while the client latched onto +the stale portfile. Server teardown now completes its state cleanup before +logging, and one failing channel shutdown can no longer skip the rest of the +exchange teardown. `reboot` returns to a working prompt. + +This addresses [#9095][i9095]. + +[i9095]: https://github.com/sbt/sbt/issues/9095 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4422e8e67..be3dcd46c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -13,7 +13,7 @@ object Dependencies { // sbt modules val ioVersion = nightlyVersion.getOrElse("1.12.2") - val zincVersion = nightlyVersion.getOrElse("2.0.3") + val zincVersion = nightlyVersion.getOrElse("2.0.4") private val sbtIO = "org.scala-sbt" %% "io" % ioVersion diff --git a/sbt b/sbt index e4f8f6740..e869c7146 100755 --- a/sbt +++ b/sbt @@ -25,7 +25,7 @@ declare use_sbtn= declare use_jvm_client= declare no_server= declare sbtn_command="$SBTN_CMD" -declare sbtn_version="2.0.0-b4d628dd" +declare sbtn_version="2.0.0-a0c4773a" declare use_colors=1 declare is_this_dir_sbt="" declare hide_jdk_warnings=1 diff --git a/sbt-app/src/sbt-test/project/scala-instance/build.sbt b/sbt-app/src/sbt-test/project/scala-instance/build.sbt index cc4dd3aa7..2eff52a1f 100644 --- a/sbt-app/src/sbt-test/project/scala-instance/build.sbt +++ b/sbt-app/src/sbt-test/project/scala-instance/build.sbt @@ -1,4 +1,4 @@ -import Configurations.{ ScalaTool, ScalaDocTool } +import Configurations.{ ScalaTool, ScalaDocTool, ZincTool } @transient lazy val check = taskKey[Unit]("") @@ -6,11 +6,12 @@ lazy val scala213 = "2.13.16" scalaVersion := scala213 autoScalaLibrary := false managedScalaInstance := false -ivyConfigurations ++= List(ScalaTool, ScalaDocTool) +ivyConfigurations ++= List(ScalaTool, ScalaDocTool, ZincTool) libraryDependencies ++= Seq( "org.scala-lang" % "scala-library" % scala213, "org.scala-lang" % "scala-compiler" % scala213 % ScalaTool, "org.scala-lang" % "scala-compiler" % scala213 % ScalaDocTool, + "org.scala-lang" % "scala2-sbt-bridge" % scala213 % ZincTool, ) check := { val si = scalaInstance.value diff --git a/sbt-app/src/sbt-test/remote-cache/basic/Hello.scala b/sbt-app/src/sbt-test/remote-cache/basic/Hello.scala new file mode 100644 index 000000000..8e9836d74 --- /dev/null +++ b/sbt-app/src/sbt-test/remote-cache/basic/Hello.scala @@ -0,0 +1,2 @@ +object Hello: + def main(args: Array[String]): Unit = println("Hello, world!") diff --git a/sbt-app/src/sbt-test/remote-cache/basic/build.sbt b/sbt-app/src/sbt-test/remote-cache/basic/build.sbt new file mode 100644 index 000000000..310376625 --- /dev/null +++ b/sbt-app/src/sbt-test/remote-cache/basic/build.sbt @@ -0,0 +1,23 @@ +import sbt.internal.util.CacheEventSummary + +scalaVersion := "3.8.4" + +Global / remoteCache := Some(new java.net.URI("grpc://127.0.0.1:2024")) +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +val checkHit = taskKey[Unit]("asserts the previous compile was served from the remote cache") + +checkHit := Def.uncached { + val config = Def.cacheConfiguration.value + val prev = config.cacheEventLog.previous match + case data: CacheEventSummary.Data => data + case _ => sys.error("empty event log") + streams.value.log.info( + s"prev hitCount=${prev.hitCount} missCount=${prev.missCount} remoteHitCount=${prev.remoteHitCount}" + ) + assert(prev.missCount == 0, s"expected 100% hit rate but missCount=${prev.missCount}") + assert( + prev.remoteHitCount == prev.hitCount, + s"expected 100% remote hit rate but remoteHitCount=${prev.remoteHitCount} hitCount=${prev.hitCount}" + ) +} diff --git a/sbt-app/src/sbt-test/remote-cache/basic/project/plugins.sbt b/sbt-app/src/sbt-test/remote-cache/basic/project/plugins.sbt new file mode 100644 index 000000000..a41304ee3 --- /dev/null +++ b/sbt-app/src/sbt-test/remote-cache/basic/project/plugins.sbt @@ -0,0 +1 @@ +addRemoteCachePlugin diff --git a/sbt-app/src/sbt-test/remote-cache/basic/test b/sbt-app/src/sbt-test/remote-cache/basic/test new file mode 100644 index 000000000..29e096be4 --- /dev/null +++ b/sbt-app/src/sbt-test/remote-cache/basic/test @@ -0,0 +1,5 @@ +> compile +$ delete diskcache +> clean +> compile +> checkHit diff --git a/sbt-app/src/sbt-test/run/fork/disabled b/sbt-app/src/sbt-test/run/fork/disabled index 8f4fd8c25..60760439f 100644 --- a/sbt-app/src/sbt-test/run/fork/disabled +++ b/sbt-app/src/sbt-test/run/fork/disabled @@ -1,18 +1,29 @@ -> run fork +# non-forked run executes in sbt's working directory +> run $ exists flag $ delete flag -$ mkdir forked +# forked run inherits sbt's working directory by default, +# even when run / baseDirectory points elsewhere (#1032) > set fork := true -> set baseDirectory in run := baseDirectory(_ / "forked").value +> set run / baseDirectory := baseDirectory(_ / "forked").value +> run +$ exists flag +$ absent forked/flag +$ delete flag -> run forked +# run / forkOptions configures the forked working directory +> session clear +> set fork := true +> set Compile / run / forkOptions := Def.uncached((Compile / run / forkOptions).value.withWorkingDirectory(Some(baseDirectory.value / "forked"))) +$ mkdir forked +> run $ exists forked/flag $ absent flag $ delete forked/flag > set envVars += ("flag.name" -> "env.flag") -> run forked +> run $ exists forked/env.flag $ absent flag $ absent forked/flag diff --git a/sbt-app/src/sbt-test/tests/fork-working-directory/build.sbt b/sbt-app/src/sbt-test/tests/fork-working-directory/build.sbt new file mode 100644 index 000000000..dbf929bb1 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/fork-working-directory/build.sbt @@ -0,0 +1,11 @@ +val scalatest = "org.scalatest" %% "scalatest" % "3.2.19" + +ThisBuild / scalaVersion := "3.8.4" + +lazy val root = (project in file(".")) + +lazy val sub = project + .settings( + Test / fork := true, + libraryDependencies += scalatest % Test, + ) diff --git a/sbt-app/src/sbt-test/tests/fork-working-directory/changes/forkdir.sbt b/sbt-app/src/sbt-test/tests/fork-working-directory/changes/forkdir.sbt new file mode 100644 index 000000000..5790d7a56 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/fork-working-directory/changes/forkdir.sbt @@ -0,0 +1,3 @@ +Test / forkOptions := Def.uncached( + (Test / forkOptions).value.withWorkingDirectory(Some((ThisBuild / baseDirectory).value)) +) diff --git a/sbt-app/src/sbt-test/tests/fork-working-directory/sub/src/test/scala/CwdSpec.scala b/sbt-app/src/sbt-test/tests/fork-working-directory/sub/src/test/scala/CwdSpec.scala new file mode 100644 index 000000000..79887d224 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/fork-working-directory/sub/src/test/scala/CwdSpec.scala @@ -0,0 +1,8 @@ +import org.scalatest.funsuite.AnyFunSuite + +class CwdSpec extends AnyFunSuite { + test("create marker in the forked working directory") { + val marker = new java.io.File("cwd-marker").getAbsoluteFile + assert(marker.createNewFile() || marker.exists()) + } +} diff --git a/sbt-app/src/sbt-test/tests/fork-working-directory/test b/sbt-app/src/sbt-test/tests/fork-working-directory/test new file mode 100644 index 000000000..f96568e94 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/fork-working-directory/test @@ -0,0 +1,12 @@ +# a forked test's working directory remains the project's baseDirectory +> sub/testFull +$ exists sub/cwd-marker +$ absent cwd-marker +$ delete sub/cwd-marker + +# Test / forkOptions configures the forked working directory +$ copy-file changes/forkdir.sbt sub/forkdir.sbt +> reload +> sub/testFull +$ exists cwd-marker +$ absent sub/cwd-marker diff --git a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala index bb22de175..defd96654 100644 --- a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala +++ b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala @@ -44,8 +44,10 @@ import sbt.util.{ GetActionResultRequest, UpdateActionResultRequest, } +import scala.collection.concurrent.TrieMap import scala.concurrent.{ Await, ExecutionContext, Future, Promise, TimeoutException } import scala.concurrent.duration.* +import scala.ref.WeakReference import scala.util.Using import scala.util.control.NonFatal import scala.jdk.CollectionConverters.* @@ -57,6 +59,16 @@ object GrpcActionCacheStore: val remoteTimeoutInSec = 60 val remoteTimeout = (remoteTimeoutInSec + 2).second + private case class CacheValue( + rootCerts: Option[Path], + clientCertChain: Option[Path], + clientPrivateKey: Option[Path], + remoteHeaders: List[String], + store: WeakReference[GrpcActionCacheStore], + ) + + private val instances: TrieMap[URI, CacheValue] = TrieMap.empty + def apply( uri: URI, rootCerts: Option[Path], @@ -64,6 +76,36 @@ object GrpcActionCacheStore: clientPrivateKey: Option[Path], remoteHeaders: List[String], disk: DiskActionCacheStore, + ): GrpcActionCacheStore = + def mkStore(): GrpcActionCacheStore = + val store = build(uri, rootCerts, clientCertChain, clientPrivateKey, remoteHeaders, disk) + instances.put( + uri, + CacheValue( + rootCerts, + clientCertChain, + clientPrivateKey, + remoteHeaders, + WeakReference(store) + ) + ) + store + instances.get(uri) match + case Some(v) + if v.rootCerts == rootCerts && v.clientCertChain == clientCertChain + && v.clientPrivateKey == clientPrivateKey && v.remoteHeaders == remoteHeaders => + v.store.get match + case Some(existing) => existing + case None => mkStore() + case _ => mkStore() + + private def build( + uri: URI, + rootCerts: Option[Path], + clientCertChain: Option[Path], + clientPrivateKey: Option[Path], + remoteHeaders: List[String], + disk: DiskActionCacheStore, ): GrpcActionCacheStore = val b: ManagedChannelBuilder[?] = uri.getScheme() match case "grpc" => @@ -96,7 +138,7 @@ object GrpcActionCacheStore: case Some(x) if x.startsWith("/") => x.drop(1) case Some(x) => x case None => "" - new GrpcActionCacheStore(channel, instanceName, remoteHeaders, disk) + new GrpcActionCacheStore(channel, instanceName, remoteHeaders, disk, uri) class AuthCallCredentials(remoteHeaders: List[String]) extends CallCredentials: val pairs = remoteHeaders.map: h => @@ -132,12 +174,14 @@ end GrpcActionCacheStore * https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto * https://github.com/googleapis/googleapis/blob/ff15be54722218705740b9fc6223d264c4cdb6dd/google/bytestream/bytestream.proto */ -class GrpcActionCacheStore( +class GrpcActionCacheStore private ( channel: ManagedChannel, instanceName: String, remoteHeaders: List[String], disk: DiskActionCacheStore, -) extends AbstractActionCacheStore: + cacheKey: URI, +) extends AbstractActionCacheStore + with AutoCloseable: import GrpcActionCacheStore.* lazy val creds = GrpcActionCacheStore.AuthCallCredentials(remoteHeaders) @@ -167,6 +211,17 @@ class GrpcActionCacheStore( val fixedThreadPool = Executors.newFixedThreadPool(100) given ExecutionContext = ExecutionContext.fromExecutor(fixedThreadPool) + override def close(): Unit = + instances.get(cacheKey).foreach { v => + if v.store.get.contains(this) then instances.remove(cacheKey, v) + } + try + try + channel.shutdown() + if !channel.awaitTermination(5, TimeUnit.SECONDS) then channel.shutdownNow() + catch case NonFatal(_) => channel.shutdownNow() + finally fixedThreadPool.shutdown() + /** * https://github.com/bazelbuild/remote-apis/blob/9ff14cecffe5287ba337f857731ceadfc2d80de9/build/bazel/remote/execution/v2/remote_execution.proto#L170 */ diff --git a/server-test/src/server-test/buildserver/build.sbt b/server-test/src/server-test/buildserver/build.sbt index 62465c8e5..02608cd43 100644 --- a/server-test/src/server-test/buildserver/build.sbt +++ b/server-test/src/server-test/buildserver/build.sbt @@ -9,6 +9,9 @@ lazy val runAndTest = project.in(file("run-and-test")) libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.8" % "test", Compile / javaOptions := Vector("Xmx256M"), Compile / envVars := Map("KEY" -> "VALUE"), + Compile / run / forkOptions := Def.uncached( + (Compile / run / forkOptions).value.withWorkingDirectory(Some(baseDirectory.value)) + ), Test / javaOptions := Vector("Xmx512M"), Test / envVars := Map("KEY_TEST" -> "VALUE_TEST"), diff --git a/server-test/src/test/scala/testpkg/RebootTest.scala b/server-test/src/test/scala/testpkg/RebootTest.scala new file mode 100644 index 000000000..23a7c60db --- /dev/null +++ b/server-test/src/test/scala/testpkg/RebootTest.scala @@ -0,0 +1,67 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package testpkg + +import java.io.{ InputStream, PrintStream } +import java.util.concurrent.{ LinkedBlockingQueue, TimeUnit, TimeoutException } +import sbt.internal.client.NetworkClient +import sbt.internal.util.Util + +/** + * Regression for https://github.com/sbt/sbt/issues/9095: `reboot` from a client must bring the + * server back and complete instead of leaving a zombie server that drops the client. + */ +class RebootTest extends AbstractServerTest { + override val testDirectory: String = "client" + + private object BlockingInputStream extends InputStream { + override def read(): Int = { + try Thread.sleep(Long.MaxValue) + catch { case _: InterruptedException => } + -1 + } + } + private val nullPrintStream = new PrintStream(_ => {}, false) + + private def background[R](f: => R): R = { + val result = new LinkedBlockingQueue[Either[Throwable, R]] + val thread = new Thread("reboot-test-client") { + setDaemon(true) + override def run(): Unit = + try Util.ignoreResult(result.put(Right(f))) + catch { case e: Throwable => Util.ignoreResult(result.put(Left(e))) } + } + thread.start() + result.poll(3, TimeUnit.MINUTES) match { + case null => + thread.interrupt() + thread.join(10000) + throw new TimeoutException("client did not complete within 3 minutes") + case Left(e) => throw e + case Right(r) => r + } + } + + private def client(args: String*): Int = + background( + NetworkClient.client( + testPath.toFile, + args.toArray, + BlockingInputStream, + nullPrintStream, + nullPrintStream, + false + ) + ) + + test("reboot completes and the rebooted server serves the next command") { + assert(client("reboot") == 0, "reboot from a client must complete with exit 0") + assert(client("willSucceed") == 0, "the rebooted server must serve a new client connection") + } +} diff --git a/util-cache/src/main/scala/sbt/internal/util/CacheEventLog.scala b/util-cache/src/main/scala/sbt/internal/util/CacheEventLog.scala index 16c88b39f..6e026eca3 100644 --- a/util-cache/src/main/scala/sbt/internal/util/CacheEventLog.scala +++ b/util-cache/src/main/scala/sbt/internal/util/CacheEventLog.scala @@ -18,7 +18,8 @@ enum CacheEventSummary: missCount: Long, hitRate: Double, onsiteCount: Option[Long], - errorCount: Option[Long] + errorCount: Option[Long], + remoteHitCount: Long ) override def toString(): String = this match case Empty => "" @@ -28,7 +29,8 @@ enum CacheEventSummary: missCount, hitRate, onsiteCount, - errorCount + errorCount, + remoteHitCount ) => val hitDescs = hits.map { case (id, 1) => s"1 $id cache hit" @@ -74,12 +76,14 @@ class CacheEventLog: val hitRate = if total > 0 then (hitCount.toDouble / total.toDouble) else 0.0 val onsiteCount = events.get(ActionCacheEvent.OnsiteTask) val errorCount = events.get(ActionCacheEvent.Error) + val remoteHitCount = hits.view.collect { case (id, v) if id.startsWith("remote") => v }.sum CacheEventSummary.Data( hits.toSeq, hitCount, missCount, hitRate, onsiteCount, - errorCount + errorCount, + remoteHitCount ) end CacheEventLog diff --git a/util-cache/src/test/scala/sbt/util/CacheEventLogTest.scala b/util-cache/src/test/scala/sbt/util/CacheEventLogTest.scala index ffd7310fc..2b3e89515 100644 --- a/util-cache/src/test/scala/sbt/util/CacheEventLogTest.scala +++ b/util-cache/src/test/scala/sbt/util/CacheEventLogTest.scala @@ -1,6 +1,6 @@ package sbt.util -import sbt.internal.util.{ ActionCacheEvent, CacheEventLog } +import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, CacheEventSummary } import verify.BasicTestSuite object CacheEventLogTest extends BasicTestSuite: @@ -59,6 +59,9 @@ object CacheEventLogTest extends BasicTestSuite: logger.append(ActionCacheEvent.OnsiteTask) val expectedSummary = "cache 75%, 1 disk cache hit, 2 remote cache hits, 1 onsite task" assertEquals(logger.summary.toString(), expectedSummary) + logger.summary match + case data: CacheEventSummary.Data => assert(data.remoteHitCount == 2L) + case _ => sys.error("expected CacheEventSummary.Data") } test("summary of 1 disk event after clear") { diff --git a/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincLmUtil.scala b/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincLmUtil.scala index 7a8b3b53a..2151745b5 100644 --- a/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincLmUtil.scala +++ b/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincLmUtil.scala @@ -15,10 +15,8 @@ import sbt.librarymanagement.{ DependencyResolution, ModuleID, ScalaArtifacts, - SemanticSelector, UnresolvedWarningConfiguration, UpdateConfiguration, - VersionNumber, } import sbt.librarymanagement.syntax.* import xsbti.ArtifactInfo.SbtOrganization @@ -27,11 +25,9 @@ import xsbti.compile.{ ClasspathOptions, ScalaInstance as XScalaInstance } object ZincLmUtil { - final val scala2SbtBridgeStart = "2.13.12" + final val scala2SbtBridgeStart = ScalaArtifacts.scala2SbtBridgeStart def hasScala2SbtBridge(sv: String): Boolean = - VersionNumber(sv).matchesSemVer( - SemanticSelector(s"=2.13 >=$scala2SbtBridgeStart") - ) + ScalaArtifacts.hasScala2SbtBridge(sv) /** * Instantiate a Scala compiler that is instrumented to analyze dependencies.