From 81b81ce6b0ef697e141134a66dfcab58ec9c2c8c Mon Sep 17 00:00:00 2001 From: Dream <42954461+eureka0928@users.noreply.github.com> Date: Thu, 16 Apr 2026 01:37:24 -0400 Subject: [PATCH] [2.x] fix: Fix consoleProject for Scala 3 (#9073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem** consoleProject does not work with Scala 3 because the compiler bridge does not implement REPL binding injection (scala/scala3#5069). The bindings currentState, extracted, and cpHelpers are never injected into the REPL session, causing Not found errors. **Solution** Work around the missing binding support by storing the three runtime objects in a static holder (ConsoleProjectBindings) before launching the REPL, and generating val definitions via initialCommands that read from the holder. The original bindings are still passed to Console to preserve Scala 2 backward compatibility. Pass -Xrepl-interrupt-instrumentation:local to the REPL when the consoleProject scala instance is 3.8 or newer. In local mode the instrumented loader skips re-defining classpath classes and falls through to standard parent-first delegation, so REPL code sees the same singleton sbt.* and scala.* classes as the surrounding sbt process — while still keeping interrupt support for REPL-defined code, preserving Ctrl+C on long-running expressions like (Compile / compile).eval. --------- Co-authored-by: Claude Opus 4.6 (1M context) --- main/src/main/scala/sbt/Defaults.scala | 55 ++++++-- .../main/scala/sbt/internal/Compiler.scala | 69 ++++++++++ .../scala/sbt/internal/ConsoleProject.scala | 126 ++++++++++++++++-- .../sbt/internal/ConsoleProjectBindings.scala | 45 +++++++ .../project/consoleProject/{pending => test} | 0 .../project/scala3-console-project/build.sbt | 23 +++- .../project/scala3-console-project/pending | 1 - .../project/scala3-console-project/test | 2 + 8 files changed, 295 insertions(+), 26 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/ConsoleProjectBindings.scala rename sbt-app/src/sbt-test/project/consoleProject/{pending => test} (100%) delete mode 100644 sbt-app/src/sbt-test/project/scala3-console-project/pending create mode 100644 sbt-app/src/sbt-test/project/scala3-console-project/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 90b9a5655..7ec2d417b 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -790,6 +790,28 @@ object Defaults extends BuildCommon with DefExtra { ScalaArtifacts.Organization, appConfiguration.value.provider.scalaProvider.version ), + // consoleProject compiles the *build definition*, which targets sbt's own + // Scala version (the launcher's scalaProvider.version), not the project's. + // The default `scalaCompilerBridgeBin` resolves via the `update` report, + // which uses the project's scalaVersion; resolve the pre-built Scala 3 + // bridge directly for sbt's Scala version instead. See sbt/sbt#7722. + consoleProject / scalaCompilerBridgeBin := Def.uncached { + val sv = appConfiguration.value.provider.scalaProvider.version + val st = state.value + val g = BuildPaths.getGlobalBase(st) + val zincDir = BuildPaths.getZincDirectory(st, g) + val conv = fileConverter.value + Compiler + .scala3ConsoleProjectBridgeJar( + scalaVersion = sv, + scalaOrg = ScalaArtifacts.Organization, + dr = (LocalRootProject / dependencyResolution).value, + retrieveDir = zincDir, + log = streams.value.log, + ) + .toVector + .map(jar => conv.toVirtualFile(jar.toPath()): HashedVirtualFileRef) + }, classpathOptions := ClasspathOptionsUtil.noboot(scalaVersion.value), console / classpathOptions := ClasspathOptionsUtil.replNoboot(scalaVersion.value), ) @@ -811,22 +833,35 @@ object Defaults extends BuildCommon with DefExtra { ), consoleProject := ConsoleProject.consoleProjectTask.value, consoleProject / scalaInstance := { - val topLoader = classOf[org.jline.terminal.Terminal].getClassLoader + // Use the classloader that has `ConsoleProjectBindings` as the top so + // that code generated by `initialCommands` in the REPL sees the same + // class instance that sbt has set fields on. See sbt/sbt#7722. + val topLoader = classOf[ConsoleProjectBindings.type].getClassLoader val scalaProvider = appConfiguration.value.provider.scalaProvider - val allJars = scalaProvider.jars - val libraryJars = allJars.filter { jar => + val launcherJars = scalaProvider.jars + val libraryJars = launcherJars.filter { jar => jar.getName == "scala-library.jar" || jar.getName.startsWith("scala3-library_3") } - val compilerJar = allJars.filter { jar => - jar.getName == "scala-compiler.jar" || jar.getName.startsWith("scala3-compiler_3") - } - ScalaInstance(scalaProvider.version, scalaProvider.launcher) + // Scala 3.8+ extracted ReplDriver from scala3-compiler_3 into scala3-repl_3, + // which the sbt launcher does not ship. Resolve it explicitly so that + // consoleProject can start the Scala 3 REPL. See sbt/sbt#7722. + val st = state.value + val g = BuildPaths.getGlobalBase(st) + val zincDir = BuildPaths.getZincDirectory(st, g) + val replToolJars = Compiler.scala3ReplToolJars( + scalaVersion = scalaProvider.version, + scalaOrg = ScalaArtifacts.Organization, + dr = (LocalRootProject / dependencyResolution).value, + retrieveDir = zincDir, + log = streams.value.log, + ) + val allJars = (launcherJars.toSeq ++ replToolJars).distinct Compiler.makeScalaInstance( scalaProvider.version, libraryJars, - allJars.toSeq, - Seq.empty, - state.value, + allJars, + replToolJars, + st, topLoader, ) }, diff --git a/main/src/main/scala/sbt/internal/Compiler.scala b/main/src/main/scala/sbt/internal/Compiler.scala index 57e9c1604..879a4bf44 100644 --- a/main/src/main/scala/sbt/internal/Compiler.scala +++ b/main/src/main/scala/sbt/internal/Compiler.scala @@ -27,6 +27,8 @@ import sbt.librarymanagement.{ Configuration, Configurations, ConfigurationReport, + CrossVersion, + DependencyResolution, ModuleID, ScalaArtifacts, SemanticSelector, @@ -110,6 +112,73 @@ object Compiler: case _ => ScalaInstance(sv, scalaProvider) } + /** + * Resolves extra REPL tool jars required for Scala 3.8+. + * + * In Scala 3.8+, `dotty.tools.repl.ReplDriver` was extracted from + * `scala3-compiler_3` into a new artifact `scala3-repl_3`. The sbt launcher's + * Scala provider only includes `scala3-compiler_3`, so we need to resolve + * `scala3-repl_3` explicitly for `consoleProject` to work. + * + * Returns an empty sequence for Scala versions < 3.8. + * + * @see https://github.com/sbt/sbt/issues/7722 + * @see https://github.com/scala/scala3/pull/24243 + */ + private[sbt] def scala3ReplToolJars( + scalaVersion: String, + scalaOrg: String, + dr: DependencyResolution, + retrieveDir: File, + log: Logger + ): Seq[File] = + if !ScalaArtifacts.isScala3_8Plus(scalaVersion) then Nil + else + // Scala 3 artifacts use the `_3` suffix, not `_3.8`. Bake the suffix into + // the artifact name and disable cross-version resolution, because the + // caller's scalaModuleInfo may be None or reflect a different project + // Scala version than the one we actually need. + val replModule = ModuleID(scalaOrg, s"${ScalaArtifacts.Scala3ReplID}_3", scalaVersion) + .withCrossVersion(CrossVersion.disabled) + dr.retrieve(replModule, scalaModuleInfo = None, retrieveDir, log) match + case Right(resolved) => resolved.toSeq + case Left(unresolved) => + log.warn( + s"Could not resolve $replModule for consoleProject; REPL may fail to start: ${unresolved.resolveException.getMessage}" + ) + Nil + + /** + * Resolves the pre-built Scala 3 compiler bridge jar for `consoleProject`. + * + * The default `scalaCompilerBridgeBin` uses the project's Scala version, but + * `consoleProject` compiles the build definition with sbt's own Scala + * version. Resolve the pre-built bridge directly for that version to keep + * the bridge consistent with `consoleProject / scalaInstance`. + * + * The bridge is a Java-compiled jar (cross-version disabled). + * + * @see https://github.com/sbt/sbt/issues/7722 + */ + private[sbt] def scala3ConsoleProjectBridgeJar( + scalaVersion: String, + scalaOrg: String, + dr: DependencyResolution, + retrieveDir: File, + log: Logger + ): Option[File] = + if !ScalaArtifacts.isScala3(scalaVersion) then None + else + val bridgeModule = ModuleID(scalaOrg, "scala3-sbt-bridge", scalaVersion) + .withCrossVersion(CrossVersion.disabled) + dr.retrieve(bridgeModule, scalaModuleInfo = None, retrieveDir, log) match + case Right(resolved) => resolved.find(_.getName.startsWith("scala3-sbt-bridge")) + case Left(unresolved) => + log.warn( + s"Could not resolve $bridgeModule for consoleProject: ${unresolved.resolveException.getMessage}" + ) + None + def scalaInstanceConfigFromHome(dir: File): Def.Initialize[Task[ScalaInstanceConfig]] = Def.task { val dummy = ScalaInstance(dir)(Keys.state.value.classLoaderCache.apply) diff --git a/main/src/main/scala/sbt/internal/ConsoleProject.scala b/main/src/main/scala/sbt/internal/ConsoleProject.scala index 35fd97906..08a0f588a 100644 --- a/main/src/main/scala/sbt/internal/ConsoleProject.scala +++ b/main/src/main/scala/sbt/internal/ConsoleProject.scala @@ -53,7 +53,9 @@ object ConsoleProject: ): Unit = { val extracted = Project.extract(state) val cpImports = new Imports(extracted, state) - // Bindings are blocked by https://github.com/scala/scala3/issues/5069 + // Bindings are ignored by Scala 3 bridge: https://github.com/scala/scala3/issues/5069 + // Workaround: vals are injected via initialCommands from ConsoleProjectBindings holder. + // bindings are still passed to Console for Scala 2 backward compatibility. val bindings = ("currentState" -> state) :: ("extracted" -> extracted) :: ("cpHelpers" -> cpImports) :: Nil val unit = extracted.currentUnit @@ -86,22 +88,120 @@ object ConsoleProject: classLoaderCache = state.get(BasicKeys.classLoaderCache), log = log ) - val imports = BuildUtil.getImports(unit.unit) ++ BuildUtil.importAll(bindings.map(_._1)) - val importString = imports.mkString("", ";\n", ";\n\n") - val initCommands = importString + extra - val loader = ClasspathUtil.makeLoader(unit.classpath, si, tempDir) + ConsoleProjectBindings.set(state, extracted, cpImports) + val baseImports = BuildUtil.getImports(unit.unit) + val bindingDefs = Seq( + "val currentState = _root_.sbt.internal.ConsoleProjectBindings.state", + "val extracted = _root_.sbt.internal.ConsoleProjectBindings.extracted", + "val cpHelpers = _root_.sbt.internal.ConsoleProjectBindings.cpHelpers", + ) + val bindingImports = BuildUtil.importAll(bindings.map(_._1)) + val allLines = baseImports ++ bindingDefs ++ bindingImports + val initCommands = allLines.mkString("", ";\n", ";\n\n") + extra + // Two things are required so the REPL resolves `sbt.*` (e.g. + // `sbt.TaskKey`, `sbt.Keys`, `sbt.State`, `sbt.internal.ConsoleProjectBindings`) + // and `scala.*` (e.g. `scala.Function2`) via the *same* class objects + // that sbt itself uses. See sbt/sbt#7722. + // + // 1. Remove sbt's own module jars from the runtime URL classloader, + // so that `sbt.*` references resolve via parent delegation back + // to sbt's `MetaBuildLoader` rather than being defined a second + // time by the REPL's URL classloader (which would break the + // `ConsoleProjectBindings` singleton's static state and + // trigger `LinkageError: loader constraint violation` when REPL + // code touches a method whose signature mentions a duplicated + // type — e.g. `sbt.TaskKey.zipWith(_, scala.Function2)`). + // + // 2. On Scala 3.8+ switch the REPL's bytecode interrupt + // instrumentation to `local` mode. The default (`true`) for + // `dotty.tools.repl.AbstractFileClassLoader` (added in 3.8) + // reads every class's bytes from the parent loader via + // `getResourceAsStream` and `defineClass`-es them a *second* + // time inside the REPL loader, producing duplicate `Class` + // objects for every `sbt.*` and `scala.*` class. `local` skips + // that re-definition and falls through to standard parent-first + // delegation (so the REPL sees the same singleton classes as + // the surrounding sbt process) while still keeping interrupt + // support for REPL-defined code — preserving Ctrl+C for long- + // running expressions like `(Compile / compile).eval`. The flag + // does not exist on Scala 3.7 and earlier (which use the older + // AFClassLoader without instrumentation), so we only pass it + // when the consoleProject scala instance is 3.8+ to avoid a + // "bad option" warning. + // + // The full classpath is still passed to `Console` below so the REPL's + // compile-time classpath is unchanged. + val runtimeClasspath = unit.classpath.filterNot(isSbtModuleJar) + val loader = ClasspathUtil.makeLoader(runtimeClasspath, si, tempDir) + val replOptions = + if needsInterruptInstrumentationOptOut(si.version) then + "-Xrepl-interrupt-instrumentation:local" +: options + else options val terminal = Terminal.get // TODO - Hook up dsl classpath correctly... - (new Console(compiler))( - unit.classpath.map(_.toFile), - options, - initCommands, - cleanupCommands, - terminal - )(Some(loader), bindings).get - () + try + (new Console(compiler))( + unit.classpath.map(_.toFile), + replOptions, + initCommands, + cleanupCommands, + terminal + )(Some(loader), bindings).get + () + finally ConsoleProjectBindings.clear() } + /** + * `dotty.tools.repl.AbstractFileClassLoader`'s bytecode interrupt + * instrumentation was added in Scala 3.8 and is enabled by default — + * see Scala 3 PR scala/scala3#22720. The setting that disables it is + * also a 3.8+ addition (see `Xrepl-interrupt-instrumentation` in + * `dotty.tools.dotc.config.ScalaSettings`). For 3.7 and earlier the + * REPL classloader doesn't re-define classes locally, so the flag is + * unnecessary and would only produce a "bad option" warning. + */ + private def needsInterruptInstrumentationOptOut(scalaVersion: String): Boolean = + scalaVersion match + case s"3.$rest" => + rest.takeWhile(_.isDigit).toIntOption.exists(_ >= 8) + case _ => false + + /** + * Returns true when a `Path` refers to a jar published by + * `org.scala-sbt`. These jars ship sbt's own classes (e.g. `sbt.State`, + * `sbt.TaskKey`, `sbt.Keys`) that are already reachable via the parent + * class loader used by `consoleProject`. They must be excluded from the + * REPL's runtime classloader so that `sbt.*` references resolve via + * parent delegation and reach sbt's singleton copies — rather than + * being defined fresh by the URL classloader from `unit.classpath`, + * which would trigger a `LinkageError: loader constraint violation` + * whenever those classes are used from the REPL. See sbt/sbt#7722. + * + * Detection is done via `META-INF/MANIFEST.MF`'s `Implementation-Vendor-Id` + * attribute, which all sbt module jars set to `org.scala-sbt`. This is + * more robust than checking for specific class entries, because it + * uniformly catches every sbt module (main, main-settings, command, io, + * util-*, etc.) without enumerating them. + */ + private def isSbtModuleJar(p: java.nio.file.Path): Boolean = + val name = p.getFileName.toString + if !name.endsWith(".jar") || !java.nio.file.Files.isRegularFile(p) then false + else + try + val zf = new java.util.zip.ZipFile(p.toFile) + try + val entry = zf.getEntry("META-INF/MANIFEST.MF") + if entry eq null then false + else + val is = zf.getInputStream(entry) + try + val manifest = new java.util.jar.Manifest(is) + val attrs = manifest.getMainAttributes + attrs != null && "org.scala-sbt" == attrs.getValue("Implementation-Vendor-Id") + finally is.close() + finally zf.close() + catch case _: java.io.IOException => false + /** Conveniences for consoleProject that shouldn't normally be used for builds. */ final class Imports private[sbt] (extracted: Extracted, state: State) { import extracted.* diff --git a/main/src/main/scala/sbt/internal/ConsoleProjectBindings.scala b/main/src/main/scala/sbt/internal/ConsoleProjectBindings.scala new file mode 100644 index 000000000..1e4e69fba --- /dev/null +++ b/main/src/main/scala/sbt/internal/ConsoleProjectBindings.scala @@ -0,0 +1,45 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import scala.compiletime.uninitialized + +/** + * Static holder for consoleProject bindings. + * + * Scala 3 compiler bridge does not implement REPL binding injection + * (https://github.com/scala/scala3/issues/5069), so sbt generates `val` + * definitions in `initialCommands` that read from this holder. + * + * The holder has to be resolved to the same JVM `Class` object from both + * sbt itself and the Scala 3 REPL's `AbstractFileClassLoader`. sbt's + * own sbt module jars are removed from the REPL runtime classloader in + * `ConsoleProject.apply`, so that all `sbt.*` references in the REPL go + * through the parent chain and reach the sbt singleton. + */ +object ConsoleProjectBindings: + @volatile private var _state: State = uninitialized + @volatile private var _extracted: Extracted = uninitialized + @volatile private var _cpHelpers: ConsoleProject.Imports = uninitialized + + def set(state: State, extracted: Extracted, cpHelpers: ConsoleProject.Imports): Unit = + _state = state + _extracted = extracted + _cpHelpers = cpHelpers + + def clear(): Unit = + _state = null + _extracted = null + _cpHelpers = null + + def state: State = _state + def extracted: Extracted = _extracted + def cpHelpers: ConsoleProject.Imports = _cpHelpers +end ConsoleProjectBindings diff --git a/sbt-app/src/sbt-test/project/consoleProject/pending b/sbt-app/src/sbt-test/project/consoleProject/test similarity index 100% rename from sbt-app/src/sbt-test/project/consoleProject/pending rename to sbt-app/src/sbt-test/project/consoleProject/test diff --git a/sbt-app/src/sbt-test/project/scala3-console-project/build.sbt b/sbt-app/src/sbt-test/project/scala3-console-project/build.sbt index 208b8c684..4ce54ef9f 100644 --- a/sbt-app/src/sbt-test/project/scala3-console-project/build.sbt +++ b/sbt-app/src/sbt-test/project/scala3-console-project/build.sbt @@ -1,3 +1,22 @@ -ThisBuild / scalaVersion := "3.0.0-M2" +ThisBuild / scalaVersion := "3.8.3" -lazy val root = project.in(file(".")) +lazy val markerFile = settingKey[java.io.File]("marker file written by consoleProject REPL when bindings resolve") + +lazy val root = project.in(file(".")).settings( + markerFile := target.value / "console-bindings-ok", + Global / initialCommands := { + val path = markerFile.value.getAbsolutePath.replace("\\", "\\\\") + // Exercise the exact code paths that hit `LinkageError: loader + // constraint violation` in earlier iterations of sbt/sbt#7722: + // 1. Resolving `sbt.Keys.compile` loads `sbt.TaskKey` through the + // REPL's classloader chain (first regression, pre-#9073). + // 2. Calling `TaskKey.zipWith(_, Function2)` tripped on + // `scala.Function2` being defined twice (second regression, + // reported on PR #9073 after the initial fix). + // If either resolution fails, the marker file is never written. + s"""val _compileKey = _root_.sbt.Keys.compile + |val _zipped = _compileKey.zipWith(_compileKey)((_, _) => 0) + |_root_.java.nio.file.Files.writeString(_root_.java.nio.file.Paths.get("$path"), currentState.toString.length.toString + "/" + extracted.toString.length.toString + "/" + cpHelpers.toString.length.toString + "/" + _compileKey.key.label + "/" + _zipped.getClass.getName) + |""".stripMargin + }, +) diff --git a/sbt-app/src/sbt-test/project/scala3-console-project/pending b/sbt-app/src/sbt-test/project/scala3-console-project/pending deleted file mode 100644 index b80db65f1..000000000 --- a/sbt-app/src/sbt-test/project/scala3-console-project/pending +++ /dev/null @@ -1 +0,0 @@ -> consoleProject diff --git a/sbt-app/src/sbt-test/project/scala3-console-project/test b/sbt-app/src/sbt-test/project/scala3-console-project/test new file mode 100644 index 000000000..02e890c91 --- /dev/null +++ b/sbt-app/src/sbt-test/project/scala3-console-project/test @@ -0,0 +1,2 @@ +> consoleProject +$ exists target/out/**/console-bindings-ok