Merge pull request #9869 from eed3si9n/wip/backports

[2.x] Backports
This commit is contained in:
eugene yokota
2026-10-05 16:31:46 -04:00
committed by GitHub
20 changed files with 316 additions and 34 deletions
+1
View File
@@ -620,6 +620,7 @@ lazy val actionsProj = (project in file("main-actions"))
// an internal (sbt.internal) type not meant for external consumption.
exclude[DirectMissingMethodProblem]("sbt.internal.WorkerConnection.valueOf"),
exclude[DirectMissingMethodProblem]("sbt.internal.WorkerConnection.values"),
exclude[DirectMissingMethodProblem]("sbt.internal.*"),
),
)
.dependsOn(lmCore)
@@ -13,7 +13,7 @@ import java.util.Locale
import scala.collection.concurrent.TrieMap
import scala.reflect.Selectable.reflectiveSelectable
import scala.util.Properties
import scala.util.{ Properties, Try }
object Util:
def makeList[T](size: Int, value: T): List[T] = List.fill(size)(value)
@@ -50,6 +50,8 @@ object Util:
()
}
def ignoreTry[A](f: => A): Unit = ignoreResult(Try(f))
lazy val isMac: Boolean =
System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("mac")
@@ -9,10 +9,11 @@ package sbt
package internal
package librarymanagement
import java.io.File
import java.util.Locale
import scala.util.control.NonFatal
object LMSysProp {
object LMSysProp:
def booleanOpt(name: String): Option[Boolean] =
sys.props.get(name).flatMap { x =>
x.toLowerCase(Locale.ENGLISH) match {
@@ -63,4 +64,12 @@ object LMSysProp {
lazy val maxPublishAttempts: Int =
java.lang.Integer.getInteger("sbt.repository.publish.attempts", 3)
}
/**
* Ivy home directory: `sbt.ivy.home`, then `ivy.home`, else `user.home/.ivy2`.
*/
lazy val defaultIvyHome: File =
def prop(name: String): Option[String] = sys.props.get(name).map(_.trim).filter(_.nonEmpty)
prop("sbt.ivy.home").orElse(prop("ivy.home")) match
case Some(home) => File(home)
case None => File(sbt.io.Path.userHome, ".ivy2")
end LMSysProp
@@ -8,6 +8,7 @@ import coursier.core.*
import coursier.params.rule.Strict
import lmcoursier.FallbackDependency
import lmcoursier.definitions.ToCoursier
import sbt.internal.librarymanagement.LMSysProp
import coursier.util.Task
import scala.collection.mutable
@@ -114,7 +115,7 @@ object ResolutionParams {
val ivyHome = sys.props
.get("ivy.home")
.orElse(ivyHomeOpt.map(_.getAbsoluteFile.toURI.getPath))
.getOrElse(new File(sys.props("user.home")).toURI.getPath + ".ivy2")
.getOrElse(LMSysProp.defaultIvyHome.getAbsoluteFile.toURI.getPath)
val sbtIvyHome = sys.props.getOrElse(
"sbt.ivy.home",
@@ -1,6 +1,7 @@
package lmcoursier.internal
import coursier.{ Resolution, Resolve }
import coursier.{ CoursierEnv, Resolution, Resolve }
import coursier.cache.CacheEnv
import coursier.cache.internal.ThreadUtil
import coursier.cache.loggers.{ FallbackRefreshDisplay, ProgressBarRefreshDisplay, RefreshLogger }
import coursier.core.*
@@ -8,8 +9,9 @@ import coursier.error.ResolutionError
import coursier.error.ResolutionError.CantDownloadModule
import coursier.ivy.IvyRepository
import coursier.maven.MavenRepositoryLike
import coursier.params.Mirror
import coursier.params.rule.RuleResolution
import coursier.util.Task
import coursier.util.{ EnvValues, Task }
import sbt.util.Logger
import scala.annotation.nowarn
@@ -19,6 +21,35 @@ import scala.collection.mutable
// private[coursier]
object ResolutionRun {
/**
* Mirrors from the coursier configuration, plus the ones of Maven's settings.xml
* only when COURSIER_MAVEN_SETTINGS (or coursier.maven-settings) is set explicitly,
* since a settings.xml mirror of `*` would otherwise shadow the resolvers of the build.
* See https://github.com/sbt/sbt/issues/9821
*/
lazy val defaultMirrors: Seq[Mirror] =
CoursierEnv.defaultMirrors(
CoursierEnv.mirrors.read(),
CoursierEnv.mirrorsExtra.read(),
CoursierEnv.scalaCliConfig.read(),
CacheEnv.configDir.read()
) ++
mavenSettingsMirrors(
CoursierEnv.mavenSettings.read(),
CoursierEnv.mavenHome.read(),
CoursierEnv.mavenHomeFallback.read()
)
private[lmcoursier] def mavenSettingsMirrors(
mavenSettings: EnvValues,
mavenHome: EnvValues,
mavenHomeFallback: EnvValues
): Seq[Mirror] =
val explicit = mavenSettings.env.orElse(mavenSettings.prop).exists(_.trim.nonEmpty)
if explicit then
CoursierEnv.defaultMavenSettingsMirrors(mavenSettings, mavenHome, mavenHomeFallback)
else Nil
private def resolution(
params: ResolutionParams,
verbosityLevel: Int,
@@ -103,6 +134,7 @@ object ResolutionRun {
},
boms = params.boms,
repositories = repositories,
mirrors = defaultMirrors,
resolutionParams = params.params
.addForceVersion(
(if (isSandboxConfig) Nil
@@ -1,11 +1,16 @@
package lmcoursier.internal
import coursier.CoursierEnv
import coursier.core.{ Module, ModuleName, Organization, Resolution }
import coursier.error.ResolutionError.CantDownloadModule
import coursier.util.EnvValues
import coursier.version.VersionConstraint
import sbt.io.IO
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import java.io.File
class ResolutionRunSpec extends AnyFunSuite with Matchers:
private def cantDownload(errors: String*): CantDownloadModule =
@@ -37,3 +42,48 @@ class ResolutionRunSpec extends AnyFunSuite with Matchers:
"Server returned HTTP response code: 404 for URL: https://repo.example.com/org/mod/1.0/mod-1.0.pom"
)
ResolutionRun.isTransientResolutionError(err) shouldBe false
private val unset = EnvValues(None, None)
private def withMirrorOfAll(f: File => Unit): Unit =
IO.withTemporaryDirectory: dir =>
IO.write(
new File(dir, "settings.xml"),
"""<settings>
| <mirrors>
| <mirror>
| <id>internal</id>
| <url>https://nexus.example.com/repository/maven-public</url>
| <mirrorOf>*</mirrorOf>
| </mirror>
| </mirrors>
|</settings>
|""".stripMargin
)
f(dir)
test("settings.xml mirrors are ignored unless the Maven settings are set explicitly"):
withMirrorOfAll: dir =>
val mavenHome = EnvValues(Some(dir.getAbsolutePath), None)
assert(CoursierEnv.defaultMavenSettingsMirrors(unset, mavenHome, unset).nonEmpty)
assert(ResolutionRun.mavenSettingsMirrors(unset, mavenHome, unset).isEmpty)
test("settings.xml mirrors are read when the Maven settings are set explicitly"):
withMirrorOfAll: dir =>
val settings = new File(dir, "settings.xml").getAbsolutePath
assert(
ResolutionRun.mavenSettingsMirrors(EnvValues(Some(settings), None), unset, unset).size == 1
)
assert(
ResolutionRun.mavenSettingsMirrors(EnvValues(None, Some(settings)), unset, unset).size == 1
)
val mavenHome = EnvValues(Some(dir.getAbsolutePath), None)
assert(
ResolutionRun
.mavenSettingsMirrors(EnvValues(Some("true"), None), mavenHome, unset)
.size == 1
)
assert(
ResolutionRun.mavenSettingsMirrors(EnvValues(Some("false"), None), mavenHome, unset).isEmpty
)
end ResolutionRunSpec
@@ -24,8 +24,9 @@ import scala.sys.process.{ BasicIO, Process, ProcessIO }
import scala.collection.mutable
import scala.collection.concurrent.TrieMap
import scala.collection.mutable.ListBuffer
import scala.concurrent.{ Await, Promise }
import scala.concurrent.{ Await, Future, Promise }
import scala.concurrent.duration.*
import scala.util.Try
import scala.util.control.NonFatal
object WorkerExchange:
@@ -83,11 +84,13 @@ object WorkerExchange:
IO.classLocationPath(classOf[Gson]).toFile,
)
val inputRef = Promise[OutputStream]()
val responsesRead = Promise[Unit]()
def runAccepter(out: OutputStream, in: InputStream): Unit =
inputRef.success(out)
val scanner = Scanner(in, "UTF-8")
while scanner.hasNextLine() do notifyListeners(scanner.nextLine())
val (connArgs, closer): (Seq[String], Option[AutoCloseable]) = connectionType match
responsesRead.trySuccess(())
val (connArgs, closer) = connectionType match
case WorkerConnection.Tcp =>
val serverSocket = Retry(ServerSocket(0, 1, loopback))
val accepter = Thread(() => {
@@ -130,20 +133,24 @@ object WorkerExchange:
val onStdoutLine: String => Unit = connectionType match
case WorkerConnection.Stdio => notifyListeners
case _ => (line) => scala.Console.out.println(line)
def readStdout(stdout: InputStream): Unit =
BasicIO.processFully(onStdoutLine)(stdout)
if connectionType == WorkerConnection.Stdio then responsesRead.trySuccess(())
val processIo = ProcessIO(
in = (input) =>
(connectionType match
case WorkerConnection.Stdio => inputRef.success(input)
case _ => ()
),
out = BasicIO.processFully(onStdoutLine),
out = readStdout,
err = BasicIO.processFully((line) => scala.Console.err.println(line)),
)
val forkWithIo = fo.withOutputStrategy(OutputStrategy.CustomInputOutput(processIo))
val p = Fork.java.fork(forkWithIo, options)
val forkTimeout = fo.connectionTimeout.getOrElse(30.seconds)
val input = Await.result(inputRef.future, forkTimeout)
WorkerProxy(input, p, options, closer)
WorkerProxy(input, p, options, closer, responsesRead.future)
end startWorker
/** Generates a fresh path suitable for binding a `WorkerConnection.Ipc` socket. */
def newIpcSocketPath(): NioPath =
@@ -178,6 +185,7 @@ class WorkerProxy(
val process: Process,
val options: Seq[String],
closer: Option[AutoCloseable],
responsesRead: Future[Unit],
) extends AutoCloseable:
lazy val inputStream = PrintStream(input)
def close(): Unit =
@@ -194,11 +202,21 @@ class WorkerProxy(
val watch = Thread(() => {
while process.isAlive() do Thread.sleep(100)
Try(Await.ready(responsesRead, WorkerProxy.responseDrainTimeout))
WorkerExchange.listeners.foreach(_.notifyExit(process))
})
watch.start()
end WorkerProxy
object WorkerProxy:
/**
* How long to keep reading a worker's responses after it exits, before reporting the exit.
* The reader reaches end of stream as soon as it has consumed everything the worker wrote;
* the bound only matters for a worker that died before connecting.
*/
private val responseDrainTimeout = 10.seconds
end WorkerProxy
abstract class WorkerResponseListener extends Function1[String, Unit]:
def notifyExit(p: Process): Unit
@@ -0,0 +1,47 @@
/*
* sbt
* Copyright 2023, Scala center
* Copyright 2011 - 2022, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt
package internal
import java.util.concurrent.atomic.AtomicReference
import sbt.internal.util.Util
/** Holds a closeable that a caller replaces, and closes the one it replaces. */
private[sbt] class AtomicCloseable[A >: Null <: AutoCloseable](val ref: AtomicReference[A])
extends AnyVal:
def get: A = ref.get
def set(c: A): Unit = AtomicCloseable.close(ref.getAndSet(c))
def close(): Unit = AtomicCloseable.close(AtomicCloseable.release(ref))
/** Keeps the value another caller put here, and closes the one this caller built. */
def setIfEmpty(ctor: => A): A =
var obj = ref.get
if obj eq null then
val value = ctor
require(value ne null, "AtomicCloseable.setIfEmpty: `ctor` must not return null")
while obj eq null do obj = if ref.compareAndSet(null, value) then value else ref.get
if obj ne value then AtomicCloseable.close(value)
obj
end AtomicCloseable
private[sbt] object AtomicCloseable:
def apply[A >: Null <: AutoCloseable](): AtomicCloseable[A] =
new AtomicCloseable(new AtomicReference[A])
def apply[A >: Null <: AutoCloseable](c: A): AtomicCloseable[A] =
new AtomicCloseable(new AtomicReference[A](c))
def close(obj: AutoCloseable): Unit =
if obj ne null then Util.ignoreTry(obj.close())
def release[A >: Null <: AutoCloseable](ref: AtomicReference[A]): A =
ref.getAndSet(null)
end AtomicCloseable
@@ -46,7 +46,7 @@ private[sbt] object Server {
def start(
connection: ServerConnection,
onIncomingSocket: (Socket, ServerInstance) => Unit,
onIncomingSocket: (AtomicReference[Socket], ServerInstance) => Unit,
log: Logger
): ServerInstance =
new ServerInstance { self =>
@@ -109,14 +109,19 @@ private[sbt] object Server {
running.set(true)
p.success(())
while (running.get()) {
val clientSocket = AtomicCloseable[Socket]()
try {
val socket = serverSocket.accept()
onIncomingSocket(socket, self)
clientSocket.set(serverSocket.accept())
onIncomingSocket(clientSocket.ref, self)
} catch {
case scala.util.control.NonFatal(e) if clientSocket.get ne null =>
log.error(s"sbt server failed to serve a client: $e")
log.trace(e)
case e: IOException if Option(e.getMessage).exists(_.contains("connect")) =>
case _: SocketTimeoutException => // its ok
case _: SocketException if !running.get => // the server is shutting down
}
clientSocket.close()
}
serverSocketHolder.get match {
case null =>
@@ -0,0 +1,72 @@
/*
* sbt
* Copyright 2023, Scala center
* Copyright 2011 - 2022, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt
package internal
import verify.BasicTestSuite
object AtomicCloseableSpec extends BasicTestSuite:
class Probe extends AutoCloseable:
var closed: Boolean = false
override def close(): Unit = closed = true
end Probe
test("a holder built around a value"):
val probe = new Probe
val holder = AtomicCloseable(probe)
assert(holder.get == probe)
assert(!probe.closed)
holder.close()
assert(probe.closed)
test("a value that replaces another closes it"):
val holder = AtomicCloseable[Probe]()
val first, second = new Probe
holder.set(first)
holder.set(second)
assert(first.closed)
assert(!second.closed)
assert(holder.get == second)
test("closing empties the holder"):
val holder = AtomicCloseable[Probe]()
val probe = new Probe
holder.set(probe)
holder.close()
assert(probe.closed)
assert(holder.get == null)
test("closing an empty holder does nothing"):
AtomicCloseable[Probe]().close()
test("a holder that has a value keeps it"):
val holder = AtomicCloseable[Probe]()
val first = new Probe
holder.set(first)
val second = new Probe
assert(holder.setIfEmpty(second) == first)
assert(!first.closed)
assert(!second.closed)
test("a holder that has no value takes the one it is given"):
val holder = AtomicCloseable[Probe]()
val probe = new Probe
assert(holder.setIfEmpty(probe) == probe)
assert(holder.get == probe)
assert(!probe.closed)
test("a value that loses a race is closed"):
val holder = AtomicCloseable[Probe]()
val winner, loser = new Probe
// the holder fills while this caller builds its own value
val result = holder.setIfEmpty { holder.set(winner); loser }
assert(result == winner)
assert(loser.closed)
assert(!winner.closed)
end AtomicCloseableSpec
@@ -61,7 +61,7 @@ object LMCoursier {
sys.env
.get("LOCALAPPDATA")
.map(absoluteFile)
.getOrElse(absoluteFile(sys.props("user.home")) / "AppData" / "Local")
.getOrElse(sbt.io.Path.userHome.getAbsoluteFile / "AppData" / "Local")
base / "Coursier" / "Cache" / "v1"
}
sys.props
@@ -205,19 +205,21 @@ private[sbt] final class CommandExchange {
lazy val enableBsp = s.get(bspEnabled).getOrElse(true)
lazy val portfile = s.baseDir / "project" / "target" / "active.json"
def onIncomingSocket(socket: Socket, instance: ServerInstance): Unit = {
def onIncomingSocket(socket: AtomicReference[Socket], instance: ServerInstance): Unit = {
val name = newNetworkName
Terminal.consoleLog(s"new client connected: $name")
val channel =
new NetworkChannel(
name,
socket,
socket.get,
auth,
instance,
handlers,
mkAskUser(name),
)
subscribe(channel)
channel.start()
AtomicCloseable.release(socket) // i took over
}
if (server.isEmpty && firstInstance.get) {
val h = Hash.halfHashString(IO.toURI(portfile).toString)
@@ -202,7 +202,7 @@ object SysProp:
def secondaryIdleTimeoutSec: Long = long("sbt.server.secondaryIdleTimeout", 600L)
private def file(value: String): File = new File(value)
private def home: File = file(sys.props("user.home"))
private def home: File = sbt.io.Path.userHome
/**
* Default directory for global sbt config (plugins, settings). Respects XDG Base Directory
@@ -965,8 +965,10 @@ final class NetworkChannel(
private[sbt] def isAttached: Boolean = attached.get
private[sbt] def isInitialized: Boolean = initialized
thread.start()
writeThread.start()
private[sbt] def start(): Unit =
thread.start()
writeThread.start()
}
object NetworkChannel {
@@ -13,7 +13,10 @@ import java.net.{ InetAddress, ServerSocket, Socket }
import sbt.{ State, StandardMain }
import sbt.internal.util.{ AttributeMap, ConsoleOut, GlobalLogging, MainAppender, Terminal, Util }
import sbt.internal.util.Terminal.TerminalImpl
import sbt.internal.protocol.JsonRpcRequestMessage
import sbt.protocol.Serialization
import sjsonnew.support.scalajson.unsafe.Parser
import scala.concurrent.duration.*
import scala.jdk.CollectionConverters.*
import scala.util.Using
import verify.BasicTestSuite
@@ -78,6 +81,31 @@ object NetworkChannelSpec extends BasicTestSuite:
val outcome = whileInterrupted(StandardMain.exchange.removeChannel(channel))
assertSucceededAndStillInterrupted(outcome)
test("an attach sent before the channel is subscribed still attaches the client"):
withServerState:
withChannelThreadsJoined:
withLoopbackConnection: (client, connection) =>
client.getOutputStream.write(attachRequest)
client.getOutputStream.flush()
Using.resource(newChannel(connection, Seq(VirtualTerminal.handler))): channel =>
Thread.sleep(acceptThreadDelay.toMillis)
StandardMain.exchange.subscribe(channel)
channel.start()
assert(eventually(channel.isAttached), "the attach request was dropped")
private val acceptThreadDelay = 200.millis
private def attachRequest: Array[Byte] =
val params = Parser.parseUnsafe("""{"interactive": false}""")
Serialization.serializeRequestMessage(
JsonRpcRequestMessage("2.0", "attach-id", Serialization.attach, Some(params))
)
private def eventually(condition: => Boolean): Boolean =
val deadline = 5.seconds.fromNow
while !condition && deadline.hasTimeLeft() do Thread.sleep(10)
condition
private type Outcome = (thrown: Option[Exception], stillInterrupted: Boolean)
private given Using.Releasable[NetworkChannel] = _.shutdown(false)
@@ -104,7 +132,7 @@ object NetworkChannelSpec extends BasicTestSuite:
private def withAttachedClient[A](test: NetworkChannel => A): A =
withServerState:
withChannelThreadsJoined:
withLoopbackConnection: connection =>
withLoopbackConnection: (_, connection) =>
Using.resource(attachedChannel(connection))(test)
private def withChannelThreadsJoined[A](f: => A): A =
@@ -120,26 +148,30 @@ object NetworkChannelSpec extends BasicTestSuite:
thread.getName.startsWith("sbt-networkchannel-") ||
thread.getName.startsWith(s"sbt-$channelName-")
private def attachedChannel(connection: Socket): NetworkChannel =
val channel = new NetworkChannel(
private def newChannel(connection: Socket, handlers: Seq[ServerHandler]): NetworkChannel =
new NetworkChannel(
name = channelName,
connection = connection,
auth = Set.empty,
instance = null,
handlers = Nil,
handlers = handlers,
mkUIThreadImpl = (_, _) => null,
)
private def attachedChannel(connection: Socket): NetworkChannel =
val channel = newChannel(connection, Nil)
channel.start()
val attachRequestId = "attached-id"
channel.setInteractive(attachRequestId, value = false)
channel
private def withLoopbackConnection[A](f: Socket => A): A =
private def withLoopbackConnection[A](f: (Socket, Socket) => A): A =
val loopback = InetAddress.getLoopbackAddress
val anyFreePort = 0
val backlog = 1
Using.resource(new ServerSocket(anyFreePort, backlog, loopback)): server =>
Using.resource(new Socket(loopback, server.getLocalPort)): _ =>
Using.resource(server.accept())(f)
Using.resource(new Socket(loopback, server.getLocalPort)): client =>
Using.resource(server.accept())(f(client, _))
private def withServerState[A](f: => A): A =
val previous = StandardMain.exchange.withState(Option(_))
+1 -1
View File
@@ -12,7 +12,7 @@ object Dependencies {
sys.env.get("BUILD_VERSION") orElse sys.props.get("sbt.build.version")
// sbt modules
val ioVersion = nightlyVersion.getOrElse("1.13.4")
val ioVersion = nightlyVersion.getOrElse("1.13.5")
val zincVersion = nightlyVersion.getOrElse("2.0.4")
private val sbtIO = "org.scala-sbt" %% "io" % ioVersion
@@ -0,0 +1,11 @@
scalaVersion := "3.9.0"
Test / fork := true
libraryDependencies += "org.scalameta" %% "munit" % "1.0.4" % Test
Test / testListeners += new TestsListener:
def doInit(): Unit = ()
def startGroup(name: String): Unit = ()
def testEvent(event: TestEvent): Unit = Thread.sleep(50)
def endGroup(name: String, t: Throwable): Unit = ()
def endGroup(name: String, result: TestResult): Unit = ()
def doComplete(finalResult: TestResult): Unit = ()
@@ -0,0 +1,2 @@
class ManyTests extends munit.FunSuite:
(1 to 40).foreach(i => test(s"test $i")(assert(i < 40)))
@@ -0,0 +1 @@
-> test
@@ -11,7 +11,7 @@ package internal
package inc
import sbt.internal.inc.classpath.ClasspathUtil
import sbt.internal.librarymanagement.JsonUtil
import sbt.internal.librarymanagement.{ JsonUtil, LMSysProp }
import sbt.internal.util.{ BufferedLogger, FullLogger }
import sbt.io.IO
import sbt.librarymanagement.*
@@ -171,12 +171,7 @@ private[sbt] object ZincComponentCompiler {
new ZincCompilerBridgeProvider(None, manager, dependencyResolution, scalaJarsTarget)
private final val LocalIvy =
(sys.props.get("sbt.ivy.home") match {
case Some(home) =>
if (home.endsWith("/")) home
else home + "/"
case _ => s"$${user.home}/.ivy2/"
}) + "local/" + Resolver.localBasePattern
Resolver.resolvePattern(LMSysProp.defaultIvyHome.toString, "local/" + Resolver.localBasePattern)
final val LocalResolver: Resolver = {
val toUse = Vector(LocalIvy)