Merge 3f6fd595ff into e27f3df021

main/src/main/scala/sbt/Keys.scala

@@ -727,6 +727,8 @@ object Keys {
   }
   private[sbt] val currentCommandProgress = AttributeKey[ExecuteProgress2]("current-command-progress")
   private[sbt] val taskProgress = AttributeKey[sbt.internal.TaskProgress]("active-task-progress")
+  private[sbt] val resolutionProgress =
+    AttributeKey[sbt.coursierint.ResolutionProgress]("resolution-progress", Invisible)
   val useSuperShell = settingKey[Boolean]("Enables (true) or disables the super shell.")
   val superShellMaxTasks = settingKey[Int]("The max number of tasks to display in the supershell progress report")
   val superShellSleep = settingKey[FiniteDuration]("The minimum duration to sleep between progress reports")

main/src/main/scala/sbt/MainLoop.scala

@@ -168,11 +168,13 @@ private[sbt] object MainLoop:
       state.get(Keys.superShellSleep.key).getOrElse(SysProp.supershellSleep.millis)
     val superShellThreshold =
       state.get(Keys.superShellThreshold.key).getOrElse(SysProp.supershellThreshold)
+    val resolutionProgress = new sbt.coursierint.ResolutionProgress
     val taskProgress =
       new TaskProgress(
         superShellSleep,
         superShellThreshold,
         state.log,
+        resolutionProgress,
         Project.configNameToIdent(state)
       )
     val gcMonitor = if (SysProp.gcMonitor) Some(new sbt.internal.GCMonitor(state.log)) else None
@@ -181,6 +183,7 @@ private[sbt] object MainLoop:
         state
           .put(Keys.loggerContext, context)
           .put(Keys.taskProgress, taskProgress)
+          .put(Keys.resolutionProgress, resolutionProgress)
           .process(processCommand)
       } match {
         case Right(s)                  => s.remove(Keys.loggerContext)

main/src/main/scala/sbt/coursierint/LMCoursier.scala

@@ -255,7 +255,17 @@ object LMCoursier {
   def coursierLoggerTask: Def.Initialize[Task[Option[CacheLogger]]] = Def.task {
     val st = Keys.streams.value
     val progress = (ThisBuild / useSuperShell).value
-    if (progress) None
+    // Always supply a logger: this suppresses coursier's own per-module progress bar and lets
+    // resolution run in parallel across modules. Under the super shell we feed the per-command
+    // resolution-progress sink (rendered as one task-level line by TaskProgress); otherwise the
+    // quiet debug logger.
+    if (progress)
+      Some(
+        Keys.state.value
+          .get(Keys.resolutionProgress)
+          .map(new ResolutionProgressLogger(_))
+          .getOrElse(CacheLogger.nop)
+      )
     else Some(new CoursierLogger(st.log))
   }
 

main/src/main/scala/sbt/coursierint/ResolutionProgress.scala

@@ -0,0 +1,89 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package coursierint
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.{ AtomicInteger, AtomicLong }
+import lmcoursier.definitions.CacheLogger
+
+/**
+ * Per-command running total of dependency-resolution progress.
+ *
+ * One instance is created per command in `MainLoop.next` (held under `Keys.resolutionProgress`, the
+ * same lifecycle as `Keys.taskProgress`). It is fed by [[ResolutionProgressLogger]] from coursier's
+ * download-pool threads and read by `TaskProgress` to render a single super-shell line. Because
+ * those callbacks run in parallel across modules, every field is atomic and byte accounting uses a
+ * monotonic per-url delta and so can never go backwards. There is no cross-command reset: the
+ * instance is born empty and discarded with the command.
+ */
+private[sbt] final class ResolutionProgress {
+  private val inFlight = new AtomicInteger(0)
+  private val modules = new AtomicInteger(0)
+  private val artifacts = new AtomicLong(0L)
+  private val bytes = new AtomicLong(0L)
+  private val seen = new ConcurrentHashMap[String, java.lang.Long]
+
+  def onInit(): Unit = {
+    inFlight.incrementAndGet()
+    modules.incrementAndGet()
+    ()
+  }
+
+  def onStop(): Unit = {
+    inFlight.updateAndGet(n => math.max(0, n - 1))
+    ()
+  }
+
+  def onArtifact(): Unit = {
+    artifacts.incrementAndGet()
+    ()
+  }
+
+  def onProgress(url: String, downloaded: Long): Unit = {
+    // compute keeps the read-compare-add atomic per url, so concurrent progress callbacks for the
+    // same url can neither double-count nor lose a byte delta; `seen` always holds the max seen.
+    seen.compute(
+      url,
+      (_: String, prev: java.lang.Long) => {
+        val p: Long = if (prev == null) 0L else prev.longValue
+        if (downloaded > p) bytes.addAndGet(downloaded - p)
+        java.lang.Long.valueOf(math.max(downloaded, p))
+      }
+    )
+    ()
+  }
+
+  /** A render string while at least one resolution is in flight, else None (the line disappears). */
+  def snapshot(): Option[String] =
+    if (inFlight.get() <= 0) None
+    else {
+      val m = modules.get()
+      val a = artifacts.get()
+      val mib = bytes.get().toDouble / (1024.0 * 1024.0)
+      val mLabel = if (m == 1) "module" else "modules"
+      val aLabel = if (a == 1) "artifact" else "artifacts"
+      Some(f"Updating $m $mLabel, $a $aLabel, $mib%.1f MiB")
+    }
+}
+
+/**
+ * A coursier `CacheLogger` that feeds a per-command [[ResolutionProgress]]. Supplying any logger to
+ * lm-coursier suppresses coursier's own per-module progress bar and lets resolution run in parallel
+ * across modules; the aggregate is rendered at the sbt task level instead.
+ */
+private[sbt] final class ResolutionProgressLogger(sink: ResolutionProgress) extends CacheLogger {
+  override def init(sizeHint: Option[Int]): Unit = sink.onInit()
+  override def stop(): Unit = sink.onStop()
+  override def foundLocally(url: String): Unit = sink.onArtifact()
+  override def downloadedArtifact(url: String, success: Boolean): Unit =
+    if (success) sink.onArtifact()
+  override def downloadProgress(url: String, downloaded: Long): Unit =
+    sink.onProgress(url, downloaded)
+}

main/src/main/scala/sbt/internal/TaskProgress.scala

@@ -18,6 +18,7 @@ import scala.jdk.CollectionConverters.*
 import scala.concurrent.duration.*
 import java.util.concurrent.{ ConcurrentHashMap, Executors, TimeoutException }
 import sbt.util.Logger
+import sbt.coursierint.ResolutionProgress
 
 /**
  * implements task progress display on the shell.
@@ -26,6 +27,7 @@ private[sbt] class TaskProgress(
     sleepDuration: FiniteDuration,
     threshold: FiniteDuration,
     logger: Logger,
+    resolutionProgress: ResolutionProgress,
     configNameToIdent: String => String = Scope.guessConfigIdent
 ) extends AbstractTaskExecuteProgress(configNameToIdent)
     with ExecuteProgress
@@ -181,6 +183,11 @@ private[sbt] class TaskProgress(
           val name = taskName(task)
           distinct.put(name, ProgressItem(name, elapsed))
         }
+        // Append one aggregate dependency-resolution line while `update` resolves in parallel,
+        // since coursier no longer renders its own per-module progress bars.
+        resolutionProgress.snapshot().foreach { line =>
+          distinct.put(line, ProgressItem(line, 0L))
+        }
         ProgressEvent(
           "Info",
           distinct.values.asScala.toVector,

main/src/test/scala/sbt/coursierint/ResolutionProgressSpec.scala

@@ -0,0 +1,60 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.coursierint
+
+import verify.BasicTestSuite
+
+object ResolutionProgressSpec extends BasicTestSuite:
+
+  test("a fresh per-command instance starts with no line") {
+    val p = new ResolutionProgress
+    assert(p.snapshot().isEmpty)
+  }
+
+  test("aggregates modules, artifacts, and monotonic bytes while resolving") {
+    val p = new ResolutionProgress
+    val log = new ResolutionProgressLogger(p)
+    log.init(None)
+    log.init(None) // two resolutions in flight
+    log.downloadProgress("a.jar", 1000L)
+    log.downloadProgress("a.jar", 4000L) // monotonic increase, total 4000
+    log.downloadProgress("a.jar", 2000L) // out-of-order, must be ignored
+    log.foundLocally("b.jar") // counts as an artifact
+    log.downloadedArtifact("a.jar", success = true) // counts
+    log.downloadedArtifact("c.jar", success = false) // failed, must not count
+    val line = p.snapshot()
+    assert(line.isDefined, "expected a progress line while resolving")
+    assert(line.exists(_.contains("2 modules")), line.toString)
+    assert(line.exists(_.contains("2 artifacts")), line.toString)
+    log.stop()
+    log.stop()
+    assert(p.snapshot().isEmpty)
+  }
+
+  test("counts persist across the resolve and artifacts phases of one command") {
+    val p = new ResolutionProgress
+    val log = new ResolutionProgressLogger(p)
+    // resolve phase
+    log.init(None)
+    log.foundLocally("x.jar")
+    log.stop()
+    assert(p.snapshot().isEmpty) // idle between phases
+    // artifacts phase of the SAME command: counts accumulate, they do not reset
+    log.init(None)
+    log.downloadedArtifact("y.jar", success = true)
+    val line = p.snapshot()
+    assert(
+      line.exists(_.contains("2 modules")),
+      line.toString
+    ) // resolve + artifacts, not reset to 1
+    assert(line.exists(_.contains("2 artifacts")), line.toString) // x + y, not reset
+    log.stop()
+  }
+
+end ResolutionProgressSpec

notes/2.0.0/parallel-resolution-supershell.md

@@ -0,0 +1,19 @@
+### `update` resolves in parallel under the super shell
+
+Building on #9270 (parallel dependency resolution for non-interactive runs), `update` now also
+resolves in parallel under the interactive super shell. Previously sbt let coursier draw its own
+per-module progress bars there, and those bars cannot be rendered from more than one module at
+once, so resolution was serialized one module at a time. sbt now suppresses coursier's bars and
+renders a single aggregate progress line at the task level instead, e.g.:
+
+```
+Updating 18 modules, 240 artifacts, 31.0 MiB
+```
+
+so resolution can run concurrently across modules. Setting `csrLogger := Some(...)` opts out: your
+logger is used instead, and coursier's behavior is unchanged.
+
+This is the first step of [#5627][i5627]. Per-module status lines (which would add a `status`
+field to `ProgressItem`) remain a possible follow-up.
+
+[i5627]: https://github.com/sbt/sbt/issues/5627