diff --git a/main-actions/src/main/scala/sbt/TestResultLogger.scala b/main-actions/src/main/scala/sbt/TestResultLogger.scala index f344bbe7d..fbcdbb010 100644 --- a/main-actions/src/main/scala/sbt/TestResultLogger.scala +++ b/main-actions/src/main/scala/sbt/TestResultLogger.scala @@ -8,7 +8,8 @@ package sbt -import sbt.Tests.{ Output, Summary } +import sbt.Tests.Output +import sbt.internal.util.Terminal import sbt.protocol.testing.TestResult import sbt.util.{ Level, Logger } @@ -20,7 +21,7 @@ import sbt.util.{ Level, Logger } * * @since 0.13.5 */ -trait TestResultLogger { +trait TestResultLogger: /** * Perform logging. @@ -31,6 +32,14 @@ trait TestResultLogger { */ def run(log: Logger, results: Output, taskName: String): Unit + def run(log: Logger, results: Output, taskName: String, cached: Vector[String]): Unit = + run(log, results, taskName) + + def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit = + entries.foreach { case (results, taskName, _) => + run(log, results, taskName) + } + /** Only allow invocation if certain criteria is met, else use another `TestResultLogger` (defaulting to nothing) . */ final def onlyIf( f: (Output, String) => Boolean, @@ -44,9 +53,9 @@ trait TestResultLogger { otherwise: TestResultLogger = TestResultLogger.Null ) = TestResultLogger.choose(f, otherwise, this) -} +end TestResultLogger -object TestResultLogger { +object TestResultLogger: /** A `TestResultLogger` that does nothing. */ val Null = const(_ => ()) @@ -61,6 +70,19 @@ object TestResultLogger { def apply(f: (Logger, Output, String) => Unit): TestResultLogger = (log, results, taskName) => f(log, results, taskName) + /** Creates a `TestResultLogger` using a given function that also receives the cached suite names. */ + def apply(f: (Logger, Output, String, Vector[String]) => Unit): TestResultLogger = + new TestResultLogger: + def run(log: Logger, results: Output, taskName: String): Unit = + f(log, results, taskName, Vector.empty) + override def run( + log: Logger, + results: Output, + taskName: String, + cached: Vector[String] + ): Unit = + f(log, results, taskName, cached) + /** Creates a `TestResultLogger` that ignores its input and always performs the same logging. */ def const(f: Logger => Unit) = apply((l, _, _) => f(l)) @@ -71,8 +93,8 @@ object TestResultLogger { * @param f The `TestResultLogger` to choose if the predicate fails. */ def choose(cond: (Output, String) => Boolean, t: TestResultLogger, f: TestResultLogger) = - TestResultLogger((log, results, taskName) => - (if (cond(results, taskName)) t else f).run(log, results, taskName) + TestResultLogger((log, results, taskName, cached) => + (if cond(results, taskName) then t else f).run(log, results, taskName, cached) ) /** Transforms the input to be completely silent when the subject module doesn't contain any tests. */ @@ -82,7 +104,44 @@ object TestResultLogger { printNoTests = Null ) - object Defaults { + object Defaults: + private val suitePadding = " " * 8 + + private[sbt] enum SummaryStatus: + case Passed, Failed, Errored + + def label: String = this match + case SummaryStatus.Passed => "passed" + case SummaryStatus.Failed => "failed" + case SummaryStatus.Errored => "error" + + def word: String = this match + case SummaryStatus.Passed => "succeeded" + case SummaryStatus.Failed => "failed" + case SummaryStatus.Errored => "errored" + end SummaryStatus + + private[sbt] enum SuiteStatus: + case Pass, CachedPass, Fail, Error + + def isPassing: Boolean = this match + case SuiteStatus.Pass | SuiteStatus.CachedPass => true + case SuiteStatus.Fail | SuiteStatus.Error => false + + def render(isColorEnabled: Boolean): String = + val (prefix, word, color) = this match + case SuiteStatus.Pass => ("", "PASS", scala.Console.GREEN) + case SuiteStatus.CachedPass => ("(cached) ", "PASS", scala.Console.GREEN) + case SuiteStatus.Fail => ("", "FAIL", scala.Console.RED) + case SuiteStatus.Error => ("", "ERROR", scala.Console.RED) + if isColorEnabled then s"$prefix$color$word${scala.Console.RESET}" else s"$prefix$word" + end SuiteStatus + + private[sbt] object SuiteStatus: + def apply(r: TestResult): SuiteStatus = r match + case TestResult.Passed | TestResult.Empty => SuiteStatus.Pass + case TestResult.Failed => SuiteStatus.Fail + case TestResult.Error => SuiteStatus.Error /** sbt's default `TestResultLogger`. Use `copy()` to change selective portions. */ case class Main( @@ -93,8 +152,16 @@ object TestResultLogger { printNoTests: TestResultLogger = Defaults.printNoTests ) extends TestResultLogger { - override def run(log: Logger, results: Output, taskName: String): Unit = { - def run(r: TestResultLogger): Unit = r.run(log, results, taskName) + override def run(log: Logger, results: Output, taskName: String): Unit = + run(log, results, taskName, Vector.empty) + + override def run( + log: Logger, + results: Output, + taskName: String, + cached: Vector[String] + ): Unit = { + def run(r: TestResultLogger): Unit = r.run(log, results, taskName, cached) run(printSummary) @@ -117,7 +184,7 @@ object TestResultLogger { val printSummary = TestResultLogger((log, results, _) => { val multipleFrameworks = results.summaries.size > 1 - for (Summary(name, message) <- results.summaries) + for Tests.Summary(name, message) <- results.summaries do if (message.isEmpty) log.debug("Summary for " + name + " not available.") else { if (multipleFrameworks) log.info(name) @@ -130,54 +197,40 @@ object TestResultLogger { // Print the standard one-liner statistic if no framework summary is defined, or when > 1 framework is in used. results.summaries.size > 1 || results.summaries.headOption.forall(_.summaryText.isEmpty) - val printStandard = TestResultLogger((log, results, _) => { - val counts = countsString(results) + val printStandard = TestResultLogger((log, results, _, cached) => { + val counts = countsString(results.events.values, cached.size, true) results.overall match case TestResult.Empty => () - case TestResult.Error => log.error("Error: " + counts) - case TestResult.Passed => log.info("Passed: " + counts) - case TestResult.Failed => log.error("Failed: " + counts) + case TestResult.Error => log.error(s"${SummaryStatus.Errored.label}: $counts") + case TestResult.Passed => log.info(s"${SummaryStatus.Passed.label}: $counts") + case TestResult.Failed => log.error(s"${SummaryStatus.Failed.label}: $counts") }) + private[sbt] def countsString(events: Iterable[SuiteResult]): String = + countsString(events, 0, false) + /** - * Renders `Tests.Output`'s aggregate counts as a single line like - * `Total 10, Failed 2, Errors 0, Passed 8`. Shared between `printStandard` - * and the cross-project recap formatter (see `TestRecap`). + * Renders suite counts as a single line like `total 10, failed 2, errors + * 0, passed 8`. Counts suites (classes/objects), not individual test examples. */ - private[sbt] def countsString(results: Output): String = { - val ( - skippedCount, - errorsCount, - passedCount, - failuresCount, - ignoredCount, - canceledCount, - pendingCount, - ) = - results.events.foldLeft((0, 0, 0, 0, 0, 0, 0)) { case (acc, (_, testEvent)) => - val (skippedAcc, errorAcc, passedAcc, failureAcc, ignoredAcc, canceledAcc, pendingAcc) = - acc - ( - skippedAcc + testEvent.skippedCount, - errorAcc + testEvent.errorCount, - passedAcc + testEvent.passedCount, - failureAcc + testEvent.failureCount, - ignoredAcc + testEvent.ignoredCount, - canceledAcc + testEvent.canceledCount, - pendingAcc + testEvent.pendingCount, - ) + private[sbt] def countsString( + events: Iterable[SuiteResult], + cachedCount: Int, + alwaysShowCached: Boolean, + ): String = { + val (failuresCount, errorsCount, passedCount) = + events.foldLeft((0, 0, 0)) { case ((failureAcc, errorAcc, passedAcc), suite) => + suite.result match + case TestResult.Failed => (failureAcc + 1, errorAcc, passedAcc) + case TestResult.Error => (failureAcc, errorAcc + 1, passedAcc) + case TestResult.Passed | TestResult.Empty => (failureAcc, errorAcc, passedAcc + 1) } - val totalCount = failuresCount + errorsCount + skippedCount + passedCount + val totalCount = failuresCount + errorsCount + passedCount + cachedCount val base = - s"Total $totalCount, Failed $failuresCount, Errors $errorsCount, Passed $passedCount" - val otherCounts = Seq( - "Skipped" -> skippedCount, - "Ignored" -> ignoredCount, - "Canceled" -> canceledCount, - "Pending" -> pendingCount - ) - val extra = otherCounts.withFilter(_._2 > 0).map { (label, count) => s", $label $count" } - base + extra.mkString + s"total $totalCount, failed $failuresCount, errors $errorsCount, passed ${passedCount + cachedCount}" + val cachedField = + if cachedCount > 0 || alwaysShowCached then s", cached $cachedCount" else "" + base + cachedField } val printFailures = TestResultLogger((log, results, _) => { @@ -189,15 +242,109 @@ object TestResultLogger { def show(label: String, level: Level.Value, tests: Iterable[String]): Unit = if (tests.nonEmpty) { log.log(level, label) - log.log(level, tests.mkString("\t", "\n\t", "")) + log.log(level, tests.mkString(suitePadding, s"\n$suitePadding", "")) } - show("Passed tests:", Level.Debug, select(TestResult.Passed)) - show("Failed tests:", Level.Error, select(TestResult.Failed)) - show("Error during tests:", Level.Error, select(TestResult.Error)) + show("passed tests:", Level.Debug, select(TestResult.Passed)) + show("failed tests:", Level.Error, select(TestResult.Failed)) + show("error during tests:", Level.Error, select(TestResult.Error)) }) - val printNoTests = - TestResultLogger((log, results, taskName) => log.info("No tests to run for " + taskName)) - } -} + val printNoTests = TestResultLogger((log, results, taskName, cached) => + val suffix = if cached.nonEmpty then s" (${cached.size} cached)" else "" + log.debug(s"no tests to run for $taskName$suffix") + ) + + /** + * Renders a cross-project aggregate summary at the end of an aggregated + * run, in the style selected by `mode` (see `TestSummary`). Per-task + * logging is unchanged (delegates to `Default`); `summary` is where this + * differs from a plain `TestResultLogger`. + */ + case class Summary(mode: TestSummary) extends TestResultLogger: + override def run(log: Logger, results: Output, taskName: String): Unit = + Default.run(log, results, taskName) + + override def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit = + val lines = Summary.render(mode, entries, Terminal.get.isColorEnabled) + val logLine: String => Unit = Summary.overallStatus(entries) match + case Some(SummaryStatus.Errored) | Some(SummaryStatus.Failed) => log.error(_) + case _ => log.info(_) + lines.foreach(line => logLine(if line.isEmpty then " " else line)) + end Summary + + object Summary: + private val columnGap = 5 + + def apply(): Summary = Summary(mode = TestSummary.default) + + /** + * The rendered summary; empty when nothing ran. Both styles end with an + * aggregate line, e.g. `passed: total 4, failed 0, errors 0, passed 4, + * cached 2`. Cached classes count into `total` and `passed`, each as one + * (their case counts are unknown without running them). + */ + private[sbt] def render( + mode: TestSummary, + entries: Vector[(Output, String, Vector[String])], + isColorEnabled: Boolean, + ): Vector[String] = + overallStatus(entries) match + case None => Vector.empty + case Some(status) => + val executed = entries.flatMap(_._1.events.values) + val cached = entries.map(_._3.size).sum + val counts = + s"${status.label}: ${countsString(executed, cached, alwaysShowCached = true)}" + def withCounts(detailLines: Vector[String]): Vector[String] = + if detailLines.isEmpty then Vector(counts) else detailLines :+ "" :+ counts + mode match + case TestSummary.None => Vector.empty + case TestSummary.Failure => + withCounts(detail(status, entries, failuresOnly = true, isColorEnabled)) + case TestSummary.Success => + withCounts(detail(status, entries, failuresOnly = false, isColorEnabled)) + + /** The overall status across `entries`, or `None` when nothing ran. */ + private def overallStatus( + entries: Vector[(Output, String, Vector[String])] + ): Option[SummaryStatus] = + val executed = entries.flatMap(_._1.events.values) + val cached = entries.map(_._3.size).sum + if executed.isEmpty && cached == 0 then None + else if executed.exists(_.result == TestResult.Error) then Some(SummaryStatus.Errored) + else if executed.exists(_.result == TestResult.Failed) then Some(SummaryStatus.Failed) + else Some(SummaryStatus.Passed) + + private def detail( + status: SummaryStatus, + entries: Vector[(Output, String, Vector[String])], + failuresOnly: Boolean, + isColorEnabled: Boolean, + ): Vector[String] = + val tasks = entries + .map: (output, taskName, cachedNames) => + val executed = output.events.view.mapValues(s => SuiteStatus(s.result)).toVector + val suites = + if failuresOnly then executed.filter { case (_, st) => !st.isPassing } + else executed ++ cachedNames.map(_ -> SuiteStatus.CachedPass) + taskName -> suites.sortBy(_._1) + .filter(_._2.nonEmpty) + .sortBy(_._1) + if tasks.isEmpty then Vector.empty + else + val headerCount = + if status == SummaryStatus.Passed then tasks.size + else entries.count(_._1.events.values.exists(s => !SuiteStatus(s.result).isPassing)) + val plural = if headerCount == 1 then "" else "s" + val width = tasks.flatMap(_._2.map(_._1.length)).max + columnGap + val lines = Vector.newBuilder[String] + lines += s"Test summary ($headerCount test task$plural ${status.word}):" + tasks.foreach: (taskName, suites) => + lines += s" $taskName" + suites.foreach: (name, st) => + lines += s"${suitePadding}${name.padTo(width, ' ')}${st.render(isColorEnabled)}" + lines.result() + end Summary + end Defaults +end TestResultLogger diff --git a/main-actions/src/main/scala/sbt/TestSummary.scala b/main-actions/src/main/scala/sbt/TestSummary.scala new file mode 100644 index 000000000..584b4b9e3 --- /dev/null +++ b/main-actions/src/main/scala/sbt/TestSummary.scala @@ -0,0 +1,109 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import java.util.concurrent.ConcurrentLinkedQueue +import sbt.internal.util.AttributeKey + +/** + * Style of the test summary displayed after an aggregated test run: + * - `None`: prints nothing + * - `Failure` (the default): lists only non-passing suites, staying terse when everything passes + * - `Success`: always lists every suite, marking cache reuse with `(cached)` + * + * Rendering itself lives in `TestResultLogger.Defaults.Summary`, which + * consumes the entries collected here. + */ +enum TestSummary: + case None + case Failure + case Success + +/** + * Collector surfacing a test summary at the end of an aggregated run, + * success or failure alike. + */ +object TestSummary: + val none: TestSummary = TestSummary.None + val failure: TestSummary = TestSummary.Failure + val success: TestSummary = TestSummary.Success + def default: TestSummary = TestSummary.failure + + /** `cached` names the test classes reused from the action cache (see `IncrementalTest.cachedTestNames`). */ + private[sbt] final case class Entry( + taskName: String, + testOutput: Tests.Output, + cached: Vector[String], + options: Vector[Tests.AdhocOption] + ) + + /** + * State attribute holding the entries from the most recent aggregated run + * that produced at least one test result. Monotonic-latest-run semantics: + * never cleared, only overwritten by the next non-empty run; lets in-JVM + * tools (IDE plugins, BSP servers, scripted tests staying inside one sbt + * invocation via `Command.process`) inspect the last test results without + * parsing log output. Scripted tests crossing a `->` boundary cannot read + * this because the inner sbt's IPC server is torn down on failure and a + * fresh JVM is spawned for the next statement. + */ + private[sbt] val entriesKey: AttributeKey[Vector[Entry]] = AttributeKey[Vector[Entry]]( + "testSummaryEntries", + "Entries collected from the most recent aggregated test run" + ) + + private val entries = new ConcurrentLinkedQueue[Entry] + + /** + * `output`'s `SuiteResult.throwables` are dropped before retention: a + * test-thrown Throwable's backtrace pins the `Class` objects of every + * frame, and a `Class` strongly references its defining class loader. + * Since entries are stashed on `State.attributes` where they outlive the + * command, retaining the throwables would keep the test class loader -- + * and its open jar handles -- alive for the rest of the session. On + * Windows those handles make the cached jars undeletable (e.g. by + * `clearCaches`). + */ + private[sbt] def append( + taskName: String, + output: Tests.Output, + cached: Vector[String], + adhocOptions: Vector[Tests.AdhocOption] + ): Unit = + entries.add(Entry(taskName, dropThrowables(output), cached, adhocOptions)) + () + + private def dropThrowables(o: Tests.Output): Tests.Output = + o.copy(events = o.events.view.mapValues(dropThrowables).toMap) + + private def dropThrowables(s: SuiteResult): SuiteResult = + if s.throwables.isEmpty then s + else + new SuiteResult( + s.result, + s.passedCount, + s.failureCount, + s.errorCount, + s.skippedCount, + s.ignoredCount, + s.canceledCount, + s.pendingCount, + ) + + private[sbt] def clear(): Unit = entries.clear() + + private[sbt] def drain(): Vector[Entry] = + val b = Vector.newBuilder[Entry] + var e = entries.poll() + while e != null do + b += e + e = entries.poll() + b.result() + +end TestSummary diff --git a/main-actions/src/main/scala/sbt/Tests.scala b/main-actions/src/main/scala/sbt/Tests.scala index 883d00ed5..2b42b7e91 100644 --- a/main-actions/src/main/scala/sbt/Tests.scala +++ b/main-actions/src/main/scala/sbt/Tests.scala @@ -8,6 +8,7 @@ package sbt +import java.util.Locale import std.* import xsbt.api.{ Discovered, Discovery } import sbt.internal.inc.Analysis @@ -38,11 +39,33 @@ import sbt.util.{ Digest, Logger } import sbt.protocol.testing.TestResult import scala.runtime.AbstractFunction3 +import sbt.internal.util.complete.{ DefaultParsers, Parser } sealed trait TestOption object Tests { + private[sbt] sealed trait AdhocOption + private[sbt] object AdhocOption: + case class Summary(summary: TestSummary) extends AdhocOption + + private[sbt] def parser: Parser[AdhocOption] = + import DefaultParsers.* + val value = token("--test_summary=") ~> token(NotSpace.examples("none", "failure", "success")) + value.flatMap: v => + Tests.parseTestSummary(v) match + case Some(ts) => Parser.success(Summary(ts)) + case None => Parser.failure(s"Invalid test_summary value: $v") + end AdhocOption + + private[sbt] def parseTestSummary(value: String): Option[TestSummary] = + value.toLowerCase(Locale.ENGLISH) match + case "0" | "never" | "false" | "none" => Some(TestSummary.none) + case "1" | "always" | "true" => Some(TestSummary.default) + case "failure" => Some(TestSummary.failure) + case "2" | "success" | "verbose" => Some(TestSummary.success) + case _ => None + /** * The result of a test run. * diff --git a/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala b/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala deleted file mode 100644 index 1dd2adbb8..000000000 --- a/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala +++ /dev/null @@ -1,171 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt -package internal -package testing - -import sbt.Incomplete -import sbt.Tests -import sbt.TestResultLogger -import sbt.TestsFailedException -import sbt.protocol.testing.TestResult -import sbt.internal.util.AttributeKey -import sbt.util.Logger - -/** - * Stateless formatter that surfaces every failed test task at the end of an - * aggregated run (see sbt/sbt#2998). The data is read directly off the - * `Incomplete` tree returned by `Aggregation.runTasks`: each subproject's - * `testFull` / `inputTests0` catches the `TestsFailedException` thrown by - * `TestResultLogger.Defaults.Main.run` and re-throws with `(taskName, - * Some(Tests.Output))` attached, and we collect those instances from the - * tree. - * - * The collected `Vector[Failure]` is also stashed on `State.attributes` - * under `recapKey` so in-JVM tools (IDE plugins, BSP servers, scripted - * tests that stay inside one sbt invocation via `Command.process`) can - * inspect the most recent recap without parsing log output. Scripted tests - * crossing a `->` boundary cannot read this because the inner sbt's IPC - * server is torn down on failure and a fresh JVM is spawned for the next - * statement. - * - * Lifecycle is monotonic-latest-failure: `Aggregation.runTasks` writes - * `recapKey` whenever a run produces at least one `TestsFailedException`, - * and never removes it. A successful test run after a failure leaves the - * stale attribute in place; the next failure will overwrite it. We do not - * attempt to recognize "this is a test invocation" at the aggregation - * boundary to avoid a hardcoded list of test-task labels (or a - * Tags-detection design exercise). - */ -private[sbt] object TestRecap: - - /** - * A single failed test task contributing to the recap. - * - * `testOutput` is sanitized by [[collect]]: its `SuiteResult.throwables` are dropped so the - * recap cannot pin the test class loader. See [[collect]] for why. - */ - final case class Failure(taskName: String, testOutput: Option[Tests.Output]) - - /** - * State attribute holding the collected failures from the most recent - * aggregated run that produced at least one `TestsFailedException`. - * Monotonic-latest-failure semantics: never cleared on success, only - * overwritten by the next failure. - * - * Note for consumers: the `SuiteResult.throwables` reachable from these - * failures are always empty -- [[collect]] strips them so the recap cannot - * pin the test class loader. An empty `throwables` here therefore means - * "not retained", NOT "no exception was thrown". The real throwables live on - * the `TestsFailedException` in the `Incomplete` tree, which is where error - * reporting reads them from. - */ - val recapKey: AttributeKey[Vector[Failure]] = AttributeKey[Vector[Failure]]( - "testRecap", - "Failures collected from the most recent aggregated test run" - ) - - /** - * Walk the `Incomplete` tree and return one `Failure` per - * `TestsFailedException`. Exceptions without a payload (e.g., the - * back-compat no-arg constructor escaping a path that didn't get wrapped - * at the task boundary) still contribute a stub entry so the recap lists - * at least the task name when one is available. - * - * Identity-deduplicated via `Incomplete.allExceptions` (which uses an - * `IDSet[Throwable]` internally), so a single failing task shared across - * multiple Incomplete paths in a DAG is counted once. - * - * The retained `Tests.Output` is stripped of `SuiteResult.throwables`: the - * recap only renders names and counts, but a test-thrown Throwable's - * backtrace pins the `Class` objects of every frame, and a `Class` strongly - * references its defining class loader. Since this data is stashed on - * `State.attributes` where it outlives the command, retaining the throwables - * would keep the test class loader -- and its open jar handles -- alive for - * the rest of the session. On Windows those handles make the cached jars - * undeletable (e.g. by `clearCaches`). - */ - def collect(i: Incomplete): Vector[Failure] = - Incomplete - .allExceptions(i) - .iterator - .flatMap { - case e: TestsFailedException => - Some(Failure(e.taskName, e.testOutput.map(dropThrowables))) - case _ => None - } - .toVector - - private def dropThrowables(o: Tests.Output): Tests.Output = - o.copy(events = o.events.view.mapValues(dropThrowables).toMap) - - private def dropThrowables(s: SuiteResult): SuiteResult = - if s.throwables.isEmpty then s - else - new SuiteResult( - s.result, - s.passedCount, - s.failureCount, - s.errorCount, - s.skippedCount, - s.ignoredCount, - s.canceledCount, - s.pendingCount, - ) - - /** - * The rendered recap as a sequence of `\n`-free lines. Failures are - * sorted by `taskName` (lexicographically; empty task names last) for - * stable, diff-friendly output across runs. - */ - def render(failures: Vector[Failure]): Vector[String] = - if failures.isEmpty then Vector.empty - else - val sorted = failures.sortBy(f => (f.taskName.isEmpty, f.taskName)) - val n = sorted.size - val plural = if n == 1 then "" else "s" - val lines = Vector.newBuilder[String] - lines += s"Test failures recap ($n test task$plural failed):" - sorted.foreach { f => - val displayName = if f.taskName.isEmpty then "" else f.taskName - f.testOutput match - case None => - lines += s" $displayName: (no details)" - case Some(out) => - lines += s" $displayName: ${TestResultLogger.Defaults.countsString(out)}" - val failed = collectByResult(out, TestResult.Failed) - val errored = collectByResult(out, TestResult.Error) - if failed.nonEmpty then - lines += " Failed tests:" - failed.foreach(name => lines += s" $name") - if errored.nonEmpty then - lines += " Error during tests:" - errored.foreach(name => lines += s" $name") - } - lines.result() - - /** Render `failures` and emit one error-level log line per rendered line. */ - def formatTo(log: Logger, failures: Vector[Failure]): Unit = - render(failures).foreach(line => log.error(line)) - - private def collectByResult(o: Tests.Output, target: TestResult): Vector[String] = - // Mirrors `TestResultLogger.Defaults.printFailures` so the per-task - // "Failed tests:" block and the cross-project recap render the same - // suite name. Whether `NameTransformer.decode` should be applied to - // suite FQNs at all is debatable, but changing both sites belongs in - // a separate cleanup. - o.events.iterator - .collect { - case (name, suite) if suite.result == target => - scala.reflect.NameTransformer.decode(name) - } - .toVector - .sorted - -end TestRecap diff --git a/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala b/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala new file mode 100644 index 000000000..679e343e4 --- /dev/null +++ b/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala @@ -0,0 +1,267 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import sbt.TestResultLogger.Defaults.Summary +import sbt.internal.util.Terminal +import sbt.protocol.testing.TestResult +import sbt.util.Logger + +import scala.Console.* +import scala.util.Using + +object TestResultLoggerSummaryTest extends verify.BasicTestSuite: + + private def output(suites: (String, SuiteResult)*): Tests.Output = + Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty) + + private def suite(passed: Int, failed: Int = 0, errors: Int = 0): SuiteResult = + val result = + if errors > 0 then TestResult.Error + else if failed > 0 then TestResult.Failed + else TestResult.Passed + new SuiteResult(result, passed, failed, errors, 0, 0, 0, 0) + + private def entry( + taskName: String, + out: Tests.Output, + cached: Vector[String] = Vector.empty + ): (Tests.Output, String, Vector[String]) = (out, taskName, cached) + + private def render(mode: TestSummary, entries: Vector[(Tests.Output, String, Vector[String])]) = + Summary.render(mode, entries, isColorEnabled = false) + + private class Capture extends Logger with AutoCloseable: + val lines: scala.collection.mutable.ArrayBuffer[(String, String)] = + scala.collection.mutable.ArrayBuffer.empty + override def trace(t: => Throwable): Unit = () + override def success(msg: => String): Unit = () + override def log(level: sbt.util.Level.Value, msg: => String): Unit = + lines += level.toString -> msg + def close(): Unit = () + end Capture + + private def withLog[A1](f: Capture => A1): A1 = + Using.resource(new Capture): log => + f(log) + + test("render is empty when nothing ran and nothing was cached") { + val entries = Vector(entry("a / Test / test", output())) + assert(render(TestSummary.default, entries).isEmpty) + assert(render(TestSummary.Success, entries).isEmpty) + } + + test("Default renders one aggregate line across executed and cached tests") { + val entries = Vector( + entry("a / Test / test", output("A" -> suite(passed = 2))), + entry("b / Test / test", output("B" -> suite(passed = 1)), Vector("B2")), + ) + assert( + render(TestSummary.default, entries) == + Vector("passed: total 3, failed 0, errors 0, passed 3, cached 1") + ) + } + + test("render leads with the aggregate status") { + val entries = Vector(entry("a / Test / test", output("A" -> suite(passed = 1, failed = 1)))) + assert( + render(TestSummary.default, entries).last == + "failed: total 1, failed 1, errors 0, passed 0, cached 0" + ) + val errored = Vector(entry("a / Test / test", output("A" -> suite(passed = 0, errors = 1)))) + assert( + render(TestSummary.default, errored).last == + "error: total 1, failed 0, errors 1, passed 0, cached 0" + ) + } + + test("Default lists only the non-passing task among a mix of pass/fail/cached") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ), + entry("b / Test / test", output("example.Passing" -> suite(passed = 1))), + entry("c / Test / test", output(), Vector("example.Cached")), + ) + assert( + render(TestSummary.default, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + "", + "failed: total 3, failed 1, errors 0, passed 2, cached 1", + ) + ) + } + + test("Success renders the per-suite detail with cached markers") { + val entries = Vector( + entry("core2 / Test / testQuick", output("example.ExampleSuite2" -> suite(passed = 1))), + entry( + "core1 / Test / testQuick", + output("example.ExampleSuite1" -> suite(passed = 1)), + Vector("example.ExampleTest1B"), + ), + entry("core3 / Test / testQuick", output(), Vector("example.ExampleSuite3")), + ) + assert( + render(TestSummary.Success, entries) == + Vector( + "Test summary (3 test tasks succeeded):", + " core1 / Test / testQuick", + " example.ExampleSuite1 PASS", + " example.ExampleTest1B (cached) PASS", + " core2 / Test / testQuick", + " example.ExampleSuite2 PASS", + " core3 / Test / testQuick", + " example.ExampleSuite3 (cached) PASS", + "", + "passed: total 4, failed 0, errors 0, passed 4, cached 2", + ) + ) + } + + test("Success header counts only the failing task even though the body lists every task") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ), + entry("b / Test / test", output("example.Passing" -> suite(passed = 1))), + ) + assert( + render(TestSummary.Success, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + " b / Test / test", + " example.Passing PASS", + "", + "failed: total 2, failed 1, errors 0, passed 1, cached 0", + ) + ) + } + + test("Failure lists only failing suites, collapsing to the aggregate line when all pass") { + val failedSuite = new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0) + val entries = Vector( + entry( + "a / Test / test", + output("example.Passing" -> suite(passed = 1), "example.Failing" -> failedSuite), + Vector("example.Cached"), + ) + ) + assert( + render(TestSummary.Failure, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + "", + "failed: total 3, failed 1, errors 0, passed 2, cached 1", + ) + ) + val allPassing = + Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1)))) + assert( + render(TestSummary.Failure, allPassing) == + Vector("passed: total 1, failed 0, errors 0, passed 1, cached 0") + ) + } + + test("isColorEnabled wraps only the status word, not a (cached) prefix") { + val entries = Vector( + entry( + "a / Test / test", + output( + "example.Passing" -> suite(passed = 1), + "example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0), + ), + Vector("example.Cached"), + ) + ) + val lines = Summary.render(TestSummary.Success, entries, isColorEnabled = true) + assert(lines.exists(_.endsWith(s"${GREEN}PASS$RESET"))) + assert(lines.exists(_.endsWith(s"${RED}FAIL$RESET"))) + assert(lines.exists(l => l.contains("(cached) ") && l.endsWith(s"${GREEN}PASS$RESET"))) + assert(!lines.exists(_.contains(s"$GREEN(cached)"))) + } + + test("None suppresses all output, even on failure") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ) + ) + assert(render(TestSummary.none, entries).isEmpty) + } + + test("summary logs at info level when everything passes") { + withLog: log => + val entries = Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1)))) + Summary(TestSummary.Success).summary(log, entries) + val expected = Summary + .render(TestSummary.Success, entries, Terminal.get.isColorEnabled) + .map(line => if line.isEmpty then " " else line) + assert(log.lines.map(_._2).toVector == expected) + assert(log.lines.forall(_._1 == "info")) + } + + test("summary logs at error level when entries contain a failure") { + withLog: log => + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)) + ) + ) + Summary(TestSummary.default).summary(log, entries) + val expected = Summary + .render(TestSummary.default, entries, Terminal.get.isColorEnabled) + .map(line => if line.isEmpty then " " else line) + assert(log.lines.map(_._2).toVector == expected) + assert(log.lines.nonEmpty) + assert(log.lines.forall(_._1 == "error")) + } + + test("Summary.run delegates to the Default per-task logger"): + withLog: log => + Summary().run(log, output("A" -> suite(passed = 1)), "a / Test / test") + assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0")) + + test("printStandard's 4-arg run folds the cached count into total and passed"): + withLog: log => + TestResultLogger.Default.run( + log, + output("A" -> suite(passed = 1)), + "a / Test / test", + Vector("B", "C"), + ) + assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2")) + + test("printStandard's 3-arg run always shows the cached count, even at zero"): + withLog: log => + TestResultLogger.Default.run(log, output("A" -> suite(passed = 1)), "a / Test / test") + assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0")) + + test("choose (via SilentWhenNoTests) forwards the cached count to the chosen branch"): + withLog: log => + TestResultLogger.SilentWhenNoTests.run( + log, + output("A" -> suite(passed = 1)), + "a / Test / test", + Vector("B", "C"), + ) + assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2")) + +end TestResultLoggerSummaryTest diff --git a/main-actions/src/test/scala/sbt/TestSummaryTest.scala b/main-actions/src/test/scala/sbt/TestSummaryTest.scala new file mode 100644 index 000000000..94dc15f7b --- /dev/null +++ b/main-actions/src/test/scala/sbt/TestSummaryTest.scala @@ -0,0 +1,30 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import sbt.protocol.testing.TestResult + +object TestSummaryTest extends verify.BasicTestSuite: + + private def output(suites: (String, SuiteResult)*): Tests.Output = + Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty) + + test("append strips SuiteResult.throwables so lingering entries cannot pin the classloader") { + val thrown = new AssertionError("boom") + val withThrowables = + new SuiteResult(TestResult.Passed, 1, 0, 0, 0, 0, 0, 0, thrown :: Nil) + TestSummary.clear() + TestSummary.append("a / Test / test", output("A" -> withThrowables), Vector.empty, Vector.empty) + val drained = TestSummary.drain() + assert(drained.size == 1) + assert(drained.head.testOutput.events("A").throwables.isEmpty) + assert(TestSummary.drain().isEmpty) + } + +end TestSummaryTest diff --git a/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala b/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala deleted file mode 100644 index c8be710d6..000000000 --- a/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala +++ /dev/null @@ -1,263 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt -package internal -package testing - -import sbt.Incomplete -import sbt.SuiteResult -import sbt.Tests -import sbt.TestsFailedException -import sbt.protocol.testing.TestResult -import sbt.util.Logger - -object TestRecapTest extends verify.BasicTestSuite: - - private def output(result: TestResult, suites: (String, SuiteResult)*): Tests.Output = - Tests.Output(result, suites.toMap, Iterable.empty) - - private def suite(result: TestResult): SuiteResult = - new SuiteResult(result, 0, 1, 0, 0, 0, 0, 0) - - private def failure( - taskName: String, - result: TestResult, - suiteName: String - ): TestsFailedException = - new TestsFailedException( - taskName, - Some(output(result, suiteName -> suite(result))) - ) - - /** Build an Incomplete tree carrying the given TestsFailedExceptions as direct causes. */ - private def incompleteOf(exceptions: TestsFailedException*): Incomplete = - new Incomplete( - node = None, - causes = exceptions.map(e => new Incomplete(node = None, directCause = Some(e))) - ) - - private class Capture extends Logger: - val lines: scala.collection.mutable.ArrayBuffer[(String, String)] = - scala.collection.mutable.ArrayBuffer.empty - override def trace(t: => Throwable): Unit = () - override def success(msg: => String): Unit = () - override def log(level: sbt.util.Level.Value, msg: => String): Unit = - lines += level.toString -> msg - - test("collect strips SuiteResult.throwables so the recap cannot pin the test classloader") { - // A test-thrown Throwable's backtrace pins the Class objects of every frame, and a Class - // strongly references its defining classloader; the recap is stashed on State.attributes, - // so retaining them would keep the test classloader (and its jar handles) alive. - val thrown = new AssertionError("boom") - val withThrowables = - new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil) - val i = incompleteOf( - new TestsFailedException( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> withThrowables)) - ) - ) - val collected = TestRecap.collect(i) - val retained = collected.head.testOutput.get.events("AFailing") - assert(retained.throwables.isEmpty, "throwables should be dropped from the retained output") - // Everything the recap actually renders must survive. - assert(retained.result == TestResult.Failed) - assert(retained.failureCount == 1) - assert(collected.head.testOutput.get.overall == TestResult.Failed) - } - - test("collect sanitizes a copy and leaves the source exception's throwables intact") { - // The stripping above must not mutate the TestsFailedException in the Incomplete tree: - // that exception is what error reporting and the ClassLoaderLayeringStrategy diagnostic - // in Defaults read, and both need the real throwables. Only the copy parked on - // State.attributes is sanitized. - val thrown = new AssertionError("boom") - val source = new TestsFailedException( - "a / Test / test", - Some( - output( - TestResult.Failed, - "AFailing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil) - ) - ) - ) - val collected = TestRecap.collect(incompleteOf(source)) - assert(collected.head.testOutput.get.events("AFailing").throwables.isEmpty) - val original = source.testOutput.get.events("AFailing").throwables - assert( - original == (thrown :: Nil), - s"collect must not strip the source exception, but its throwables became: $original" - ) - } - - test("collect picks up TestsFailedException payloads from the Incomplete tree") { - val i = incompleteOf( - failure("a / Test / test", TestResult.Failed, "AFailing"), - failure("c / Test / test", TestResult.Error, "CErroring"), - ) - val collected = TestRecap.collect(i) - assert(collected.map(_.taskName).sorted == Vector("a / Test / test", "c / Test / test")) - val resultsBy = collected.flatMap(f => f.testOutput.map(o => f.taskName -> o.overall)).toMap - assert(resultsBy("a / Test / test") == TestResult.Failed) - assert(resultsBy("c / Test / test") == TestResult.Error) - } - - test("collect retains TestsFailedException without payload as a stub entry") { - val noDetail = new TestsFailedException // back-compat no-arg constructor - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(noDetail)), - new Incomplete( - node = None, - directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) - ), - ) - ) - val collected = TestRecap.collect(i) - assert(collected.size == 2, s"expected 2 entries, got $collected") - val stub = collected.find(_.testOutput.isEmpty) - assert(stub.isDefined, "no-detail failure should still produce a Failure entry") - assert(stub.get.taskName == "") - } - - test("collect skips exceptions that aren't TestsFailedException") { - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(new RuntimeException("nope"))), - new Incomplete( - node = None, - directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) - ), - ) - ) - assert(TestRecap.collect(i).map(_.taskName) == Vector("ok / Test / test")) - } - - test("collect retains taskName when a TestsFailedException is rebranded with task name only") { - // Simulates the `testFull` / `inputTests0` catch path: an upstream - // legacy code path threw `new TestsFailedException()` (no detail), the - // task wrapper caught it and re-threw with `(taskName, e.testOutput)` - // to attach the project context. - val tagged = new TestsFailedException("a / Test / test", None) - val i = new Incomplete(node = None, directCause = Some(tagged)) - val collected = TestRecap.collect(i) - assert(collected == Vector(TestRecap.Failure("a / Test / test", None))) - } - - test("collect deduplicates a TestsFailedException shared across Incomplete paths") { - val shared = failure("a / Test / test", TestResult.Failed, "AFail") - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(shared)), - new Incomplete(node = None, directCause = Some(shared)), - ) - ) - val collected = TestRecap.collect(i) - assert( - collected.size == 1, - s"shared failure should be reported once, got ${collected.size}: $collected" - ) - } - - test("render emits header, per-task counts, and indented suite names") { - val failures = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ), - TestRecap.Failure( - "c / Test / test", - Some(output(TestResult.Error, "CErroring" -> suite(TestResult.Error))) - ), - ) - val lines = TestRecap.render(failures) - assert(lines.headOption.contains("Test failures recap (2 test tasks failed):")) - assert(lines.exists(_.contains("a / Test / test:"))) - assert(lines.exists(_.contains("c / Test / test:"))) - assert(lines.exists(_.contains("AFailing"))) - assert(lines.exists(_.contains("CErroring"))) - assert(lines.contains(" Failed tests:")) - assert(lines.contains(" Error during tests:")) - } - - test("render sorts failures by taskName for stable output (empty names last)") { - val failures = Vector( - TestRecap.Failure("zz / Test / test", None), - TestRecap.Failure("", None), - TestRecap.Failure("aa / Test / test", None), - TestRecap.Failure("mm / Test / test", None), - ) - val lines = TestRecap.render(failures) - val headerLines = lines.filter(_.startsWith(" ")) - val order = headerLines.map(_.trim.takeWhile(_ != ':')) - assert( - order == Vector("aa / Test / test", "mm / Test / test", "zz / Test / test", ""), - s"unexpected order: $order" - ) - } - - test("render emits singular header when exactly one task failed") { - val one = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ) - ) - assert(TestRecap.render(one).head == "Test failures recap (1 test task failed):") - } - - test("render shows '(no details)' for failures without a Tests.Output payload") { - val failures = Vector(TestRecap.Failure("a / Test / test", testOutput = None)) - val lines = TestRecap.render(failures) - assert( - lines.exists(_.contains("a / Test / test: (no details)")), - s"expected '(no details)' entry, got $lines" - ) - } - - test("render shows '' when a failure carries no task name") { - val failures = Vector(TestRecap.Failure(taskName = "", testOutput = None)) - val lines = TestRecap.render(failures) - assert( - lines.exists(_.contains(": (no details)")), - s"expected '' placeholder, got $lines" - ) - } - - test("render is empty when there are no failures") { - assert(TestRecap.render(Vector.empty).isEmpty) - } - - test("formatTo emits one error-level log line per rendered line") { - val failures = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ) - ) - val log = new Capture - TestRecap.formatTo(log, failures) - val rendered = TestRecap.render(failures) - assert( - log.lines.size == rendered.size, - s"expected ${rendered.size} log calls, got ${log.lines.size}" - ) - assert(log.lines.forall(_._1 == "error"), s"all lines should be error level: ${log.lines}") - } - - test("formatTo is a no-op when there are no failures") { - val log = new Capture - TestRecap.formatTo(log, Vector.empty) - assert(log.lines.isEmpty) - } - -end TestRecapTest diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 116427c4d..dd9f5a930 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1227,7 +1227,9 @@ object Defaults extends BuildCommon with DefExtra { testListeners :== Nil, testOptions :== Nil, testOptionDigests :== Nil, - testResultLogger :== TestResultLogger.Default, + testResultLogger :== TestResultLogger.SilentWhenNoTests, + testSummary :== SysProp.testSummary, + testSummaryLogger := TestResultLogger.Defaults.Summary(testSummary.value), testOnly / testFilter :== (IncrementalTest.selectedFilter), testSelected / testFilter :== (IncrementalTest.selectedFilter), extraTestDigests :== Nil, @@ -1295,9 +1297,10 @@ object Defaults extends BuildCommon with DefExtra { val taskName = Project.showContextKey(state.value).show(resolvedScoped.value) try val output = executeTests.value + TestSummary.append(taskName, output, cached = Vector.empty, adhocOptions = Vector.empty) trl.run(streams.value.log, output, taskName) - // Throw with task name + Output so the cross-project recap - // (TestRecap.collect) can surface them. The throw lives here + // Throw with task name + Output so the aggregation boundary + // (Aggregation.runTasks) can signal the failure. The throw lives here // rather than in TestResultLogger so user-overridden loggers // cannot accidentally suppress the failure signal. output.overall match @@ -1309,7 +1312,7 @@ object Defaults extends BuildCommon with DefExtra { // Tag any no-detail TestsFailedException (legacy executeTests // adapters, third-party Tests.Setup actions, anything constructing // `new TestsFailedException()` via the back-compat ctor) with the - // task name on its way out so the recap doesn't render . + // task name on its way out so error reporting has it to show. case e: TestsFailedException if e.taskName.isEmpty => throw new TestsFailedException(taskName, e.testOutput) finally close(testLoader.value) @@ -1463,8 +1466,9 @@ object Defaults extends BuildCommon with DefExtra { inputTests0.mapReferenced(Def.mapScope((s) => s.rescope(key.key))) private lazy val inputTests0: Initialize[InputTask[TestResult]] = { - val parser = loadForParser(definedTestNames)((s, i) => testOnlyParser(s, i getOrElse Nil)) - ParserGen(parser).flatMapTask { (selected, frameworkOptions) => + val parser = + loadForParser(definedTestNames)((s, i) => testOnlyParserWithOption(s, i getOrElse Nil)) + ParserGen(parser).flatMapTask { (selected, frameworkOptions, adhocOptions) => val s = streams.value val filter = testFilter.value val config = testExecution.value @@ -1505,11 +1509,21 @@ object Defaults extends BuildCommon with DefExtra { ) val taskName = display.show(resolvedScoped.value) val trl = testResultLogger.value + val digests = definedTestDigests.value + val cacheConfig = Def.cacheConfiguration.value (Def .value[Task[Tests.Output]] { output }) .map: out => + val cached = IncrementalTest.cachedTestNames( + digests, + cacheConfig, + out.events.keySet, + selected, + frameworkOptions, + ) + TestSummary.append(taskName, out, cached, adhocOptions.toVector) try - trl.run(s.log, out, taskName) + trl.run(s.log, out, taskName, cached) out.overall match case TestResult.Error | TestResult.Failed => throw new TestsFailedException(taskName, Some(out)) @@ -2629,6 +2643,17 @@ object Defaults extends BuildCommon with DefExtra { Space ~> token(NotSpace.examples(mainClasses.toSet)) ~ spaceDelimited("") } + private def testOnlyParserWithOption + : (State, Seq[String]) => Parser[(Seq[String], Seq[String], Seq[Tests.AdhocOption])] = + (state, tests) => + import DefaultParsers.* + val selectTests = distinctParser(tests.toSet, true) + val frameworkOpts = (token(Space) ~> token("--") ~> spaceDelimited("