From 27c3f035e55307fc9ca30365175ad8092528d7ad Mon Sep 17 00:00:00 2001 From: eugene yokota Date: Mon, 17 Aug 2026 23:54:32 -0400 Subject: [PATCH] [2.x] feat: Test summary (#9602) **Problem/Solution** This extends the idea started with TestRecap, and applies it to both test success and failures. 1. Existing TestResultLogger trait is extended to handle the summary rendering. 2. TestSummary enum is added to control the verbosity via commandline option, system property, or a setting. 3. Script test captures the log. --- .../src/main/scala/sbt/TestResultLogger.scala | 263 +++++++++++++---- .../src/main/scala/sbt/TestSummary.scala | 109 +++++++ main-actions/src/main/scala/sbt/Tests.scala | 23 ++ .../sbt/internal/testing/TestRecap.scala | 171 ----------- .../sbt/TestResultLoggerSummaryTest.scala | 267 ++++++++++++++++++ .../src/test/scala/sbt/TestSummaryTest.scala | 30 ++ .../sbt/internal/testing/TestRecapTest.scala | 263 ----------------- main/src/main/scala/sbt/Defaults.scala | 39 ++- main/src/main/scala/sbt/Keys.scala | 3 + .../main/scala/sbt/internal/Aggregation.scala | 28 +- .../scala/sbt/internal/IncrementalTest.scala | 27 ++ .../main/scala/sbt/internal/LintUnused.scala | 1 + .../src/main/scala/sbt/internal/SysProp.scala | 17 +- .../tests/multi-failure-recap/build.sbt | 16 -- .../c/src/test/java/FailingTestC.java | 6 - .../multi-failure-recap/project/Checks.scala | 72 ----- .../sbt-test/tests/multi-failure-recap/test | 16 -- .../a/src/test/java/com/example/a/TestA.java | 7 + .../a/src/test/java/com/example/a/TestAA.java | 7 + .../b/src/test/java/com/example/b/TestB.java | 7 + .../src/sbt-test/tests/summary/both.log.check | 10 + sbt-app/src/sbt-test/tests/summary/build.sbt | 57 ++++ .../c/src/test/java/TestC.java} | 2 +- .../changes/BadTestA.java} | 4 +- .../tests/summary/changes/BadTestB.java | 8 + .../sbt-test/tests/summary/failure.log.check | 7 + .../src/sbt-test/tests/summary/none.log.check | 0 .../tests/summary/project/Capture.scala | 69 +++++ .../sbt-test/tests/summary/success.log.check | 10 + sbt-app/src/sbt-test/tests/summary/test | 39 +++ .../src/sbt-test/tests/summary/test.log.check | 2 + .../tests/summary/testquick.log.check | 3 + .../tests/summary/testquick2.log.check | 9 + .../src/test/scala/testpkg/ClientTest.scala | 2 +- .../main/scala/sbt/TestReportListener.scala | 4 +- 35 files changed, 970 insertions(+), 628 deletions(-) create mode 100644 main-actions/src/main/scala/sbt/TestSummary.scala delete mode 100644 main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala create mode 100644 main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala create mode 100644 main-actions/src/test/scala/sbt/TestSummaryTest.scala delete mode 100644 main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala delete mode 100644 sbt-app/src/sbt-test/tests/multi-failure-recap/build.sbt delete mode 100644 sbt-app/src/sbt-test/tests/multi-failure-recap/c/src/test/java/FailingTestC.java delete mode 100644 sbt-app/src/sbt-test/tests/multi-failure-recap/project/Checks.scala delete mode 100644 sbt-app/src/sbt-test/tests/multi-failure-recap/test create mode 100644 sbt-app/src/sbt-test/tests/summary/a/src/test/java/com/example/a/TestA.java create mode 100644 sbt-app/src/sbt-test/tests/summary/a/src/test/java/com/example/a/TestAA.java create mode 100644 sbt-app/src/sbt-test/tests/summary/b/src/test/java/com/example/b/TestB.java create mode 100644 sbt-app/src/sbt-test/tests/summary/both.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/build.sbt rename sbt-app/src/sbt-test/tests/{multi-failure-recap/b/src/test/java/PassingTestB.java => summary/c/src/test/java/TestC.java} (72%) rename sbt-app/src/sbt-test/tests/{multi-failure-recap/a/src/test/java/FailingTestA.java => summary/changes/BadTestA.java} (74%) create mode 100644 sbt-app/src/sbt-test/tests/summary/changes/BadTestB.java create mode 100644 sbt-app/src/sbt-test/tests/summary/failure.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/none.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/project/Capture.scala create mode 100644 sbt-app/src/sbt-test/tests/summary/success.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/test create mode 100644 sbt-app/src/sbt-test/tests/summary/test.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/testquick.log.check create mode 100644 sbt-app/src/sbt-test/tests/summary/testquick2.log.check diff --git a/main-actions/src/main/scala/sbt/TestResultLogger.scala b/main-actions/src/main/scala/sbt/TestResultLogger.scala index f344bbe7d..fbcdbb010 100644 --- a/main-actions/src/main/scala/sbt/TestResultLogger.scala +++ b/main-actions/src/main/scala/sbt/TestResultLogger.scala @@ -8,7 +8,8 @@ package sbt -import sbt.Tests.{ Output, Summary } +import sbt.Tests.Output +import sbt.internal.util.Terminal import sbt.protocol.testing.TestResult import sbt.util.{ Level, Logger } @@ -20,7 +21,7 @@ import sbt.util.{ Level, Logger } * * @since 0.13.5 */ -trait TestResultLogger { +trait TestResultLogger: /** * Perform logging. @@ -31,6 +32,14 @@ trait TestResultLogger { */ def run(log: Logger, results: Output, taskName: String): Unit + def run(log: Logger, results: Output, taskName: String, cached: Vector[String]): Unit = + run(log, results, taskName) + + def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit = + entries.foreach { case (results, taskName, _) => + run(log, results, taskName) + } + /** Only allow invocation if certain criteria is met, else use another `TestResultLogger` (defaulting to nothing) . */ final def onlyIf( f: (Output, String) => Boolean, @@ -44,9 +53,9 @@ trait TestResultLogger { otherwise: TestResultLogger = TestResultLogger.Null ) = TestResultLogger.choose(f, otherwise, this) -} +end TestResultLogger -object TestResultLogger { +object TestResultLogger: /** A `TestResultLogger` that does nothing. */ val Null = const(_ => ()) @@ -61,6 +70,19 @@ object TestResultLogger { def apply(f: (Logger, Output, String) => Unit): TestResultLogger = (log, results, taskName) => f(log, results, taskName) + /** Creates a `TestResultLogger` using a given function that also receives the cached suite names. */ + def apply(f: (Logger, Output, String, Vector[String]) => Unit): TestResultLogger = + new TestResultLogger: + def run(log: Logger, results: Output, taskName: String): Unit = + f(log, results, taskName, Vector.empty) + override def run( + log: Logger, + results: Output, + taskName: String, + cached: Vector[String] + ): Unit = + f(log, results, taskName, cached) + /** Creates a `TestResultLogger` that ignores its input and always performs the same logging. */ def const(f: Logger => Unit) = apply((l, _, _) => f(l)) @@ -71,8 +93,8 @@ object TestResultLogger { * @param f The `TestResultLogger` to choose if the predicate fails. */ def choose(cond: (Output, String) => Boolean, t: TestResultLogger, f: TestResultLogger) = - TestResultLogger((log, results, taskName) => - (if (cond(results, taskName)) t else f).run(log, results, taskName) + TestResultLogger((log, results, taskName, cached) => + (if cond(results, taskName) then t else f).run(log, results, taskName, cached) ) /** Transforms the input to be completely silent when the subject module doesn't contain any tests. */ @@ -82,7 +104,44 @@ object TestResultLogger { printNoTests = Null ) - object Defaults { + object Defaults: + private val suitePadding = " " * 8 + + private[sbt] enum SummaryStatus: + case Passed, Failed, Errored + + def label: String = this match + case SummaryStatus.Passed => "passed" + case SummaryStatus.Failed => "failed" + case SummaryStatus.Errored => "error" + + def word: String = this match + case SummaryStatus.Passed => "succeeded" + case SummaryStatus.Failed => "failed" + case SummaryStatus.Errored => "errored" + end SummaryStatus + + private[sbt] enum SuiteStatus: + case Pass, CachedPass, Fail, Error + + def isPassing: Boolean = this match + case SuiteStatus.Pass | SuiteStatus.CachedPass => true + case SuiteStatus.Fail | SuiteStatus.Error => false + + def render(isColorEnabled: Boolean): String = + val (prefix, word, color) = this match + case SuiteStatus.Pass => ("", "PASS", scala.Console.GREEN) + case SuiteStatus.CachedPass => ("(cached) ", "PASS", scala.Console.GREEN) + case SuiteStatus.Fail => ("", "FAIL", scala.Console.RED) + case SuiteStatus.Error => ("", "ERROR", scala.Console.RED) + if isColorEnabled then s"$prefix$color$word${scala.Console.RESET}" else s"$prefix$word" + end SuiteStatus + + private[sbt] object SuiteStatus: + def apply(r: TestResult): SuiteStatus = r match + case TestResult.Passed | TestResult.Empty => SuiteStatus.Pass + case TestResult.Failed => SuiteStatus.Fail + case TestResult.Error => SuiteStatus.Error /** sbt's default `TestResultLogger`. Use `copy()` to change selective portions. */ case class Main( @@ -93,8 +152,16 @@ object TestResultLogger { printNoTests: TestResultLogger = Defaults.printNoTests ) extends TestResultLogger { - override def run(log: Logger, results: Output, taskName: String): Unit = { - def run(r: TestResultLogger): Unit = r.run(log, results, taskName) + override def run(log: Logger, results: Output, taskName: String): Unit = + run(log, results, taskName, Vector.empty) + + override def run( + log: Logger, + results: Output, + taskName: String, + cached: Vector[String] + ): Unit = { + def run(r: TestResultLogger): Unit = r.run(log, results, taskName, cached) run(printSummary) @@ -117,7 +184,7 @@ object TestResultLogger { val printSummary = TestResultLogger((log, results, _) => { val multipleFrameworks = results.summaries.size > 1 - for (Summary(name, message) <- results.summaries) + for Tests.Summary(name, message) <- results.summaries do if (message.isEmpty) log.debug("Summary for " + name + " not available.") else { if (multipleFrameworks) log.info(name) @@ -130,54 +197,40 @@ object TestResultLogger { // Print the standard one-liner statistic if no framework summary is defined, or when > 1 framework is in used. results.summaries.size > 1 || results.summaries.headOption.forall(_.summaryText.isEmpty) - val printStandard = TestResultLogger((log, results, _) => { - val counts = countsString(results) + val printStandard = TestResultLogger((log, results, _, cached) => { + val counts = countsString(results.events.values, cached.size, true) results.overall match case TestResult.Empty => () - case TestResult.Error => log.error("Error: " + counts) - case TestResult.Passed => log.info("Passed: " + counts) - case TestResult.Failed => log.error("Failed: " + counts) + case TestResult.Error => log.error(s"${SummaryStatus.Errored.label}: $counts") + case TestResult.Passed => log.info(s"${SummaryStatus.Passed.label}: $counts") + case TestResult.Failed => log.error(s"${SummaryStatus.Failed.label}: $counts") }) + private[sbt] def countsString(events: Iterable[SuiteResult]): String = + countsString(events, 0, false) + /** - * Renders `Tests.Output`'s aggregate counts as a single line like - * `Total 10, Failed 2, Errors 0, Passed 8`. Shared between `printStandard` - * and the cross-project recap formatter (see `TestRecap`). + * Renders suite counts as a single line like `total 10, failed 2, errors + * 0, passed 8`. Counts suites (classes/objects), not individual test examples. */ - private[sbt] def countsString(results: Output): String = { - val ( - skippedCount, - errorsCount, - passedCount, - failuresCount, - ignoredCount, - canceledCount, - pendingCount, - ) = - results.events.foldLeft((0, 0, 0, 0, 0, 0, 0)) { case (acc, (_, testEvent)) => - val (skippedAcc, errorAcc, passedAcc, failureAcc, ignoredAcc, canceledAcc, pendingAcc) = - acc - ( - skippedAcc + testEvent.skippedCount, - errorAcc + testEvent.errorCount, - passedAcc + testEvent.passedCount, - failureAcc + testEvent.failureCount, - ignoredAcc + testEvent.ignoredCount, - canceledAcc + testEvent.canceledCount, - pendingAcc + testEvent.pendingCount, - ) + private[sbt] def countsString( + events: Iterable[SuiteResult], + cachedCount: Int, + alwaysShowCached: Boolean, + ): String = { + val (failuresCount, errorsCount, passedCount) = + events.foldLeft((0, 0, 0)) { case ((failureAcc, errorAcc, passedAcc), suite) => + suite.result match + case TestResult.Failed => (failureAcc + 1, errorAcc, passedAcc) + case TestResult.Error => (failureAcc, errorAcc + 1, passedAcc) + case TestResult.Passed | TestResult.Empty => (failureAcc, errorAcc, passedAcc + 1) } - val totalCount = failuresCount + errorsCount + skippedCount + passedCount + val totalCount = failuresCount + errorsCount + passedCount + cachedCount val base = - s"Total $totalCount, Failed $failuresCount, Errors $errorsCount, Passed $passedCount" - val otherCounts = Seq( - "Skipped" -> skippedCount, - "Ignored" -> ignoredCount, - "Canceled" -> canceledCount, - "Pending" -> pendingCount - ) - val extra = otherCounts.withFilter(_._2 > 0).map { (label, count) => s", $label $count" } - base + extra.mkString + s"total $totalCount, failed $failuresCount, errors $errorsCount, passed ${passedCount + cachedCount}" + val cachedField = + if cachedCount > 0 || alwaysShowCached then s", cached $cachedCount" else "" + base + cachedField } val printFailures = TestResultLogger((log, results, _) => { @@ -189,15 +242,109 @@ object TestResultLogger { def show(label: String, level: Level.Value, tests: Iterable[String]): Unit = if (tests.nonEmpty) { log.log(level, label) - log.log(level, tests.mkString("\t", "\n\t", "")) + log.log(level, tests.mkString(suitePadding, s"\n$suitePadding", "")) } - show("Passed tests:", Level.Debug, select(TestResult.Passed)) - show("Failed tests:", Level.Error, select(TestResult.Failed)) - show("Error during tests:", Level.Error, select(TestResult.Error)) + show("passed tests:", Level.Debug, select(TestResult.Passed)) + show("failed tests:", Level.Error, select(TestResult.Failed)) + show("error during tests:", Level.Error, select(TestResult.Error)) }) - val printNoTests = - TestResultLogger((log, results, taskName) => log.info("No tests to run for " + taskName)) - } -} + val printNoTests = TestResultLogger((log, results, taskName, cached) => + val suffix = if cached.nonEmpty then s" (${cached.size} cached)" else "" + log.debug(s"no tests to run for $taskName$suffix") + ) + + /** + * Renders a cross-project aggregate summary at the end of an aggregated + * run, in the style selected by `mode` (see `TestSummary`). Per-task + * logging is unchanged (delegates to `Default`); `summary` is where this + * differs from a plain `TestResultLogger`. + */ + case class Summary(mode: TestSummary) extends TestResultLogger: + override def run(log: Logger, results: Output, taskName: String): Unit = + Default.run(log, results, taskName) + + override def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit = + val lines = Summary.render(mode, entries, Terminal.get.isColorEnabled) + val logLine: String => Unit = Summary.overallStatus(entries) match + case Some(SummaryStatus.Errored) | Some(SummaryStatus.Failed) => log.error(_) + case _ => log.info(_) + lines.foreach(line => logLine(if line.isEmpty then " " else line)) + end Summary + + object Summary: + private val columnGap = 5 + + def apply(): Summary = Summary(mode = TestSummary.default) + + /** + * The rendered summary; empty when nothing ran. Both styles end with an + * aggregate line, e.g. `passed: total 4, failed 0, errors 0, passed 4, + * cached 2`. Cached classes count into `total` and `passed`, each as one + * (their case counts are unknown without running them). + */ + private[sbt] def render( + mode: TestSummary, + entries: Vector[(Output, String, Vector[String])], + isColorEnabled: Boolean, + ): Vector[String] = + overallStatus(entries) match + case None => Vector.empty + case Some(status) => + val executed = entries.flatMap(_._1.events.values) + val cached = entries.map(_._3.size).sum + val counts = + s"${status.label}: ${countsString(executed, cached, alwaysShowCached = true)}" + def withCounts(detailLines: Vector[String]): Vector[String] = + if detailLines.isEmpty then Vector(counts) else detailLines :+ "" :+ counts + mode match + case TestSummary.None => Vector.empty + case TestSummary.Failure => + withCounts(detail(status, entries, failuresOnly = true, isColorEnabled)) + case TestSummary.Success => + withCounts(detail(status, entries, failuresOnly = false, isColorEnabled)) + + /** The overall status across `entries`, or `None` when nothing ran. */ + private def overallStatus( + entries: Vector[(Output, String, Vector[String])] + ): Option[SummaryStatus] = + val executed = entries.flatMap(_._1.events.values) + val cached = entries.map(_._3.size).sum + if executed.isEmpty && cached == 0 then None + else if executed.exists(_.result == TestResult.Error) then Some(SummaryStatus.Errored) + else if executed.exists(_.result == TestResult.Failed) then Some(SummaryStatus.Failed) + else Some(SummaryStatus.Passed) + + private def detail( + status: SummaryStatus, + entries: Vector[(Output, String, Vector[String])], + failuresOnly: Boolean, + isColorEnabled: Boolean, + ): Vector[String] = + val tasks = entries + .map: (output, taskName, cachedNames) => + val executed = output.events.view.mapValues(s => SuiteStatus(s.result)).toVector + val suites = + if failuresOnly then executed.filter { case (_, st) => !st.isPassing } + else executed ++ cachedNames.map(_ -> SuiteStatus.CachedPass) + taskName -> suites.sortBy(_._1) + .filter(_._2.nonEmpty) + .sortBy(_._1) + if tasks.isEmpty then Vector.empty + else + val headerCount = + if status == SummaryStatus.Passed then tasks.size + else entries.count(_._1.events.values.exists(s => !SuiteStatus(s.result).isPassing)) + val plural = if headerCount == 1 then "" else "s" + val width = tasks.flatMap(_._2.map(_._1.length)).max + columnGap + val lines = Vector.newBuilder[String] + lines += s"Test summary ($headerCount test task$plural ${status.word}):" + tasks.foreach: (taskName, suites) => + lines += s" $taskName" + suites.foreach: (name, st) => + lines += s"${suitePadding}${name.padTo(width, ' ')}${st.render(isColorEnabled)}" + lines.result() + end Summary + end Defaults +end TestResultLogger diff --git a/main-actions/src/main/scala/sbt/TestSummary.scala b/main-actions/src/main/scala/sbt/TestSummary.scala new file mode 100644 index 000000000..584b4b9e3 --- /dev/null +++ b/main-actions/src/main/scala/sbt/TestSummary.scala @@ -0,0 +1,109 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import java.util.concurrent.ConcurrentLinkedQueue +import sbt.internal.util.AttributeKey + +/** + * Style of the test summary displayed after an aggregated test run: + * - `None`: prints nothing + * - `Failure` (the default): lists only non-passing suites, staying terse when everything passes + * - `Success`: always lists every suite, marking cache reuse with `(cached)` + * + * Rendering itself lives in `TestResultLogger.Defaults.Summary`, which + * consumes the entries collected here. + */ +enum TestSummary: + case None + case Failure + case Success + +/** + * Collector surfacing a test summary at the end of an aggregated run, + * success or failure alike. + */ +object TestSummary: + val none: TestSummary = TestSummary.None + val failure: TestSummary = TestSummary.Failure + val success: TestSummary = TestSummary.Success + def default: TestSummary = TestSummary.failure + + /** `cached` names the test classes reused from the action cache (see `IncrementalTest.cachedTestNames`). */ + private[sbt] final case class Entry( + taskName: String, + testOutput: Tests.Output, + cached: Vector[String], + options: Vector[Tests.AdhocOption] + ) + + /** + * State attribute holding the entries from the most recent aggregated run + * that produced at least one test result. Monotonic-latest-run semantics: + * never cleared, only overwritten by the next non-empty run; lets in-JVM + * tools (IDE plugins, BSP servers, scripted tests staying inside one sbt + * invocation via `Command.process`) inspect the last test results without + * parsing log output. Scripted tests crossing a `->` boundary cannot read + * this because the inner sbt's IPC server is torn down on failure and a + * fresh JVM is spawned for the next statement. + */ + private[sbt] val entriesKey: AttributeKey[Vector[Entry]] = AttributeKey[Vector[Entry]]( + "testSummaryEntries", + "Entries collected from the most recent aggregated test run" + ) + + private val entries = new ConcurrentLinkedQueue[Entry] + + /** + * `output`'s `SuiteResult.throwables` are dropped before retention: a + * test-thrown Throwable's backtrace pins the `Class` objects of every + * frame, and a `Class` strongly references its defining class loader. + * Since entries are stashed on `State.attributes` where they outlive the + * command, retaining the throwables would keep the test class loader -- + * and its open jar handles -- alive for the rest of the session. On + * Windows those handles make the cached jars undeletable (e.g. by + * `clearCaches`). + */ + private[sbt] def append( + taskName: String, + output: Tests.Output, + cached: Vector[String], + adhocOptions: Vector[Tests.AdhocOption] + ): Unit = + entries.add(Entry(taskName, dropThrowables(output), cached, adhocOptions)) + () + + private def dropThrowables(o: Tests.Output): Tests.Output = + o.copy(events = o.events.view.mapValues(dropThrowables).toMap) + + private def dropThrowables(s: SuiteResult): SuiteResult = + if s.throwables.isEmpty then s + else + new SuiteResult( + s.result, + s.passedCount, + s.failureCount, + s.errorCount, + s.skippedCount, + s.ignoredCount, + s.canceledCount, + s.pendingCount, + ) + + private[sbt] def clear(): Unit = entries.clear() + + private[sbt] def drain(): Vector[Entry] = + val b = Vector.newBuilder[Entry] + var e = entries.poll() + while e != null do + b += e + e = entries.poll() + b.result() + +end TestSummary diff --git a/main-actions/src/main/scala/sbt/Tests.scala b/main-actions/src/main/scala/sbt/Tests.scala index 883d00ed5..2b42b7e91 100644 --- a/main-actions/src/main/scala/sbt/Tests.scala +++ b/main-actions/src/main/scala/sbt/Tests.scala @@ -8,6 +8,7 @@ package sbt +import java.util.Locale import std.* import xsbt.api.{ Discovered, Discovery } import sbt.internal.inc.Analysis @@ -38,11 +39,33 @@ import sbt.util.{ Digest, Logger } import sbt.protocol.testing.TestResult import scala.runtime.AbstractFunction3 +import sbt.internal.util.complete.{ DefaultParsers, Parser } sealed trait TestOption object Tests { + private[sbt] sealed trait AdhocOption + private[sbt] object AdhocOption: + case class Summary(summary: TestSummary) extends AdhocOption + + private[sbt] def parser: Parser[AdhocOption] = + import DefaultParsers.* + val value = token("--test_summary=") ~> token(NotSpace.examples("none", "failure", "success")) + value.flatMap: v => + Tests.parseTestSummary(v) match + case Some(ts) => Parser.success(Summary(ts)) + case None => Parser.failure(s"Invalid test_summary value: $v") + end AdhocOption + + private[sbt] def parseTestSummary(value: String): Option[TestSummary] = + value.toLowerCase(Locale.ENGLISH) match + case "0" | "never" | "false" | "none" => Some(TestSummary.none) + case "1" | "always" | "true" => Some(TestSummary.default) + case "failure" => Some(TestSummary.failure) + case "2" | "success" | "verbose" => Some(TestSummary.success) + case _ => None + /** * The result of a test run. * diff --git a/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala b/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala deleted file mode 100644 index 1dd2adbb8..000000000 --- a/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala +++ /dev/null @@ -1,171 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt -package internal -package testing - -import sbt.Incomplete -import sbt.Tests -import sbt.TestResultLogger -import sbt.TestsFailedException -import sbt.protocol.testing.TestResult -import sbt.internal.util.AttributeKey -import sbt.util.Logger - -/** - * Stateless formatter that surfaces every failed test task at the end of an - * aggregated run (see sbt/sbt#2998). The data is read directly off the - * `Incomplete` tree returned by `Aggregation.runTasks`: each subproject's - * `testFull` / `inputTests0` catches the `TestsFailedException` thrown by - * `TestResultLogger.Defaults.Main.run` and re-throws with `(taskName, - * Some(Tests.Output))` attached, and we collect those instances from the - * tree. - * - * The collected `Vector[Failure]` is also stashed on `State.attributes` - * under `recapKey` so in-JVM tools (IDE plugins, BSP servers, scripted - * tests that stay inside one sbt invocation via `Command.process`) can - * inspect the most recent recap without parsing log output. Scripted tests - * crossing a `->` boundary cannot read this because the inner sbt's IPC - * server is torn down on failure and a fresh JVM is spawned for the next - * statement. - * - * Lifecycle is monotonic-latest-failure: `Aggregation.runTasks` writes - * `recapKey` whenever a run produces at least one `TestsFailedException`, - * and never removes it. A successful test run after a failure leaves the - * stale attribute in place; the next failure will overwrite it. We do not - * attempt to recognize "this is a test invocation" at the aggregation - * boundary to avoid a hardcoded list of test-task labels (or a - * Tags-detection design exercise). - */ -private[sbt] object TestRecap: - - /** - * A single failed test task contributing to the recap. - * - * `testOutput` is sanitized by [[collect]]: its `SuiteResult.throwables` are dropped so the - * recap cannot pin the test class loader. See [[collect]] for why. - */ - final case class Failure(taskName: String, testOutput: Option[Tests.Output]) - - /** - * State attribute holding the collected failures from the most recent - * aggregated run that produced at least one `TestsFailedException`. - * Monotonic-latest-failure semantics: never cleared on success, only - * overwritten by the next failure. - * - * Note for consumers: the `SuiteResult.throwables` reachable from these - * failures are always empty -- [[collect]] strips them so the recap cannot - * pin the test class loader. An empty `throwables` here therefore means - * "not retained", NOT "no exception was thrown". The real throwables live on - * the `TestsFailedException` in the `Incomplete` tree, which is where error - * reporting reads them from. - */ - val recapKey: AttributeKey[Vector[Failure]] = AttributeKey[Vector[Failure]]( - "testRecap", - "Failures collected from the most recent aggregated test run" - ) - - /** - * Walk the `Incomplete` tree and return one `Failure` per - * `TestsFailedException`. Exceptions without a payload (e.g., the - * back-compat no-arg constructor escaping a path that didn't get wrapped - * at the task boundary) still contribute a stub entry so the recap lists - * at least the task name when one is available. - * - * Identity-deduplicated via `Incomplete.allExceptions` (which uses an - * `IDSet[Throwable]` internally), so a single failing task shared across - * multiple Incomplete paths in a DAG is counted once. - * - * The retained `Tests.Output` is stripped of `SuiteResult.throwables`: the - * recap only renders names and counts, but a test-thrown Throwable's - * backtrace pins the `Class` objects of every frame, and a `Class` strongly - * references its defining class loader. Since this data is stashed on - * `State.attributes` where it outlives the command, retaining the throwables - * would keep the test class loader -- and its open jar handles -- alive for - * the rest of the session. On Windows those handles make the cached jars - * undeletable (e.g. by `clearCaches`). - */ - def collect(i: Incomplete): Vector[Failure] = - Incomplete - .allExceptions(i) - .iterator - .flatMap { - case e: TestsFailedException => - Some(Failure(e.taskName, e.testOutput.map(dropThrowables))) - case _ => None - } - .toVector - - private def dropThrowables(o: Tests.Output): Tests.Output = - o.copy(events = o.events.view.mapValues(dropThrowables).toMap) - - private def dropThrowables(s: SuiteResult): SuiteResult = - if s.throwables.isEmpty then s - else - new SuiteResult( - s.result, - s.passedCount, - s.failureCount, - s.errorCount, - s.skippedCount, - s.ignoredCount, - s.canceledCount, - s.pendingCount, - ) - - /** - * The rendered recap as a sequence of `\n`-free lines. Failures are - * sorted by `taskName` (lexicographically; empty task names last) for - * stable, diff-friendly output across runs. - */ - def render(failures: Vector[Failure]): Vector[String] = - if failures.isEmpty then Vector.empty - else - val sorted = failures.sortBy(f => (f.taskName.isEmpty, f.taskName)) - val n = sorted.size - val plural = if n == 1 then "" else "s" - val lines = Vector.newBuilder[String] - lines += s"Test failures recap ($n test task$plural failed):" - sorted.foreach { f => - val displayName = if f.taskName.isEmpty then "" else f.taskName - f.testOutput match - case None => - lines += s" $displayName: (no details)" - case Some(out) => - lines += s" $displayName: ${TestResultLogger.Defaults.countsString(out)}" - val failed = collectByResult(out, TestResult.Failed) - val errored = collectByResult(out, TestResult.Error) - if failed.nonEmpty then - lines += " Failed tests:" - failed.foreach(name => lines += s" $name") - if errored.nonEmpty then - lines += " Error during tests:" - errored.foreach(name => lines += s" $name") - } - lines.result() - - /** Render `failures` and emit one error-level log line per rendered line. */ - def formatTo(log: Logger, failures: Vector[Failure]): Unit = - render(failures).foreach(line => log.error(line)) - - private def collectByResult(o: Tests.Output, target: TestResult): Vector[String] = - // Mirrors `TestResultLogger.Defaults.printFailures` so the per-task - // "Failed tests:" block and the cross-project recap render the same - // suite name. Whether `NameTransformer.decode` should be applied to - // suite FQNs at all is debatable, but changing both sites belongs in - // a separate cleanup. - o.events.iterator - .collect { - case (name, suite) if suite.result == target => - scala.reflect.NameTransformer.decode(name) - } - .toVector - .sorted - -end TestRecap diff --git a/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala b/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala new file mode 100644 index 000000000..679e343e4 --- /dev/null +++ b/main-actions/src/test/scala/sbt/TestResultLoggerSummaryTest.scala @@ -0,0 +1,267 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import sbt.TestResultLogger.Defaults.Summary +import sbt.internal.util.Terminal +import sbt.protocol.testing.TestResult +import sbt.util.Logger + +import scala.Console.* +import scala.util.Using + +object TestResultLoggerSummaryTest extends verify.BasicTestSuite: + + private def output(suites: (String, SuiteResult)*): Tests.Output = + Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty) + + private def suite(passed: Int, failed: Int = 0, errors: Int = 0): SuiteResult = + val result = + if errors > 0 then TestResult.Error + else if failed > 0 then TestResult.Failed + else TestResult.Passed + new SuiteResult(result, passed, failed, errors, 0, 0, 0, 0) + + private def entry( + taskName: String, + out: Tests.Output, + cached: Vector[String] = Vector.empty + ): (Tests.Output, String, Vector[String]) = (out, taskName, cached) + + private def render(mode: TestSummary, entries: Vector[(Tests.Output, String, Vector[String])]) = + Summary.render(mode, entries, isColorEnabled = false) + + private class Capture extends Logger with AutoCloseable: + val lines: scala.collection.mutable.ArrayBuffer[(String, String)] = + scala.collection.mutable.ArrayBuffer.empty + override def trace(t: => Throwable): Unit = () + override def success(msg: => String): Unit = () + override def log(level: sbt.util.Level.Value, msg: => String): Unit = + lines += level.toString -> msg + def close(): Unit = () + end Capture + + private def withLog[A1](f: Capture => A1): A1 = + Using.resource(new Capture): log => + f(log) + + test("render is empty when nothing ran and nothing was cached") { + val entries = Vector(entry("a / Test / test", output())) + assert(render(TestSummary.default, entries).isEmpty) + assert(render(TestSummary.Success, entries).isEmpty) + } + + test("Default renders one aggregate line across executed and cached tests") { + val entries = Vector( + entry("a / Test / test", output("A" -> suite(passed = 2))), + entry("b / Test / test", output("B" -> suite(passed = 1)), Vector("B2")), + ) + assert( + render(TestSummary.default, entries) == + Vector("passed: total 3, failed 0, errors 0, passed 3, cached 1") + ) + } + + test("render leads with the aggregate status") { + val entries = Vector(entry("a / Test / test", output("A" -> suite(passed = 1, failed = 1)))) + assert( + render(TestSummary.default, entries).last == + "failed: total 1, failed 1, errors 0, passed 0, cached 0" + ) + val errored = Vector(entry("a / Test / test", output("A" -> suite(passed = 0, errors = 1)))) + assert( + render(TestSummary.default, errored).last == + "error: total 1, failed 0, errors 1, passed 0, cached 0" + ) + } + + test("Default lists only the non-passing task among a mix of pass/fail/cached") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ), + entry("b / Test / test", output("example.Passing" -> suite(passed = 1))), + entry("c / Test / test", output(), Vector("example.Cached")), + ) + assert( + render(TestSummary.default, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + "", + "failed: total 3, failed 1, errors 0, passed 2, cached 1", + ) + ) + } + + test("Success renders the per-suite detail with cached markers") { + val entries = Vector( + entry("core2 / Test / testQuick", output("example.ExampleSuite2" -> suite(passed = 1))), + entry( + "core1 / Test / testQuick", + output("example.ExampleSuite1" -> suite(passed = 1)), + Vector("example.ExampleTest1B"), + ), + entry("core3 / Test / testQuick", output(), Vector("example.ExampleSuite3")), + ) + assert( + render(TestSummary.Success, entries) == + Vector( + "Test summary (3 test tasks succeeded):", + " core1 / Test / testQuick", + " example.ExampleSuite1 PASS", + " example.ExampleTest1B (cached) PASS", + " core2 / Test / testQuick", + " example.ExampleSuite2 PASS", + " core3 / Test / testQuick", + " example.ExampleSuite3 (cached) PASS", + "", + "passed: total 4, failed 0, errors 0, passed 4, cached 2", + ) + ) + } + + test("Success header counts only the failing task even though the body lists every task") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ), + entry("b / Test / test", output("example.Passing" -> suite(passed = 1))), + ) + assert( + render(TestSummary.Success, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + " b / Test / test", + " example.Passing PASS", + "", + "failed: total 2, failed 1, errors 0, passed 1, cached 0", + ) + ) + } + + test("Failure lists only failing suites, collapsing to the aggregate line when all pass") { + val failedSuite = new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0) + val entries = Vector( + entry( + "a / Test / test", + output("example.Passing" -> suite(passed = 1), "example.Failing" -> failedSuite), + Vector("example.Cached"), + ) + ) + assert( + render(TestSummary.Failure, entries) == + Vector( + "Test summary (1 test task failed):", + " a / Test / test", + " example.Failing FAIL", + "", + "failed: total 3, failed 1, errors 0, passed 2, cached 1", + ) + ) + val allPassing = + Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1)))) + assert( + render(TestSummary.Failure, allPassing) == + Vector("passed: total 1, failed 0, errors 0, passed 1, cached 0") + ) + } + + test("isColorEnabled wraps only the status word, not a (cached) prefix") { + val entries = Vector( + entry( + "a / Test / test", + output( + "example.Passing" -> suite(passed = 1), + "example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0), + ), + Vector("example.Cached"), + ) + ) + val lines = Summary.render(TestSummary.Success, entries, isColorEnabled = true) + assert(lines.exists(_.endsWith(s"${GREEN}PASS$RESET"))) + assert(lines.exists(_.endsWith(s"${RED}FAIL$RESET"))) + assert(lines.exists(l => l.contains("(cached) ") && l.endsWith(s"${GREEN}PASS$RESET"))) + assert(!lines.exists(_.contains(s"$GREEN(cached)"))) + } + + test("None suppresses all output, even on failure") { + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)), + ) + ) + assert(render(TestSummary.none, entries).isEmpty) + } + + test("summary logs at info level when everything passes") { + withLog: log => + val entries = Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1)))) + Summary(TestSummary.Success).summary(log, entries) + val expected = Summary + .render(TestSummary.Success, entries, Terminal.get.isColorEnabled) + .map(line => if line.isEmpty then " " else line) + assert(log.lines.map(_._2).toVector == expected) + assert(log.lines.forall(_._1 == "info")) + } + + test("summary logs at error level when entries contain a failure") { + withLog: log => + val entries = Vector( + entry( + "a / Test / test", + output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)) + ) + ) + Summary(TestSummary.default).summary(log, entries) + val expected = Summary + .render(TestSummary.default, entries, Terminal.get.isColorEnabled) + .map(line => if line.isEmpty then " " else line) + assert(log.lines.map(_._2).toVector == expected) + assert(log.lines.nonEmpty) + assert(log.lines.forall(_._1 == "error")) + } + + test("Summary.run delegates to the Default per-task logger"): + withLog: log => + Summary().run(log, output("A" -> suite(passed = 1)), "a / Test / test") + assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0")) + + test("printStandard's 4-arg run folds the cached count into total and passed"): + withLog: log => + TestResultLogger.Default.run( + log, + output("A" -> suite(passed = 1)), + "a / Test / test", + Vector("B", "C"), + ) + assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2")) + + test("printStandard's 3-arg run always shows the cached count, even at zero"): + withLog: log => + TestResultLogger.Default.run(log, output("A" -> suite(passed = 1)), "a / Test / test") + assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0")) + + test("choose (via SilentWhenNoTests) forwards the cached count to the chosen branch"): + withLog: log => + TestResultLogger.SilentWhenNoTests.run( + log, + output("A" -> suite(passed = 1)), + "a / Test / test", + Vector("B", "C"), + ) + assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2")) + +end TestResultLoggerSummaryTest diff --git a/main-actions/src/test/scala/sbt/TestSummaryTest.scala b/main-actions/src/test/scala/sbt/TestSummaryTest.scala new file mode 100644 index 000000000..94dc15f7b --- /dev/null +++ b/main-actions/src/test/scala/sbt/TestSummaryTest.scala @@ -0,0 +1,30 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import sbt.protocol.testing.TestResult + +object TestSummaryTest extends verify.BasicTestSuite: + + private def output(suites: (String, SuiteResult)*): Tests.Output = + Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty) + + test("append strips SuiteResult.throwables so lingering entries cannot pin the classloader") { + val thrown = new AssertionError("boom") + val withThrowables = + new SuiteResult(TestResult.Passed, 1, 0, 0, 0, 0, 0, 0, thrown :: Nil) + TestSummary.clear() + TestSummary.append("a / Test / test", output("A" -> withThrowables), Vector.empty, Vector.empty) + val drained = TestSummary.drain() + assert(drained.size == 1) + assert(drained.head.testOutput.events("A").throwables.isEmpty) + assert(TestSummary.drain().isEmpty) + } + +end TestSummaryTest diff --git a/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala b/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala deleted file mode 100644 index c8be710d6..000000000 --- a/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala +++ /dev/null @@ -1,263 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt -package internal -package testing - -import sbt.Incomplete -import sbt.SuiteResult -import sbt.Tests -import sbt.TestsFailedException -import sbt.protocol.testing.TestResult -import sbt.util.Logger - -object TestRecapTest extends verify.BasicTestSuite: - - private def output(result: TestResult, suites: (String, SuiteResult)*): Tests.Output = - Tests.Output(result, suites.toMap, Iterable.empty) - - private def suite(result: TestResult): SuiteResult = - new SuiteResult(result, 0, 1, 0, 0, 0, 0, 0) - - private def failure( - taskName: String, - result: TestResult, - suiteName: String - ): TestsFailedException = - new TestsFailedException( - taskName, - Some(output(result, suiteName -> suite(result))) - ) - - /** Build an Incomplete tree carrying the given TestsFailedExceptions as direct causes. */ - private def incompleteOf(exceptions: TestsFailedException*): Incomplete = - new Incomplete( - node = None, - causes = exceptions.map(e => new Incomplete(node = None, directCause = Some(e))) - ) - - private class Capture extends Logger: - val lines: scala.collection.mutable.ArrayBuffer[(String, String)] = - scala.collection.mutable.ArrayBuffer.empty - override def trace(t: => Throwable): Unit = () - override def success(msg: => String): Unit = () - override def log(level: sbt.util.Level.Value, msg: => String): Unit = - lines += level.toString -> msg - - test("collect strips SuiteResult.throwables so the recap cannot pin the test classloader") { - // A test-thrown Throwable's backtrace pins the Class objects of every frame, and a Class - // strongly references its defining classloader; the recap is stashed on State.attributes, - // so retaining them would keep the test classloader (and its jar handles) alive. - val thrown = new AssertionError("boom") - val withThrowables = - new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil) - val i = incompleteOf( - new TestsFailedException( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> withThrowables)) - ) - ) - val collected = TestRecap.collect(i) - val retained = collected.head.testOutput.get.events("AFailing") - assert(retained.throwables.isEmpty, "throwables should be dropped from the retained output") - // Everything the recap actually renders must survive. - assert(retained.result == TestResult.Failed) - assert(retained.failureCount == 1) - assert(collected.head.testOutput.get.overall == TestResult.Failed) - } - - test("collect sanitizes a copy and leaves the source exception's throwables intact") { - // The stripping above must not mutate the TestsFailedException in the Incomplete tree: - // that exception is what error reporting and the ClassLoaderLayeringStrategy diagnostic - // in Defaults read, and both need the real throwables. Only the copy parked on - // State.attributes is sanitized. - val thrown = new AssertionError("boom") - val source = new TestsFailedException( - "a / Test / test", - Some( - output( - TestResult.Failed, - "AFailing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil) - ) - ) - ) - val collected = TestRecap.collect(incompleteOf(source)) - assert(collected.head.testOutput.get.events("AFailing").throwables.isEmpty) - val original = source.testOutput.get.events("AFailing").throwables - assert( - original == (thrown :: Nil), - s"collect must not strip the source exception, but its throwables became: $original" - ) - } - - test("collect picks up TestsFailedException payloads from the Incomplete tree") { - val i = incompleteOf( - failure("a / Test / test", TestResult.Failed, "AFailing"), - failure("c / Test / test", TestResult.Error, "CErroring"), - ) - val collected = TestRecap.collect(i) - assert(collected.map(_.taskName).sorted == Vector("a / Test / test", "c / Test / test")) - val resultsBy = collected.flatMap(f => f.testOutput.map(o => f.taskName -> o.overall)).toMap - assert(resultsBy("a / Test / test") == TestResult.Failed) - assert(resultsBy("c / Test / test") == TestResult.Error) - } - - test("collect retains TestsFailedException without payload as a stub entry") { - val noDetail = new TestsFailedException // back-compat no-arg constructor - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(noDetail)), - new Incomplete( - node = None, - directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) - ), - ) - ) - val collected = TestRecap.collect(i) - assert(collected.size == 2, s"expected 2 entries, got $collected") - val stub = collected.find(_.testOutput.isEmpty) - assert(stub.isDefined, "no-detail failure should still produce a Failure entry") - assert(stub.get.taskName == "") - } - - test("collect skips exceptions that aren't TestsFailedException") { - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(new RuntimeException("nope"))), - new Incomplete( - node = None, - directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) - ), - ) - ) - assert(TestRecap.collect(i).map(_.taskName) == Vector("ok / Test / test")) - } - - test("collect retains taskName when a TestsFailedException is rebranded with task name only") { - // Simulates the `testFull` / `inputTests0` catch path: an upstream - // legacy code path threw `new TestsFailedException()` (no detail), the - // task wrapper caught it and re-threw with `(taskName, e.testOutput)` - // to attach the project context. - val tagged = new TestsFailedException("a / Test / test", None) - val i = new Incomplete(node = None, directCause = Some(tagged)) - val collected = TestRecap.collect(i) - assert(collected == Vector(TestRecap.Failure("a / Test / test", None))) - } - - test("collect deduplicates a TestsFailedException shared across Incomplete paths") { - val shared = failure("a / Test / test", TestResult.Failed, "AFail") - val i = new Incomplete( - node = None, - causes = Seq( - new Incomplete(node = None, directCause = Some(shared)), - new Incomplete(node = None, directCause = Some(shared)), - ) - ) - val collected = TestRecap.collect(i) - assert( - collected.size == 1, - s"shared failure should be reported once, got ${collected.size}: $collected" - ) - } - - test("render emits header, per-task counts, and indented suite names") { - val failures = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ), - TestRecap.Failure( - "c / Test / test", - Some(output(TestResult.Error, "CErroring" -> suite(TestResult.Error))) - ), - ) - val lines = TestRecap.render(failures) - assert(lines.headOption.contains("Test failures recap (2 test tasks failed):")) - assert(lines.exists(_.contains("a / Test / test:"))) - assert(lines.exists(_.contains("c / Test / test:"))) - assert(lines.exists(_.contains("AFailing"))) - assert(lines.exists(_.contains("CErroring"))) - assert(lines.contains(" Failed tests:")) - assert(lines.contains(" Error during tests:")) - } - - test("render sorts failures by taskName for stable output (empty names last)") { - val failures = Vector( - TestRecap.Failure("zz / Test / test", None), - TestRecap.Failure("", None), - TestRecap.Failure("aa / Test / test", None), - TestRecap.Failure("mm / Test / test", None), - ) - val lines = TestRecap.render(failures) - val headerLines = lines.filter(_.startsWith(" ")) - val order = headerLines.map(_.trim.takeWhile(_ != ':')) - assert( - order == Vector("aa / Test / test", "mm / Test / test", "zz / Test / test", ""), - s"unexpected order: $order" - ) - } - - test("render emits singular header when exactly one task failed") { - val one = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ) - ) - assert(TestRecap.render(one).head == "Test failures recap (1 test task failed):") - } - - test("render shows '(no details)' for failures without a Tests.Output payload") { - val failures = Vector(TestRecap.Failure("a / Test / test", testOutput = None)) - val lines = TestRecap.render(failures) - assert( - lines.exists(_.contains("a / Test / test: (no details)")), - s"expected '(no details)' entry, got $lines" - ) - } - - test("render shows '' when a failure carries no task name") { - val failures = Vector(TestRecap.Failure(taskName = "", testOutput = None)) - val lines = TestRecap.render(failures) - assert( - lines.exists(_.contains(": (no details)")), - s"expected '' placeholder, got $lines" - ) - } - - test("render is empty when there are no failures") { - assert(TestRecap.render(Vector.empty).isEmpty) - } - - test("formatTo emits one error-level log line per rendered line") { - val failures = Vector( - TestRecap.Failure( - "a / Test / test", - Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) - ) - ) - val log = new Capture - TestRecap.formatTo(log, failures) - val rendered = TestRecap.render(failures) - assert( - log.lines.size == rendered.size, - s"expected ${rendered.size} log calls, got ${log.lines.size}" - ) - assert(log.lines.forall(_._1 == "error"), s"all lines should be error level: ${log.lines}") - } - - test("formatTo is a no-op when there are no failures") { - val log = new Capture - TestRecap.formatTo(log, Vector.empty) - assert(log.lines.isEmpty) - } - -end TestRecapTest diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 116427c4d..dd9f5a930 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1227,7 +1227,9 @@ object Defaults extends BuildCommon with DefExtra { testListeners :== Nil, testOptions :== Nil, testOptionDigests :== Nil, - testResultLogger :== TestResultLogger.Default, + testResultLogger :== TestResultLogger.SilentWhenNoTests, + testSummary :== SysProp.testSummary, + testSummaryLogger := TestResultLogger.Defaults.Summary(testSummary.value), testOnly / testFilter :== (IncrementalTest.selectedFilter), testSelected / testFilter :== (IncrementalTest.selectedFilter), extraTestDigests :== Nil, @@ -1295,9 +1297,10 @@ object Defaults extends BuildCommon with DefExtra { val taskName = Project.showContextKey(state.value).show(resolvedScoped.value) try val output = executeTests.value + TestSummary.append(taskName, output, cached = Vector.empty, adhocOptions = Vector.empty) trl.run(streams.value.log, output, taskName) - // Throw with task name + Output so the cross-project recap - // (TestRecap.collect) can surface them. The throw lives here + // Throw with task name + Output so the aggregation boundary + // (Aggregation.runTasks) can signal the failure. The throw lives here // rather than in TestResultLogger so user-overridden loggers // cannot accidentally suppress the failure signal. output.overall match @@ -1309,7 +1312,7 @@ object Defaults extends BuildCommon with DefExtra { // Tag any no-detail TestsFailedException (legacy executeTests // adapters, third-party Tests.Setup actions, anything constructing // `new TestsFailedException()` via the back-compat ctor) with the - // task name on its way out so the recap doesn't render . + // task name on its way out so error reporting has it to show. case e: TestsFailedException if e.taskName.isEmpty => throw new TestsFailedException(taskName, e.testOutput) finally close(testLoader.value) @@ -1463,8 +1466,9 @@ object Defaults extends BuildCommon with DefExtra { inputTests0.mapReferenced(Def.mapScope((s) => s.rescope(key.key))) private lazy val inputTests0: Initialize[InputTask[TestResult]] = { - val parser = loadForParser(definedTestNames)((s, i) => testOnlyParser(s, i getOrElse Nil)) - ParserGen(parser).flatMapTask { (selected, frameworkOptions) => + val parser = + loadForParser(definedTestNames)((s, i) => testOnlyParserWithOption(s, i getOrElse Nil)) + ParserGen(parser).flatMapTask { (selected, frameworkOptions, adhocOptions) => val s = streams.value val filter = testFilter.value val config = testExecution.value @@ -1505,11 +1509,21 @@ object Defaults extends BuildCommon with DefExtra { ) val taskName = display.show(resolvedScoped.value) val trl = testResultLogger.value + val digests = definedTestDigests.value + val cacheConfig = Def.cacheConfiguration.value (Def .value[Task[Tests.Output]] { output }) .map: out => + val cached = IncrementalTest.cachedTestNames( + digests, + cacheConfig, + out.events.keySet, + selected, + frameworkOptions, + ) + TestSummary.append(taskName, out, cached, adhocOptions.toVector) try - trl.run(s.log, out, taskName) + trl.run(s.log, out, taskName, cached) out.overall match case TestResult.Error | TestResult.Failed => throw new TestsFailedException(taskName, Some(out)) @@ -2629,6 +2643,17 @@ object Defaults extends BuildCommon with DefExtra { Space ~> token(NotSpace.examples(mainClasses.toSet)) ~ spaceDelimited("") } + private def testOnlyParserWithOption + : (State, Seq[String]) => Parser[(Seq[String], Seq[String], Seq[Tests.AdhocOption])] = + (state, tests) => + import DefaultParsers.* + val selectTests = distinctParser(tests.toSet, true) + val frameworkOpts = (token(Space) ~> token("--") ~> spaceDelimited("