diff --git a/main-actions/src/main/scala/sbt/TestResultLogger.scala b/main-actions/src/main/scala/sbt/TestResultLogger.scala index 5ab02d6c0..f344bbe7d 100644 --- a/main-actions/src/main/scala/sbt/TestResultLogger.scala +++ b/main-actions/src/main/scala/sbt/TestResultLogger.scala @@ -106,9 +106,12 @@ object TestResultLogger { else run(printFailures) - results.overall match - case TestResult.Error | TestResult.Failed => throw new TestsFailedException - case TestResult.Empty | TestResult.Passed => () + // Logging only. Failure propagation lives in the task wrapper + // (`Defaults.testFull` / `inputTests0`) so the cross-project recap + // (sbt/sbt#2998) can attach the task name and `Tests.Output` to + // the `TestsFailedException` thrown there. The trait contract is + // "perform logging"; it does not document throwing on failure. + () } } @@ -128,6 +131,20 @@ object TestResultLogger { results.summaries.size > 1 || results.summaries.headOption.forall(_.summaryText.isEmpty) val printStandard = TestResultLogger((log, results, _) => { + val counts = countsString(results) + results.overall match + case TestResult.Empty => () + case TestResult.Error => log.error("Error: " + counts) + case TestResult.Passed => log.info("Passed: " + counts) + case TestResult.Failed => log.error("Failed: " + counts) + }) + + /** + * Renders `Tests.Output`'s aggregate counts as a single line like + * `Total 10, Failed 2, Errors 0, Passed 8`. Shared between `printStandard` + * and the cross-project recap formatter (see `TestRecap`). + */ + private[sbt] def countsString(results: Output): String = { val ( skippedCount, errorsCount, @@ -153,7 +170,6 @@ object TestResultLogger { val totalCount = failuresCount + errorsCount + skippedCount + passedCount val base = s"Total $totalCount, Failed $failuresCount, Errors $errorsCount, Passed $passedCount" - val otherCounts = Seq( "Skipped" -> skippedCount, "Ignored" -> ignoredCount, @@ -161,14 +177,8 @@ object TestResultLogger { "Pending" -> pendingCount ) val extra = otherCounts.withFilter(_._2 > 0).map { (label, count) => s", $label $count" } - - val postfix = base + extra.mkString - results.overall match - case TestResult.Empty => () - case TestResult.Error => log.error("Error: " + postfix) - case TestResult.Passed => log.info("Passed: " + postfix) - case TestResult.Failed => log.error("Failed: " + postfix) - }) + base + extra.mkString + } val printFailures = TestResultLogger((log, results, _) => { def select(resultTpe: TestResult) = results.events collect { diff --git a/main-actions/src/main/scala/sbt/Tests.scala b/main-actions/src/main/scala/sbt/Tests.scala index 5fae9287f..af1f460e6 100644 --- a/main-actions/src/main/scala/sbt/Tests.scala +++ b/main-actions/src/main/scala/sbt/Tests.scala @@ -587,6 +587,13 @@ object Tests { } } -final class TestsFailedException - extends RuntimeException("Tests unsuccessful") - with FeedbackProvidedException +final class TestsFailedException private[sbt] ( + val taskName: String, + val testOutput: Option[Tests.Output] +) extends RuntimeException("Tests unsuccessful") + with FeedbackProvidedException { + // Public no-arg constructor preserved for backward compatibility with + // callers outside sbt. Internal call sites always use the primary + // constructor with a real task name. + def this() = this(taskName = "", testOutput = None) +} diff --git a/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala b/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala new file mode 100644 index 000000000..ee4557d16 --- /dev/null +++ b/main-actions/src/main/scala/sbt/internal/testing/TestRecap.scala @@ -0,0 +1,132 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal +package testing + +import sbt.Incomplete +import sbt.Tests +import sbt.TestResultLogger +import sbt.TestsFailedException +import sbt.protocol.testing.TestResult +import sbt.internal.util.AttributeKey +import sbt.util.Logger + +/** + * Stateless formatter that surfaces every failed test task at the end of an + * aggregated run (see sbt/sbt#2998). The data is read directly off the + * `Incomplete` tree returned by `Aggregation.runTasks`: each subproject's + * `testFull` / `inputTests0` catches the `TestsFailedException` thrown by + * `TestResultLogger.Defaults.Main.run` and re-throws with `(taskName, + * Some(Tests.Output))` attached, and we collect those instances from the + * tree. + * + * The collected `Vector[Failure]` is also stashed on `State.attributes` + * under `recapKey` so in-JVM tools (IDE plugins, BSP servers, scripted + * tests that stay inside one sbt invocation via `Command.process`) can + * inspect the most recent recap without parsing log output. Scripted tests + * crossing a `->` boundary cannot read this because the inner sbt's IPC + * server is torn down on failure and a fresh JVM is spawned for the next + * statement. + * + * Lifecycle is monotonic-latest-failure: `Aggregation.runTasks` writes + * `recapKey` whenever a run produces at least one `TestsFailedException`, + * and never removes it. A successful test run after a failure leaves the + * stale attribute in place; the next failure will overwrite it. We do not + * attempt to recognize "this is a test invocation" at the aggregation + * boundary to avoid a hardcoded list of test-task labels (or a + * Tags-detection design exercise). + */ +private[sbt] object TestRecap: + + /** A single failed test task contributing to the recap. */ + final case class Failure(taskName: String, testOutput: Option[Tests.Output]) + + /** + * State attribute holding the collected failures from the most recent + * aggregated run that produced at least one `TestsFailedException`. + * Monotonic-latest-failure semantics: never cleared on success, only + * overwritten by the next failure. + */ + val recapKey: AttributeKey[Vector[Failure]] = AttributeKey[Vector[Failure]]( + "testRecap", + "Failures collected from the most recent aggregated test run" + ) + + /** + * Walk the `Incomplete` tree and return one `Failure` per + * `TestsFailedException`. Exceptions without a payload (e.g., the + * back-compat no-arg constructor escaping a path that didn't get wrapped + * at the task boundary) still contribute a stub entry so the recap lists + * at least the task name when one is available. + * + * Identity-deduplicated via `Incomplete.allExceptions` (which uses an + * `IDSet[Throwable]` internally), so a single failing task shared across + * multiple Incomplete paths in a DAG is counted once. + */ + def collect(i: Incomplete): Vector[Failure] = + Incomplete + .allExceptions(i) + .iterator + .flatMap { + case e: TestsFailedException => Some(Failure(e.taskName, e.testOutput)) + case _ => None + } + .toVector + + /** + * The rendered recap as a sequence of `\n`-free lines. Failures are + * sorted by `taskName` (lexicographically; empty task names last) for + * stable, diff-friendly output across runs. + */ + def render(failures: Vector[Failure]): Vector[String] = + if failures.isEmpty then Vector.empty + else + val sorted = failures.sortBy(f => (f.taskName.isEmpty, f.taskName)) + val n = sorted.size + val plural = if n == 1 then "" else "s" + val lines = Vector.newBuilder[String] + lines += s"Test failures recap ($n test task$plural failed):" + sorted.foreach { f => + val displayName = if f.taskName.isEmpty then "" else f.taskName + f.testOutput match + case None => + lines += s" $displayName: (no details)" + case Some(out) => + lines += s" $displayName: ${TestResultLogger.Defaults.countsString(out)}" + val failed = collectByResult(out, TestResult.Failed) + val errored = collectByResult(out, TestResult.Error) + if failed.nonEmpty then + lines += " Failed tests:" + failed.foreach(name => lines += s" $name") + if errored.nonEmpty then + lines += " Error during tests:" + errored.foreach(name => lines += s" $name") + } + lines.result() + + /** Render `failures` and emit one error-level log line per rendered line. */ + def formatTo(log: Logger, failures: Vector[Failure]): Unit = + render(failures).foreach(line => log.error(line)) + + private def collectByResult(o: Tests.Output, target: TestResult): Vector[String] = + // Mirrors `TestResultLogger.Defaults.printFailures` so the per-task + // "Failed tests:" block and the cross-project recap render the same + // suite name. Whether `NameTransformer.decode` should be applied to + // suite FQNs at all is debatable, but changing both sites belongs in + // a separate cleanup. + o.events.iterator + .collect { + case (name, suite) if suite.result == target => + scala.reflect.NameTransformer.decode(name) + } + .toVector + .sorted + +end TestRecap diff --git a/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala b/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala new file mode 100644 index 000000000..685112dd9 --- /dev/null +++ b/main-actions/src/test/scala/sbt/internal/testing/TestRecapTest.scala @@ -0,0 +1,217 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal +package testing + +import sbt.Incomplete +import sbt.SuiteResult +import sbt.Tests +import sbt.TestsFailedException +import sbt.protocol.testing.TestResult +import sbt.util.Logger + +object TestRecapTest extends verify.BasicTestSuite: + + private def output(result: TestResult, suites: (String, SuiteResult)*): Tests.Output = + Tests.Output(result, suites.toMap, Iterable.empty) + + private def suite(result: TestResult): SuiteResult = + new SuiteResult(result, 0, 1, 0, 0, 0, 0, 0) + + private def failure( + taskName: String, + result: TestResult, + suiteName: String + ): TestsFailedException = + new TestsFailedException( + taskName, + Some(output(result, suiteName -> suite(result))) + ) + + /** Build an Incomplete tree carrying the given TestsFailedExceptions as direct causes. */ + private def incompleteOf(exceptions: TestsFailedException*): Incomplete = + new Incomplete( + node = None, + causes = exceptions.map(e => new Incomplete(node = None, directCause = Some(e))) + ) + + private class Capture extends Logger: + val lines: scala.collection.mutable.ArrayBuffer[(String, String)] = + scala.collection.mutable.ArrayBuffer.empty + override def trace(t: => Throwable): Unit = () + override def success(msg: => String): Unit = () + override def log(level: sbt.util.Level.Value, msg: => String): Unit = + lines += level.toString -> msg + + test("collect picks up TestsFailedException payloads from the Incomplete tree") { + val i = incompleteOf( + failure("a / Test / test", TestResult.Failed, "AFailing"), + failure("c / Test / test", TestResult.Error, "CErroring"), + ) + val collected = TestRecap.collect(i) + assert(collected.map(_.taskName).sorted == Vector("a / Test / test", "c / Test / test")) + val resultsBy = collected.flatMap(f => f.testOutput.map(o => f.taskName -> o.overall)).toMap + assert(resultsBy("a / Test / test") == TestResult.Failed) + assert(resultsBy("c / Test / test") == TestResult.Error) + } + + test("collect retains TestsFailedException without payload as a stub entry") { + val noDetail = new TestsFailedException // back-compat no-arg constructor + val i = new Incomplete( + node = None, + causes = Seq( + new Incomplete(node = None, directCause = Some(noDetail)), + new Incomplete( + node = None, + directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) + ), + ) + ) + val collected = TestRecap.collect(i) + assert(collected.size == 2, s"expected 2 entries, got $collected") + val stub = collected.find(_.testOutput.isEmpty) + assert(stub.isDefined, "no-detail failure should still produce a Failure entry") + assert(stub.get.taskName == "") + } + + test("collect skips exceptions that aren't TestsFailedException") { + val i = new Incomplete( + node = None, + causes = Seq( + new Incomplete(node = None, directCause = Some(new RuntimeException("nope"))), + new Incomplete( + node = None, + directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail")) + ), + ) + ) + assert(TestRecap.collect(i).map(_.taskName) == Vector("ok / Test / test")) + } + + test("collect retains taskName when a TestsFailedException is rebranded with task name only") { + // Simulates the `testFull` / `inputTests0` catch path: an upstream + // legacy code path threw `new TestsFailedException()` (no detail), the + // task wrapper caught it and re-threw with `(taskName, e.testOutput)` + // to attach the project context. + val tagged = new TestsFailedException("a / Test / test", None) + val i = new Incomplete(node = None, directCause = Some(tagged)) + val collected = TestRecap.collect(i) + assert(collected == Vector(TestRecap.Failure("a / Test / test", None))) + } + + test("collect deduplicates a TestsFailedException shared across Incomplete paths") { + val shared = failure("a / Test / test", TestResult.Failed, "AFail") + val i = new Incomplete( + node = None, + causes = Seq( + new Incomplete(node = None, directCause = Some(shared)), + new Incomplete(node = None, directCause = Some(shared)), + ) + ) + val collected = TestRecap.collect(i) + assert( + collected.size == 1, + s"shared failure should be reported once, got ${collected.size}: $collected" + ) + } + + test("render emits header, per-task counts, and indented suite names") { + val failures = Vector( + TestRecap.Failure( + "a / Test / test", + Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) + ), + TestRecap.Failure( + "c / Test / test", + Some(output(TestResult.Error, "CErroring" -> suite(TestResult.Error))) + ), + ) + val lines = TestRecap.render(failures) + assert(lines.headOption.contains("Test failures recap (2 test tasks failed):")) + assert(lines.exists(_.contains("a / Test / test:"))) + assert(lines.exists(_.contains("c / Test / test:"))) + assert(lines.exists(_.contains("AFailing"))) + assert(lines.exists(_.contains("CErroring"))) + assert(lines.contains(" Failed tests:")) + assert(lines.contains(" Error during tests:")) + } + + test("render sorts failures by taskName for stable output (empty names last)") { + val failures = Vector( + TestRecap.Failure("zz / Test / test", None), + TestRecap.Failure("", None), + TestRecap.Failure("aa / Test / test", None), + TestRecap.Failure("mm / Test / test", None), + ) + val lines = TestRecap.render(failures) + val headerLines = lines.filter(_.startsWith(" ")) + val order = headerLines.map(_.trim.takeWhile(_ != ':')) + assert( + order == Vector("aa / Test / test", "mm / Test / test", "zz / Test / test", ""), + s"unexpected order: $order" + ) + } + + test("render emits singular header when exactly one task failed") { + val one = Vector( + TestRecap.Failure( + "a / Test / test", + Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) + ) + ) + assert(TestRecap.render(one).head == "Test failures recap (1 test task failed):") + } + + test("render shows '(no details)' for failures without a Tests.Output payload") { + val failures = Vector(TestRecap.Failure("a / Test / test", testOutput = None)) + val lines = TestRecap.render(failures) + assert( + lines.exists(_.contains("a / Test / test: (no details)")), + s"expected '(no details)' entry, got $lines" + ) + } + + test("render shows '' when a failure carries no task name") { + val failures = Vector(TestRecap.Failure(taskName = "", testOutput = None)) + val lines = TestRecap.render(failures) + assert( + lines.exists(_.contains(": (no details)")), + s"expected '' placeholder, got $lines" + ) + } + + test("render is empty when there are no failures") { + assert(TestRecap.render(Vector.empty).isEmpty) + } + + test("formatTo emits one error-level log line per rendered line") { + val failures = Vector( + TestRecap.Failure( + "a / Test / test", + Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed))) + ) + ) + val log = new Capture + TestRecap.formatTo(log, failures) + val rendered = TestRecap.render(failures) + assert( + log.lines.size == rendered.size, + s"expected ${rendered.size} log calls, got ${log.lines.size}" + ) + assert(log.lines.forall(_._1 == "error"), s"all lines should be error level: ${log.lines}") + } + + test("formatTo is a no-op when there are no failures") { + val log = new Capture + TestRecap.formatTo(log, Vector.empty) + assert(log.lines.isEmpty) + } + +end TestRecapTest diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f7a4c4f06..8cca4c6bb 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1274,7 +1274,22 @@ object Defaults extends BuildCommon with DefExtra { try val output = executeTests.value trl.run(streams.value.log, output, taskName) + // Throw with task name + Output so the cross-project recap + // (TestRecap.collect) can surface them. The throw lives here + // rather than in TestResultLogger so user-overridden loggers + // cannot accidentally suppress the failure signal. + output.overall match + case TestResult.Error | TestResult.Failed => + throw new TestsFailedException(taskName, Some(output)) + case _ => () output.overall + catch + // Tag any no-detail TestsFailedException (legacy executeTests + // adapters, third-party Tests.Setup actions, anything constructing + // `new TestsFailedException()` via the back-compat ctor) with the + // task name on its way out so the recap doesn't render . + case e: TestsFailedException if e.taskName.isEmpty => + throw new TestsFailedException(taskName, e.testOutput) finally close(testLoader.value) }, testSelected := { @@ -1447,8 +1462,18 @@ object Defaults extends BuildCommon with DefExtra { (Def .value[Task[Tests.Output]] { output }) .map: out => - trl.run(s.log, out, taskName) - out.overall + try + trl.run(s.log, out, taskName) + out.overall match + case TestResult.Error | TestResult.Failed => + throw new TestsFailedException(taskName, Some(out)) + case _ => () + out.overall + catch + // Tag any no-detail TestsFailedException with the task name + // on its way out, mirroring the catch in `testFull`. + case e: TestsFailedException if e.taskName.isEmpty => + throw new TestsFailedException(taskName, e.testOutput) } } diff --git a/main/src/main/scala/sbt/internal/Aggregation.scala b/main/src/main/scala/sbt/internal/Aggregation.scala index 6e1105484..212d98578 100644 --- a/main/src/main/scala/sbt/internal/Aggregation.scala +++ b/main/src/main/scala/sbt/internal/Aggregation.scala @@ -142,7 +142,13 @@ object Aggregation { val complete = timedRun[A1](s, ts, extra) showRun(complete, show) complete.results match - case Result.Inc(i) => complete.state.handleError(i) + case Result.Inc(i) => + val failures = sbt.internal.testing.TestRecap.collect(i) + val afterHandle = complete.state.handleError(i) + if failures.nonEmpty then + sbt.internal.testing.TestRecap.formatTo(afterHandle.log, failures) + afterHandle.put(sbt.internal.testing.TestRecap.recapKey, failures) + else afterHandle case Result.Value(_) => complete.state def printSuccess( diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/a/src/test/java/FailingTestA.java b/sbt-app/src/sbt-test/tests/multi-failure-recap/a/src/test/java/FailingTestA.java new file mode 100644 index 000000000..3a27d7d5d --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/a/src/test/java/FailingTestA.java @@ -0,0 +1,6 @@ +import org.junit.Test; +import static org.junit.Assert.fail; + +public class FailingTestA { + @Test public void failure() { fail("intentional failure A"); } +} diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/b/src/test/java/PassingTestB.java b/sbt-app/src/sbt-test/tests/multi-failure-recap/b/src/test/java/PassingTestB.java new file mode 100644 index 000000000..ddadada62 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/b/src/test/java/PassingTestB.java @@ -0,0 +1,5 @@ +import org.junit.Test; + +public class PassingTestB { + @Test public void success() { /* passes */ } +} diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/build.sbt b/sbt-app/src/sbt-test/tests/multi-failure-recap/build.sbt new file mode 100644 index 000000000..7d3109715 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/build.sbt @@ -0,0 +1,16 @@ +ThisBuild / scalaVersion := "2.13.16" + +def junit = libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test + +lazy val a = project.settings(junit) +lazy val b = project.settings(junit) +lazy val c = project.settings(junit) + +lazy val root = (project in file(".")) + .aggregate(a, b, c) + .settings( + commands ++= Seq( + sbt.multifailurerecap.Checks.verifyRecap, + sbt.multifailurerecap.Checks.verifyNoRecap, + ) + ) diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/c/src/test/java/FailingTestC.java b/sbt-app/src/sbt-test/tests/multi-failure-recap/c/src/test/java/FailingTestC.java new file mode 100644 index 000000000..47162189b --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/c/src/test/java/FailingTestC.java @@ -0,0 +1,6 @@ +import org.junit.Test; +import static org.junit.Assert.fail; + +public class FailingTestC { + @Test public void failure() { fail("intentional failure C"); } +} diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/project/Checks.scala b/sbt-app/src/sbt-test/tests/multi-failure-recap/project/Checks.scala new file mode 100644 index 000000000..cca897add --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/project/Checks.scala @@ -0,0 +1,62 @@ +package sbt +package multifailurerecap + +import sbt.internal.testing.TestRecap + +/** + * Lives in package `sbt` so it can access `TestRecap`, which is `private[sbt]`. + * + * A scripted statement that fails (`-> test`) closes the inner sbt's IPC + * server (see `SbtHandler.onNewSbtInstance`'s catch block), which terminates + * the inner sbt JVM. Scripted then launches a fresh JVM for the next + * statement, so a State attribute set inside a failing statement cannot be + * read by a follow-up `> check`. We avoid that by running `test` from + * inside a Command (here) via `Command.process`, all within one JVM. + * + * `recapKey` is monotonic-latest-failure: never proactively cleared. + * That means across CI scripted shards (which share an inner sbt across + * tests with `reload;initialize` between them) a prior test that left a + * recap entry is visible to this test. Both commands therefore strip + * `recapKey` from the incoming state before doing their own assertions + * and again on the way out, so they are hermetic w.r.t. anything other + * scripted tests in the same shard may have left behind. + */ +object Checks { + + val verifyRecap: Command = Command.command("verifyRecap") { state => + val cleared = state.remove(TestRecap.recapKey) + val afterTest = Command.process("test", cleared) + val recap = afterTest.get(TestRecap.recapKey).getOrElse { + sys.error("TestRecap.recapKey not set on state after aggregated test failure") + } + val names = recap.map(_.taskName).toSet + assert(recap.size == 2, s"expected 2 failures, got ${recap.size}: $names") + assert(names.exists(_.startsWith("a / ")), s"recap missing project a: $names") + assert(names.exists(_.startsWith("c / ")), s"recap missing project c: $names") + assert(!names.exists(_.startsWith("b / ")), s"recap should not list project b: $names") + recap.foreach { f => + assert(f.testOutput.isDefined, s"${f.taskName} has no Tests.Output payload") + val failedSuites = f.testOutput.get.events.values.count: s => + s.result == sbt.protocol.testing.TestResult.Failed + assert(failedSuites >= 1, s"${f.taskName} has no failed suite: ${f.testOutput.get.events}") + } + val lines = TestRecap.render(recap) + assert( + lines.head.startsWith("Test failures recap (2 test tasks failed):"), + s"unexpected header: ${lines.head}" + ) + // Return the original state with recapKey stripped so the next + // scripted statement starts hermetic. + state.remove(TestRecap.recapKey) + } + + val verifyNoRecap: Command = Command.command("verifyNoRecap") { state => + val cleared = state.remove(TestRecap.recapKey) + val afterTest = Command.process("test", cleared) + afterTest.get(TestRecap.recapKey) match { + case None => state.remove(TestRecap.recapKey) + case Some(r) => + sys.error(s"unexpected recap after passing test run: ${r.map(_.taskName)}") + } + } +} diff --git a/sbt-app/src/sbt-test/tests/multi-failure-recap/test b/sbt-app/src/sbt-test/tests/multi-failure-recap/test new file mode 100644 index 000000000..581baec07 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/multi-failure-recap/test @@ -0,0 +1,16 @@ +# Verify the aggregated test failure recap end-to-end. verifyRecap is a +# Command (not a Task) that internally runs `test` via Command.process, +# reads the State attribute set by Aggregation.runTasks, and asserts the +# recap content. The command returns the *original* state (not the +# post-failure state) so verifyRecap itself is not marked failed -- the +# inner `test` failure has already been inspected, and we want this +# statement to succeed when the recap is well-formed. See +# project/Checks.scala for details. +> verifyRecap + +# After removing the failing tests, the aggregated test should succeed +# and the recap state attribute should not be set on this fresh JVM. +$ delete a/src/test/java/FailingTestA.java +$ delete c/src/test/java/FailingTestC.java +> clean +> verifyNoRecap