mirror of https://github.com/sbt/sbt.git
[2.x] feat: Test summary (#9602)
**Problem/Solution** This extends the idea started with TestRecap, and applies it to both test success and failures. 1. Existing TestResultLogger trait is extended to handle the summary rendering. 2. TestSummary enum is added to control the verbosity via commandline option, system property, or a setting. 3. Script test captures the log.
This commit is contained in:
eugene yokota
GitHub
parent
f7f337033d
commit
27c3f035e5
|
|
@ -8,7 +8,8 @@
|
|||
|
||||
package sbt
|
||||
|
||||
import sbt.Tests.{ Output, Summary }
|
||||
import sbt.Tests.Output
|
||||
import sbt.internal.util.Terminal
|
||||
import sbt.protocol.testing.TestResult
|
||||
import sbt.util.{ Level, Logger }
|
||||
|
||||
|
|
@ -20,7 +21,7 @@ import sbt.util.{ Level, Logger }
|
|||
*
|
||||
* @since 0.13.5
|
||||
*/
|
||||
trait TestResultLogger {
|
||||
trait TestResultLogger:
|
||||
|
||||
/**
|
||||
* Perform logging.
|
||||
|
|
@ -31,6 +32,14 @@ trait TestResultLogger {
|
|||
*/
|
||||
def run(log: Logger, results: Output, taskName: String): Unit
|
||||
|
||||
def run(log: Logger, results: Output, taskName: String, cached: Vector[String]): Unit =
|
||||
run(log, results, taskName)
|
||||
|
||||
def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit =
|
||||
entries.foreach { case (results, taskName, _) =>
|
||||
run(log, results, taskName)
|
||||
}
|
||||
|
||||
/** Only allow invocation if certain criteria is met, else use another `TestResultLogger` (defaulting to nothing) . */
|
||||
final def onlyIf(
|
||||
f: (Output, String) => Boolean,
|
||||
|
|
@ -44,9 +53,9 @@ trait TestResultLogger {
|
|||
otherwise: TestResultLogger = TestResultLogger.Null
|
||||
) =
|
||||
TestResultLogger.choose(f, otherwise, this)
|
||||
}
|
||||
end TestResultLogger
|
||||
|
||||
object TestResultLogger {
|
||||
object TestResultLogger:
|
||||
|
||||
/** A `TestResultLogger` that does nothing. */
|
||||
val Null = const(_ => ())
|
||||
|
|
@ -61,6 +70,19 @@ object TestResultLogger {
|
|||
def apply(f: (Logger, Output, String) => Unit): TestResultLogger =
|
||||
(log, results, taskName) => f(log, results, taskName)
|
||||
|
||||
/** Creates a `TestResultLogger` using a given function that also receives the cached suite names. */
|
||||
def apply(f: (Logger, Output, String, Vector[String]) => Unit): TestResultLogger =
|
||||
new TestResultLogger:
|
||||
def run(log: Logger, results: Output, taskName: String): Unit =
|
||||
f(log, results, taskName, Vector.empty)
|
||||
override def run(
|
||||
log: Logger,
|
||||
results: Output,
|
||||
taskName: String,
|
||||
cached: Vector[String]
|
||||
): Unit =
|
||||
f(log, results, taskName, cached)
|
||||
|
||||
/** Creates a `TestResultLogger` that ignores its input and always performs the same logging. */
|
||||
def const(f: Logger => Unit) = apply((l, _, _) => f(l))
|
||||
|
||||
|
|
@ -71,8 +93,8 @@ object TestResultLogger {
|
|||
* @param f The `TestResultLogger` to choose if the predicate fails.
|
||||
*/
|
||||
def choose(cond: (Output, String) => Boolean, t: TestResultLogger, f: TestResultLogger) =
|
||||
TestResultLogger((log, results, taskName) =>
|
||||
(if (cond(results, taskName)) t else f).run(log, results, taskName)
|
||||
TestResultLogger((log, results, taskName, cached) =>
|
||||
(if cond(results, taskName) then t else f).run(log, results, taskName, cached)
|
||||
)
|
||||
|
||||
/** Transforms the input to be completely silent when the subject module doesn't contain any tests. */
|
||||
|
|
@ -82,7 +104,44 @@ object TestResultLogger {
|
|||
printNoTests = Null
|
||||
)
|
||||
|
||||
object Defaults {
|
||||
object Defaults:
|
||||
private val suitePadding = " " * 8
|
||||
|
||||
private[sbt] enum SummaryStatus:
|
||||
case Passed, Failed, Errored
|
||||
|
||||
def label: String = this match
|
||||
case SummaryStatus.Passed => "passed"
|
||||
case SummaryStatus.Failed => "failed"
|
||||
case SummaryStatus.Errored => "error"
|
||||
|
||||
def word: String = this match
|
||||
case SummaryStatus.Passed => "succeeded"
|
||||
case SummaryStatus.Failed => "failed"
|
||||
case SummaryStatus.Errored => "errored"
|
||||
end SummaryStatus
|
||||
|
||||
private[sbt] enum SuiteStatus:
|
||||
case Pass, CachedPass, Fail, Error
|
||||
|
||||
def isPassing: Boolean = this match
|
||||
case SuiteStatus.Pass | SuiteStatus.CachedPass => true
|
||||
case SuiteStatus.Fail | SuiteStatus.Error => false
|
||||
|
||||
def render(isColorEnabled: Boolean): String =
|
||||
val (prefix, word, color) = this match
|
||||
case SuiteStatus.Pass => ("", "PASS", scala.Console.GREEN)
|
||||
case SuiteStatus.CachedPass => ("(cached) ", "PASS", scala.Console.GREEN)
|
||||
case SuiteStatus.Fail => ("", "FAIL", scala.Console.RED)
|
||||
case SuiteStatus.Error => ("", "ERROR", scala.Console.RED)
|
||||
if isColorEnabled then s"$prefix$color$word${scala.Console.RESET}" else s"$prefix$word"
|
||||
end SuiteStatus
|
||||
|
||||
private[sbt] object SuiteStatus:
|
||||
def apply(r: TestResult): SuiteStatus = r match
|
||||
case TestResult.Passed | TestResult.Empty => SuiteStatus.Pass
|
||||
case TestResult.Failed => SuiteStatus.Fail
|
||||
case TestResult.Error => SuiteStatus.Error
|
||||
|
||||
/** sbt's default `TestResultLogger`. Use `copy()` to change selective portions. */
|
||||
case class Main(
|
||||
|
|
@ -93,8 +152,16 @@ object TestResultLogger {
|
|||
printNoTests: TestResultLogger = Defaults.printNoTests
|
||||
) extends TestResultLogger {
|
||||
|
||||
override def run(log: Logger, results: Output, taskName: String): Unit = {
|
||||
def run(r: TestResultLogger): Unit = r.run(log, results, taskName)
|
||||
override def run(log: Logger, results: Output, taskName: String): Unit =
|
||||
run(log, results, taskName, Vector.empty)
|
||||
|
||||
override def run(
|
||||
log: Logger,
|
||||
results: Output,
|
||||
taskName: String,
|
||||
cached: Vector[String]
|
||||
): Unit = {
|
||||
def run(r: TestResultLogger): Unit = r.run(log, results, taskName, cached)
|
||||
|
||||
run(printSummary)
|
||||
|
||||
|
|
@ -117,7 +184,7 @@ object TestResultLogger {
|
|||
|
||||
val printSummary = TestResultLogger((log, results, _) => {
|
||||
val multipleFrameworks = results.summaries.size > 1
|
||||
for (Summary(name, message) <- results.summaries)
|
||||
for Tests.Summary(name, message) <- results.summaries do
|
||||
if (message.isEmpty) log.debug("Summary for " + name + " not available.")
|
||||
else {
|
||||
if (multipleFrameworks) log.info(name)
|
||||
|
|
@ -130,54 +197,40 @@ object TestResultLogger {
|
|||
// Print the standard one-liner statistic if no framework summary is defined, or when > 1 framework is in used.
|
||||
results.summaries.size > 1 || results.summaries.headOption.forall(_.summaryText.isEmpty)
|
||||
|
||||
val printStandard = TestResultLogger((log, results, _) => {
|
||||
val counts = countsString(results)
|
||||
val printStandard = TestResultLogger((log, results, _, cached) => {
|
||||
val counts = countsString(results.events.values, cached.size, true)
|
||||
results.overall match
|
||||
case TestResult.Empty => ()
|
||||
case TestResult.Error => log.error("Error: " + counts)
|
||||
case TestResult.Passed => log.info("Passed: " + counts)
|
||||
case TestResult.Failed => log.error("Failed: " + counts)
|
||||
case TestResult.Error => log.error(s"${SummaryStatus.Errored.label}: $counts")
|
||||
case TestResult.Passed => log.info(s"${SummaryStatus.Passed.label}: $counts")
|
||||
case TestResult.Failed => log.error(s"${SummaryStatus.Failed.label}: $counts")
|
||||
})
|
||||
|
||||
private[sbt] def countsString(events: Iterable[SuiteResult]): String =
|
||||
countsString(events, 0, false)
|
||||
|
||||
/**
|
||||
* Renders `Tests.Output`'s aggregate counts as a single line like
|
||||
* `Total 10, Failed 2, Errors 0, Passed 8`. Shared between `printStandard`
|
||||
* and the cross-project recap formatter (see `TestRecap`).
|
||||
* Renders suite counts as a single line like `total 10, failed 2, errors
|
||||
* 0, passed 8`. Counts suites (classes/objects), not individual test examples.
|
||||
*/
|
||||
private[sbt] def countsString(results: Output): String = {
|
||||
val (
|
||||
skippedCount,
|
||||
errorsCount,
|
||||
passedCount,
|
||||
failuresCount,
|
||||
ignoredCount,
|
||||
canceledCount,
|
||||
pendingCount,
|
||||
) =
|
||||
results.events.foldLeft((0, 0, 0, 0, 0, 0, 0)) { case (acc, (_, testEvent)) =>
|
||||
val (skippedAcc, errorAcc, passedAcc, failureAcc, ignoredAcc, canceledAcc, pendingAcc) =
|
||||
acc
|
||||
(
|
||||
skippedAcc + testEvent.skippedCount,
|
||||
errorAcc + testEvent.errorCount,
|
||||
passedAcc + testEvent.passedCount,
|
||||
failureAcc + testEvent.failureCount,
|
||||
ignoredAcc + testEvent.ignoredCount,
|
||||
canceledAcc + testEvent.canceledCount,
|
||||
pendingAcc + testEvent.pendingCount,
|
||||
)
|
||||
private[sbt] def countsString(
|
||||
events: Iterable[SuiteResult],
|
||||
cachedCount: Int,
|
||||
alwaysShowCached: Boolean,
|
||||
): String = {
|
||||
val (failuresCount, errorsCount, passedCount) =
|
||||
events.foldLeft((0, 0, 0)) { case ((failureAcc, errorAcc, passedAcc), suite) =>
|
||||
suite.result match
|
||||
case TestResult.Failed => (failureAcc + 1, errorAcc, passedAcc)
|
||||
case TestResult.Error => (failureAcc, errorAcc + 1, passedAcc)
|
||||
case TestResult.Passed | TestResult.Empty => (failureAcc, errorAcc, passedAcc + 1)
|
||||
}
|
||||
val totalCount = failuresCount + errorsCount + skippedCount + passedCount
|
||||
val totalCount = failuresCount + errorsCount + passedCount + cachedCount
|
||||
val base =
|
||||
s"Total $totalCount, Failed $failuresCount, Errors $errorsCount, Passed $passedCount"
|
||||
val otherCounts = Seq(
|
||||
"Skipped" -> skippedCount,
|
||||
"Ignored" -> ignoredCount,
|
||||
"Canceled" -> canceledCount,
|
||||
"Pending" -> pendingCount
|
||||
)
|
||||
val extra = otherCounts.withFilter(_._2 > 0).map { (label, count) => s", $label $count" }
|
||||
base + extra.mkString
|
||||
s"total $totalCount, failed $failuresCount, errors $errorsCount, passed ${passedCount + cachedCount}"
|
||||
val cachedField =
|
||||
if cachedCount > 0 || alwaysShowCached then s", cached $cachedCount" else ""
|
||||
base + cachedField
|
||||
}
|
||||
|
||||
val printFailures = TestResultLogger((log, results, _) => {
|
||||
|
|
@ -189,15 +242,109 @@ object TestResultLogger {
|
|||
def show(label: String, level: Level.Value, tests: Iterable[String]): Unit =
|
||||
if (tests.nonEmpty) {
|
||||
log.log(level, label)
|
||||
log.log(level, tests.mkString("\t", "\n\t", ""))
|
||||
log.log(level, tests.mkString(suitePadding, s"\n$suitePadding", ""))
|
||||
}
|
||||
|
||||
show("Passed tests:", Level.Debug, select(TestResult.Passed))
|
||||
show("Failed tests:", Level.Error, select(TestResult.Failed))
|
||||
show("Error during tests:", Level.Error, select(TestResult.Error))
|
||||
show("passed tests:", Level.Debug, select(TestResult.Passed))
|
||||
show("failed tests:", Level.Error, select(TestResult.Failed))
|
||||
show("error during tests:", Level.Error, select(TestResult.Error))
|
||||
})
|
||||
|
||||
val printNoTests =
|
||||
TestResultLogger((log, results, taskName) => log.info("No tests to run for " + taskName))
|
||||
}
|
||||
}
|
||||
val printNoTests = TestResultLogger((log, results, taskName, cached) =>
|
||||
val suffix = if cached.nonEmpty then s" (${cached.size} cached)" else ""
|
||||
log.debug(s"no tests to run for $taskName$suffix")
|
||||
)
|
||||
|
||||
/**
|
||||
* Renders a cross-project aggregate summary at the end of an aggregated
|
||||
* run, in the style selected by `mode` (see `TestSummary`). Per-task
|
||||
* logging is unchanged (delegates to `Default`); `summary` is where this
|
||||
* differs from a plain `TestResultLogger`.
|
||||
*/
|
||||
case class Summary(mode: TestSummary) extends TestResultLogger:
|
||||
override def run(log: Logger, results: Output, taskName: String): Unit =
|
||||
Default.run(log, results, taskName)
|
||||
|
||||
override def summary(log: Logger, entries: Vector[(Output, String, Vector[String])]): Unit =
|
||||
val lines = Summary.render(mode, entries, Terminal.get.isColorEnabled)
|
||||
val logLine: String => Unit = Summary.overallStatus(entries) match
|
||||
case Some(SummaryStatus.Errored) | Some(SummaryStatus.Failed) => log.error(_)
|
||||
case _ => log.info(_)
|
||||
lines.foreach(line => logLine(if line.isEmpty then " " else line))
|
||||
end Summary
|
||||
|
||||
object Summary:
|
||||
private val columnGap = 5
|
||||
|
||||
def apply(): Summary = Summary(mode = TestSummary.default)
|
||||
|
||||
/**
|
||||
* The rendered summary; empty when nothing ran. Both styles end with an
|
||||
* aggregate line, e.g. `passed: total 4, failed 0, errors 0, passed 4,
|
||||
* cached 2`. Cached classes count into `total` and `passed`, each as one
|
||||
* (their case counts are unknown without running them).
|
||||
*/
|
||||
private[sbt] def render(
|
||||
mode: TestSummary,
|
||||
entries: Vector[(Output, String, Vector[String])],
|
||||
isColorEnabled: Boolean,
|
||||
): Vector[String] =
|
||||
overallStatus(entries) match
|
||||
case None => Vector.empty
|
||||
case Some(status) =>
|
||||
val executed = entries.flatMap(_._1.events.values)
|
||||
val cached = entries.map(_._3.size).sum
|
||||
val counts =
|
||||
s"${status.label}: ${countsString(executed, cached, alwaysShowCached = true)}"
|
||||
def withCounts(detailLines: Vector[String]): Vector[String] =
|
||||
if detailLines.isEmpty then Vector(counts) else detailLines :+ "" :+ counts
|
||||
mode match
|
||||
case TestSummary.None => Vector.empty
|
||||
case TestSummary.Failure =>
|
||||
withCounts(detail(status, entries, failuresOnly = true, isColorEnabled))
|
||||
case TestSummary.Success =>
|
||||
withCounts(detail(status, entries, failuresOnly = false, isColorEnabled))
|
||||
|
||||
/** The overall status across `entries`, or `None` when nothing ran. */
|
||||
private def overallStatus(
|
||||
entries: Vector[(Output, String, Vector[String])]
|
||||
): Option[SummaryStatus] =
|
||||
val executed = entries.flatMap(_._1.events.values)
|
||||
val cached = entries.map(_._3.size).sum
|
||||
if executed.isEmpty && cached == 0 then None
|
||||
else if executed.exists(_.result == TestResult.Error) then Some(SummaryStatus.Errored)
|
||||
else if executed.exists(_.result == TestResult.Failed) then Some(SummaryStatus.Failed)
|
||||
else Some(SummaryStatus.Passed)
|
||||
|
||||
private def detail(
|
||||
status: SummaryStatus,
|
||||
entries: Vector[(Output, String, Vector[String])],
|
||||
failuresOnly: Boolean,
|
||||
isColorEnabled: Boolean,
|
||||
): Vector[String] =
|
||||
val tasks = entries
|
||||
.map: (output, taskName, cachedNames) =>
|
||||
val executed = output.events.view.mapValues(s => SuiteStatus(s.result)).toVector
|
||||
val suites =
|
||||
if failuresOnly then executed.filter { case (_, st) => !st.isPassing }
|
||||
else executed ++ cachedNames.map(_ -> SuiteStatus.CachedPass)
|
||||
taskName -> suites.sortBy(_._1)
|
||||
.filter(_._2.nonEmpty)
|
||||
.sortBy(_._1)
|
||||
if tasks.isEmpty then Vector.empty
|
||||
else
|
||||
val headerCount =
|
||||
if status == SummaryStatus.Passed then tasks.size
|
||||
else entries.count(_._1.events.values.exists(s => !SuiteStatus(s.result).isPassing))
|
||||
val plural = if headerCount == 1 then "" else "s"
|
||||
val width = tasks.flatMap(_._2.map(_._1.length)).max + columnGap
|
||||
val lines = Vector.newBuilder[String]
|
||||
lines += s"Test summary ($headerCount test task$plural ${status.word}):"
|
||||
tasks.foreach: (taskName, suites) =>
|
||||
lines += s" $taskName"
|
||||
suites.foreach: (name, st) =>
|
||||
lines += s"${suitePadding}${name.padTo(width, ' ')}${st.render(isColorEnabled)}"
|
||||
lines.result()
|
||||
end Summary
|
||||
end Defaults
|
||||
end TestResultLogger
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
import sbt.internal.util.AttributeKey
|
||||
|
||||
/**
|
||||
* Style of the test summary displayed after an aggregated test run:
|
||||
* - `None`: prints nothing
|
||||
* - `Failure` (the default): lists only non-passing suites, staying terse when everything passes
|
||||
* - `Success`: always lists every suite, marking cache reuse with `(cached)`
|
||||
*
|
||||
* Rendering itself lives in `TestResultLogger.Defaults.Summary`, which
|
||||
* consumes the entries collected here.
|
||||
*/
|
||||
enum TestSummary:
|
||||
case None
|
||||
case Failure
|
||||
case Success
|
||||
|
||||
/**
|
||||
* Collector surfacing a test summary at the end of an aggregated run,
|
||||
* success or failure alike.
|
||||
*/
|
||||
object TestSummary:
|
||||
val none: TestSummary = TestSummary.None
|
||||
val failure: TestSummary = TestSummary.Failure
|
||||
val success: TestSummary = TestSummary.Success
|
||||
def default: TestSummary = TestSummary.failure
|
||||
|
||||
/** `cached` names the test classes reused from the action cache (see `IncrementalTest.cachedTestNames`). */
|
||||
private[sbt] final case class Entry(
|
||||
taskName: String,
|
||||
testOutput: Tests.Output,
|
||||
cached: Vector[String],
|
||||
options: Vector[Tests.AdhocOption]
|
||||
)
|
||||
|
||||
/**
|
||||
* State attribute holding the entries from the most recent aggregated run
|
||||
* that produced at least one test result. Monotonic-latest-run semantics:
|
||||
* never cleared, only overwritten by the next non-empty run; lets in-JVM
|
||||
* tools (IDE plugins, BSP servers, scripted tests staying inside one sbt
|
||||
* invocation via `Command.process`) inspect the last test results without
|
||||
* parsing log output. Scripted tests crossing a `->` boundary cannot read
|
||||
* this because the inner sbt's IPC server is torn down on failure and a
|
||||
* fresh JVM is spawned for the next statement.
|
||||
*/
|
||||
private[sbt] val entriesKey: AttributeKey[Vector[Entry]] = AttributeKey[Vector[Entry]](
|
||||
"testSummaryEntries",
|
||||
"Entries collected from the most recent aggregated test run"
|
||||
)
|
||||
|
||||
private val entries = new ConcurrentLinkedQueue[Entry]
|
||||
|
||||
/**
|
||||
* `output`'s `SuiteResult.throwables` are dropped before retention: a
|
||||
* test-thrown Throwable's backtrace pins the `Class` objects of every
|
||||
* frame, and a `Class` strongly references its defining class loader.
|
||||
* Since entries are stashed on `State.attributes` where they outlive the
|
||||
* command, retaining the throwables would keep the test class loader --
|
||||
* and its open jar handles -- alive for the rest of the session. On
|
||||
* Windows those handles make the cached jars undeletable (e.g. by
|
||||
* `clearCaches`).
|
||||
*/
|
||||
private[sbt] def append(
|
||||
taskName: String,
|
||||
output: Tests.Output,
|
||||
cached: Vector[String],
|
||||
adhocOptions: Vector[Tests.AdhocOption]
|
||||
): Unit =
|
||||
entries.add(Entry(taskName, dropThrowables(output), cached, adhocOptions))
|
||||
()
|
||||
|
||||
private def dropThrowables(o: Tests.Output): Tests.Output =
|
||||
o.copy(events = o.events.view.mapValues(dropThrowables).toMap)
|
||||
|
||||
private def dropThrowables(s: SuiteResult): SuiteResult =
|
||||
if s.throwables.isEmpty then s
|
||||
else
|
||||
new SuiteResult(
|
||||
s.result,
|
||||
s.passedCount,
|
||||
s.failureCount,
|
||||
s.errorCount,
|
||||
s.skippedCount,
|
||||
s.ignoredCount,
|
||||
s.canceledCount,
|
||||
s.pendingCount,
|
||||
)
|
||||
|
||||
private[sbt] def clear(): Unit = entries.clear()
|
||||
|
||||
private[sbt] def drain(): Vector[Entry] =
|
||||
val b = Vector.newBuilder[Entry]
|
||||
var e = entries.poll()
|
||||
while e != null do
|
||||
b += e
|
||||
e = entries.poll()
|
||||
b.result()
|
||||
|
||||
end TestSummary
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
package sbt
|
||||
|
||||
import java.util.Locale
|
||||
import std.*
|
||||
import xsbt.api.{ Discovered, Discovery }
|
||||
import sbt.internal.inc.Analysis
|
||||
|
|
@ -38,11 +39,33 @@ import sbt.util.{ Digest, Logger }
|
|||
import sbt.protocol.testing.TestResult
|
||||
|
||||
import scala.runtime.AbstractFunction3
|
||||
import sbt.internal.util.complete.{ DefaultParsers, Parser }
|
||||
|
||||
sealed trait TestOption
|
||||
|
||||
object Tests {
|
||||
|
||||
private[sbt] sealed trait AdhocOption
|
||||
private[sbt] object AdhocOption:
|
||||
case class Summary(summary: TestSummary) extends AdhocOption
|
||||
|
||||
private[sbt] def parser: Parser[AdhocOption] =
|
||||
import DefaultParsers.*
|
||||
val value = token("--test_summary=") ~> token(NotSpace.examples("none", "failure", "success"))
|
||||
value.flatMap: v =>
|
||||
Tests.parseTestSummary(v) match
|
||||
case Some(ts) => Parser.success(Summary(ts))
|
||||
case None => Parser.failure(s"Invalid test_summary value: $v")
|
||||
end AdhocOption
|
||||
|
||||
private[sbt] def parseTestSummary(value: String): Option[TestSummary] =
|
||||
value.toLowerCase(Locale.ENGLISH) match
|
||||
case "0" | "never" | "false" | "none" => Some(TestSummary.none)
|
||||
case "1" | "always" | "true" => Some(TestSummary.default)
|
||||
case "failure" => Some(TestSummary.failure)
|
||||
case "2" | "success" | "verbose" => Some(TestSummary.success)
|
||||
case _ => None
|
||||
|
||||
/**
|
||||
* The result of a test run.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,171 +0,0 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
package internal
|
||||
package testing
|
||||
|
||||
import sbt.Incomplete
|
||||
import sbt.Tests
|
||||
import sbt.TestResultLogger
|
||||
import sbt.TestsFailedException
|
||||
import sbt.protocol.testing.TestResult
|
||||
import sbt.internal.util.AttributeKey
|
||||
import sbt.util.Logger
|
||||
|
||||
/**
|
||||
* Stateless formatter that surfaces every failed test task at the end of an
|
||||
* aggregated run (see sbt/sbt#2998). The data is read directly off the
|
||||
* `Incomplete` tree returned by `Aggregation.runTasks`: each subproject's
|
||||
* `testFull` / `inputTests0` catches the `TestsFailedException` thrown by
|
||||
* `TestResultLogger.Defaults.Main.run` and re-throws with `(taskName,
|
||||
* Some(Tests.Output))` attached, and we collect those instances from the
|
||||
* tree.
|
||||
*
|
||||
* The collected `Vector[Failure]` is also stashed on `State.attributes`
|
||||
* under `recapKey` so in-JVM tools (IDE plugins, BSP servers, scripted
|
||||
* tests that stay inside one sbt invocation via `Command.process`) can
|
||||
* inspect the most recent recap without parsing log output. Scripted tests
|
||||
* crossing a `->` boundary cannot read this because the inner sbt's IPC
|
||||
* server is torn down on failure and a fresh JVM is spawned for the next
|
||||
* statement.
|
||||
*
|
||||
* Lifecycle is monotonic-latest-failure: `Aggregation.runTasks` writes
|
||||
* `recapKey` whenever a run produces at least one `TestsFailedException`,
|
||||
* and never removes it. A successful test run after a failure leaves the
|
||||
* stale attribute in place; the next failure will overwrite it. We do not
|
||||
* attempt to recognize "this is a test invocation" at the aggregation
|
||||
* boundary to avoid a hardcoded list of test-task labels (or a
|
||||
* Tags-detection design exercise).
|
||||
*/
|
||||
private[sbt] object TestRecap:
|
||||
|
||||
/**
|
||||
* A single failed test task contributing to the recap.
|
||||
*
|
||||
* `testOutput` is sanitized by [[collect]]: its `SuiteResult.throwables` are dropped so the
|
||||
* recap cannot pin the test class loader. See [[collect]] for why.
|
||||
*/
|
||||
final case class Failure(taskName: String, testOutput: Option[Tests.Output])
|
||||
|
||||
/**
|
||||
* State attribute holding the collected failures from the most recent
|
||||
* aggregated run that produced at least one `TestsFailedException`.
|
||||
* Monotonic-latest-failure semantics: never cleared on success, only
|
||||
* overwritten by the next failure.
|
||||
*
|
||||
* Note for consumers: the `SuiteResult.throwables` reachable from these
|
||||
* failures are always empty -- [[collect]] strips them so the recap cannot
|
||||
* pin the test class loader. An empty `throwables` here therefore means
|
||||
* "not retained", NOT "no exception was thrown". The real throwables live on
|
||||
* the `TestsFailedException` in the `Incomplete` tree, which is where error
|
||||
* reporting reads them from.
|
||||
*/
|
||||
val recapKey: AttributeKey[Vector[Failure]] = AttributeKey[Vector[Failure]](
|
||||
"testRecap",
|
||||
"Failures collected from the most recent aggregated test run"
|
||||
)
|
||||
|
||||
/**
|
||||
* Walk the `Incomplete` tree and return one `Failure` per
|
||||
* `TestsFailedException`. Exceptions without a payload (e.g., the
|
||||
* back-compat no-arg constructor escaping a path that didn't get wrapped
|
||||
* at the task boundary) still contribute a stub entry so the recap lists
|
||||
* at least the task name when one is available.
|
||||
*
|
||||
* Identity-deduplicated via `Incomplete.allExceptions` (which uses an
|
||||
* `IDSet[Throwable]` internally), so a single failing task shared across
|
||||
* multiple Incomplete paths in a DAG is counted once.
|
||||
*
|
||||
* The retained `Tests.Output` is stripped of `SuiteResult.throwables`: the
|
||||
* recap only renders names and counts, but a test-thrown Throwable's
|
||||
* backtrace pins the `Class` objects of every frame, and a `Class` strongly
|
||||
* references its defining class loader. Since this data is stashed on
|
||||
* `State.attributes` where it outlives the command, retaining the throwables
|
||||
* would keep the test class loader -- and its open jar handles -- alive for
|
||||
* the rest of the session. On Windows those handles make the cached jars
|
||||
* undeletable (e.g. by `clearCaches`).
|
||||
*/
|
||||
def collect(i: Incomplete): Vector[Failure] =
|
||||
Incomplete
|
||||
.allExceptions(i)
|
||||
.iterator
|
||||
.flatMap {
|
||||
case e: TestsFailedException =>
|
||||
Some(Failure(e.taskName, e.testOutput.map(dropThrowables)))
|
||||
case _ => None
|
||||
}
|
||||
.toVector
|
||||
|
||||
private def dropThrowables(o: Tests.Output): Tests.Output =
|
||||
o.copy(events = o.events.view.mapValues(dropThrowables).toMap)
|
||||
|
||||
private def dropThrowables(s: SuiteResult): SuiteResult =
|
||||
if s.throwables.isEmpty then s
|
||||
else
|
||||
new SuiteResult(
|
||||
s.result,
|
||||
s.passedCount,
|
||||
s.failureCount,
|
||||
s.errorCount,
|
||||
s.skippedCount,
|
||||
s.ignoredCount,
|
||||
s.canceledCount,
|
||||
s.pendingCount,
|
||||
)
|
||||
|
||||
/**
|
||||
* The rendered recap as a sequence of `\n`-free lines. Failures are
|
||||
* sorted by `taskName` (lexicographically; empty task names last) for
|
||||
* stable, diff-friendly output across runs.
|
||||
*/
|
||||
def render(failures: Vector[Failure]): Vector[String] =
|
||||
if failures.isEmpty then Vector.empty
|
||||
else
|
||||
val sorted = failures.sortBy(f => (f.taskName.isEmpty, f.taskName))
|
||||
val n = sorted.size
|
||||
val plural = if n == 1 then "" else "s"
|
||||
val lines = Vector.newBuilder[String]
|
||||
lines += s"Test failures recap ($n test task$plural failed):"
|
||||
sorted.foreach { f =>
|
||||
val displayName = if f.taskName.isEmpty then "<unknown>" else f.taskName
|
||||
f.testOutput match
|
||||
case None =>
|
||||
lines += s" $displayName: (no details)"
|
||||
case Some(out) =>
|
||||
lines += s" $displayName: ${TestResultLogger.Defaults.countsString(out)}"
|
||||
val failed = collectByResult(out, TestResult.Failed)
|
||||
val errored = collectByResult(out, TestResult.Error)
|
||||
if failed.nonEmpty then
|
||||
lines += " Failed tests:"
|
||||
failed.foreach(name => lines += s" $name")
|
||||
if errored.nonEmpty then
|
||||
lines += " Error during tests:"
|
||||
errored.foreach(name => lines += s" $name")
|
||||
}
|
||||
lines.result()
|
||||
|
||||
/** Render `failures` and emit one error-level log line per rendered line. */
|
||||
def formatTo(log: Logger, failures: Vector[Failure]): Unit =
|
||||
render(failures).foreach(line => log.error(line))
|
||||
|
||||
private def collectByResult(o: Tests.Output, target: TestResult): Vector[String] =
|
||||
// Mirrors `TestResultLogger.Defaults.printFailures` so the per-task
|
||||
// "Failed tests:" block and the cross-project recap render the same
|
||||
// suite name. Whether `NameTransformer.decode` should be applied to
|
||||
// suite FQNs at all is debatable, but changing both sites belongs in
|
||||
// a separate cleanup.
|
||||
o.events.iterator
|
||||
.collect {
|
||||
case (name, suite) if suite.result == target =>
|
||||
scala.reflect.NameTransformer.decode(name)
|
||||
}
|
||||
.toVector
|
||||
.sorted
|
||||
|
||||
end TestRecap
|
||||
|
|
@ -0,0 +1,267 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
|
||||
import sbt.TestResultLogger.Defaults.Summary
|
||||
import sbt.internal.util.Terminal
|
||||
import sbt.protocol.testing.TestResult
|
||||
import sbt.util.Logger
|
||||
|
||||
import scala.Console.*
|
||||
import scala.util.Using
|
||||
|
||||
object TestResultLoggerSummaryTest extends verify.BasicTestSuite:
|
||||
|
||||
private def output(suites: (String, SuiteResult)*): Tests.Output =
|
||||
Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty)
|
||||
|
||||
private def suite(passed: Int, failed: Int = 0, errors: Int = 0): SuiteResult =
|
||||
val result =
|
||||
if errors > 0 then TestResult.Error
|
||||
else if failed > 0 then TestResult.Failed
|
||||
else TestResult.Passed
|
||||
new SuiteResult(result, passed, failed, errors, 0, 0, 0, 0)
|
||||
|
||||
private def entry(
|
||||
taskName: String,
|
||||
out: Tests.Output,
|
||||
cached: Vector[String] = Vector.empty
|
||||
): (Tests.Output, String, Vector[String]) = (out, taskName, cached)
|
||||
|
||||
private def render(mode: TestSummary, entries: Vector[(Tests.Output, String, Vector[String])]) =
|
||||
Summary.render(mode, entries, isColorEnabled = false)
|
||||
|
||||
private class Capture extends Logger with AutoCloseable:
|
||||
val lines: scala.collection.mutable.ArrayBuffer[(String, String)] =
|
||||
scala.collection.mutable.ArrayBuffer.empty
|
||||
override def trace(t: => Throwable): Unit = ()
|
||||
override def success(msg: => String): Unit = ()
|
||||
override def log(level: sbt.util.Level.Value, msg: => String): Unit =
|
||||
lines += level.toString -> msg
|
||||
def close(): Unit = ()
|
||||
end Capture
|
||||
|
||||
private def withLog[A1](f: Capture => A1): A1 =
|
||||
Using.resource(new Capture): log =>
|
||||
f(log)
|
||||
|
||||
test("render is empty when nothing ran and nothing was cached") {
|
||||
val entries = Vector(entry("a / Test / test", output()))
|
||||
assert(render(TestSummary.default, entries).isEmpty)
|
||||
assert(render(TestSummary.Success, entries).isEmpty)
|
||||
}
|
||||
|
||||
test("Default renders one aggregate line across executed and cached tests") {
|
||||
val entries = Vector(
|
||||
entry("a / Test / test", output("A" -> suite(passed = 2))),
|
||||
entry("b / Test / test", output("B" -> suite(passed = 1)), Vector("B2")),
|
||||
)
|
||||
assert(
|
||||
render(TestSummary.default, entries) ==
|
||||
Vector("passed: total 3, failed 0, errors 0, passed 3, cached 1")
|
||||
)
|
||||
}
|
||||
|
||||
test("render leads with the aggregate status") {
|
||||
val entries = Vector(entry("a / Test / test", output("A" -> suite(passed = 1, failed = 1))))
|
||||
assert(
|
||||
render(TestSummary.default, entries).last ==
|
||||
"failed: total 1, failed 1, errors 0, passed 0, cached 0"
|
||||
)
|
||||
val errored = Vector(entry("a / Test / test", output("A" -> suite(passed = 0, errors = 1))))
|
||||
assert(
|
||||
render(TestSummary.default, errored).last ==
|
||||
"error: total 1, failed 0, errors 1, passed 0, cached 0"
|
||||
)
|
||||
}
|
||||
|
||||
test("Default lists only the non-passing task among a mix of pass/fail/cached") {
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)),
|
||||
),
|
||||
entry("b / Test / test", output("example.Passing" -> suite(passed = 1))),
|
||||
entry("c / Test / test", output(), Vector("example.Cached")),
|
||||
)
|
||||
assert(
|
||||
render(TestSummary.default, entries) ==
|
||||
Vector(
|
||||
"Test summary (1 test task failed):",
|
||||
" a / Test / test",
|
||||
" example.Failing FAIL",
|
||||
"",
|
||||
"failed: total 3, failed 1, errors 0, passed 2, cached 1",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
test("Success renders the per-suite detail with cached markers") {
|
||||
val entries = Vector(
|
||||
entry("core2 / Test / testQuick", output("example.ExampleSuite2" -> suite(passed = 1))),
|
||||
entry(
|
||||
"core1 / Test / testQuick",
|
||||
output("example.ExampleSuite1" -> suite(passed = 1)),
|
||||
Vector("example.ExampleTest1B"),
|
||||
),
|
||||
entry("core3 / Test / testQuick", output(), Vector("example.ExampleSuite3")),
|
||||
)
|
||||
assert(
|
||||
render(TestSummary.Success, entries) ==
|
||||
Vector(
|
||||
"Test summary (3 test tasks succeeded):",
|
||||
" core1 / Test / testQuick",
|
||||
" example.ExampleSuite1 PASS",
|
||||
" example.ExampleTest1B (cached) PASS",
|
||||
" core2 / Test / testQuick",
|
||||
" example.ExampleSuite2 PASS",
|
||||
" core3 / Test / testQuick",
|
||||
" example.ExampleSuite3 (cached) PASS",
|
||||
"",
|
||||
"passed: total 4, failed 0, errors 0, passed 4, cached 2",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
test("Success header counts only the failing task even though the body lists every task") {
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)),
|
||||
),
|
||||
entry("b / Test / test", output("example.Passing" -> suite(passed = 1))),
|
||||
)
|
||||
assert(
|
||||
render(TestSummary.Success, entries) ==
|
||||
Vector(
|
||||
"Test summary (1 test task failed):",
|
||||
" a / Test / test",
|
||||
" example.Failing FAIL",
|
||||
" b / Test / test",
|
||||
" example.Passing PASS",
|
||||
"",
|
||||
"failed: total 2, failed 1, errors 0, passed 1, cached 0",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
test("Failure lists only failing suites, collapsing to the aggregate line when all pass") {
|
||||
val failedSuite = new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output("example.Passing" -> suite(passed = 1), "example.Failing" -> failedSuite),
|
||||
Vector("example.Cached"),
|
||||
)
|
||||
)
|
||||
assert(
|
||||
render(TestSummary.Failure, entries) ==
|
||||
Vector(
|
||||
"Test summary (1 test task failed):",
|
||||
" a / Test / test",
|
||||
" example.Failing FAIL",
|
||||
"",
|
||||
"failed: total 3, failed 1, errors 0, passed 2, cached 1",
|
||||
)
|
||||
)
|
||||
val allPassing =
|
||||
Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1))))
|
||||
assert(
|
||||
render(TestSummary.Failure, allPassing) ==
|
||||
Vector("passed: total 1, failed 0, errors 0, passed 1, cached 0")
|
||||
)
|
||||
}
|
||||
|
||||
test("isColorEnabled wraps only the status word, not a (cached) prefix") {
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output(
|
||||
"example.Passing" -> suite(passed = 1),
|
||||
"example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0),
|
||||
),
|
||||
Vector("example.Cached"),
|
||||
)
|
||||
)
|
||||
val lines = Summary.render(TestSummary.Success, entries, isColorEnabled = true)
|
||||
assert(lines.exists(_.endsWith(s"${GREEN}PASS$RESET")))
|
||||
assert(lines.exists(_.endsWith(s"${RED}FAIL$RESET")))
|
||||
assert(lines.exists(l => l.contains("(cached) ") && l.endsWith(s"${GREEN}PASS$RESET")))
|
||||
assert(!lines.exists(_.contains(s"$GREEN(cached)")))
|
||||
}
|
||||
|
||||
test("None suppresses all output, even on failure") {
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0)),
|
||||
)
|
||||
)
|
||||
assert(render(TestSummary.none, entries).isEmpty)
|
||||
}
|
||||
|
||||
test("summary logs at info level when everything passes") {
|
||||
withLog: log =>
|
||||
val entries = Vector(entry("a / Test / test", output("example.Passing" -> suite(passed = 1))))
|
||||
Summary(TestSummary.Success).summary(log, entries)
|
||||
val expected = Summary
|
||||
.render(TestSummary.Success, entries, Terminal.get.isColorEnabled)
|
||||
.map(line => if line.isEmpty then " " else line)
|
||||
assert(log.lines.map(_._2).toVector == expected)
|
||||
assert(log.lines.forall(_._1 == "info"))
|
||||
}
|
||||
|
||||
test("summary logs at error level when entries contain a failure") {
|
||||
withLog: log =>
|
||||
val entries = Vector(
|
||||
entry(
|
||||
"a / Test / test",
|
||||
output("example.Failing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0))
|
||||
)
|
||||
)
|
||||
Summary(TestSummary.default).summary(log, entries)
|
||||
val expected = Summary
|
||||
.render(TestSummary.default, entries, Terminal.get.isColorEnabled)
|
||||
.map(line => if line.isEmpty then " " else line)
|
||||
assert(log.lines.map(_._2).toVector == expected)
|
||||
assert(log.lines.nonEmpty)
|
||||
assert(log.lines.forall(_._1 == "error"))
|
||||
}
|
||||
|
||||
test("Summary.run delegates to the Default per-task logger"):
|
||||
withLog: log =>
|
||||
Summary().run(log, output("A" -> suite(passed = 1)), "a / Test / test")
|
||||
assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0"))
|
||||
|
||||
test("printStandard's 4-arg run folds the cached count into total and passed"):
|
||||
withLog: log =>
|
||||
TestResultLogger.Default.run(
|
||||
log,
|
||||
output("A" -> suite(passed = 1)),
|
||||
"a / Test / test",
|
||||
Vector("B", "C"),
|
||||
)
|
||||
assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2"))
|
||||
|
||||
test("printStandard's 3-arg run always shows the cached count, even at zero"):
|
||||
withLog: log =>
|
||||
TestResultLogger.Default.run(log, output("A" -> suite(passed = 1)), "a / Test / test")
|
||||
assert(log.lines.exists(_._2 == "passed: total 1, failed 0, errors 0, passed 1, cached 0"))
|
||||
|
||||
test("choose (via SilentWhenNoTests) forwards the cached count to the chosen branch"):
|
||||
withLog: log =>
|
||||
TestResultLogger.SilentWhenNoTests.run(
|
||||
log,
|
||||
output("A" -> suite(passed = 1)),
|
||||
"a / Test / test",
|
||||
Vector("B", "C"),
|
||||
)
|
||||
assert(log.lines.exists(_._2 == "passed: total 3, failed 0, errors 0, passed 3, cached 2"))
|
||||
|
||||
end TestResultLoggerSummaryTest
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
|
||||
import sbt.protocol.testing.TestResult
|
||||
|
||||
object TestSummaryTest extends verify.BasicTestSuite:
|
||||
|
||||
private def output(suites: (String, SuiteResult)*): Tests.Output =
|
||||
Tests.Output(TestResult.Passed, suites.toMap, Iterable.empty)
|
||||
|
||||
test("append strips SuiteResult.throwables so lingering entries cannot pin the classloader") {
|
||||
val thrown = new AssertionError("boom")
|
||||
val withThrowables =
|
||||
new SuiteResult(TestResult.Passed, 1, 0, 0, 0, 0, 0, 0, thrown :: Nil)
|
||||
TestSummary.clear()
|
||||
TestSummary.append("a / Test / test", output("A" -> withThrowables), Vector.empty, Vector.empty)
|
||||
val drained = TestSummary.drain()
|
||||
assert(drained.size == 1)
|
||||
assert(drained.head.testOutput.events("A").throwables.isEmpty)
|
||||
assert(TestSummary.drain().isEmpty)
|
||||
}
|
||||
|
||||
end TestSummaryTest
|
||||
|
|
@ -1,263 +0,0 @@
|
|||
/*
|
||||
* sbt
|
||||
* Copyright 2023, Scala center
|
||||
* Copyright 2011 - 2022, Lightbend, Inc.
|
||||
* Copyright 2008 - 2010, Mark Harrah
|
||||
* Licensed under Apache License 2.0 (see LICENSE)
|
||||
*/
|
||||
|
||||
package sbt
|
||||
package internal
|
||||
package testing
|
||||
|
||||
import sbt.Incomplete
|
||||
import sbt.SuiteResult
|
||||
import sbt.Tests
|
||||
import sbt.TestsFailedException
|
||||
import sbt.protocol.testing.TestResult
|
||||
import sbt.util.Logger
|
||||
|
||||
object TestRecapTest extends verify.BasicTestSuite:
|
||||
|
||||
private def output(result: TestResult, suites: (String, SuiteResult)*): Tests.Output =
|
||||
Tests.Output(result, suites.toMap, Iterable.empty)
|
||||
|
||||
private def suite(result: TestResult): SuiteResult =
|
||||
new SuiteResult(result, 0, 1, 0, 0, 0, 0, 0)
|
||||
|
||||
private def failure(
|
||||
taskName: String,
|
||||
result: TestResult,
|
||||
suiteName: String
|
||||
): TestsFailedException =
|
||||
new TestsFailedException(
|
||||
taskName,
|
||||
Some(output(result, suiteName -> suite(result)))
|
||||
)
|
||||
|
||||
/** Build an Incomplete tree carrying the given TestsFailedExceptions as direct causes. */
|
||||
private def incompleteOf(exceptions: TestsFailedException*): Incomplete =
|
||||
new Incomplete(
|
||||
node = None,
|
||||
causes = exceptions.map(e => new Incomplete(node = None, directCause = Some(e)))
|
||||
)
|
||||
|
||||
private class Capture extends Logger:
|
||||
val lines: scala.collection.mutable.ArrayBuffer[(String, String)] =
|
||||
scala.collection.mutable.ArrayBuffer.empty
|
||||
override def trace(t: => Throwable): Unit = ()
|
||||
override def success(msg: => String): Unit = ()
|
||||
override def log(level: sbt.util.Level.Value, msg: => String): Unit =
|
||||
lines += level.toString -> msg
|
||||
|
||||
test("collect strips SuiteResult.throwables so the recap cannot pin the test classloader") {
|
||||
// A test-thrown Throwable's backtrace pins the Class objects of every frame, and a Class
|
||||
// strongly references its defining classloader; the recap is stashed on State.attributes,
|
||||
// so retaining them would keep the test classloader (and its jar handles) alive.
|
||||
val thrown = new AssertionError("boom")
|
||||
val withThrowables =
|
||||
new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil)
|
||||
val i = incompleteOf(
|
||||
new TestsFailedException(
|
||||
"a / Test / test",
|
||||
Some(output(TestResult.Failed, "AFailing" -> withThrowables))
|
||||
)
|
||||
)
|
||||
val collected = TestRecap.collect(i)
|
||||
val retained = collected.head.testOutput.get.events("AFailing")
|
||||
assert(retained.throwables.isEmpty, "throwables should be dropped from the retained output")
|
||||
// Everything the recap actually renders must survive.
|
||||
assert(retained.result == TestResult.Failed)
|
||||
assert(retained.failureCount == 1)
|
||||
assert(collected.head.testOutput.get.overall == TestResult.Failed)
|
||||
}
|
||||
|
||||
test("collect sanitizes a copy and leaves the source exception's throwables intact") {
|
||||
// The stripping above must not mutate the TestsFailedException in the Incomplete tree:
|
||||
// that exception is what error reporting and the ClassLoaderLayeringStrategy diagnostic
|
||||
// in Defaults read, and both need the real throwables. Only the copy parked on
|
||||
// State.attributes is sanitized.
|
||||
val thrown = new AssertionError("boom")
|
||||
val source = new TestsFailedException(
|
||||
"a / Test / test",
|
||||
Some(
|
||||
output(
|
||||
TestResult.Failed,
|
||||
"AFailing" -> new SuiteResult(TestResult.Failed, 0, 1, 0, 0, 0, 0, 0, thrown :: Nil)
|
||||
)
|
||||
)
|
||||
)
|
||||
val collected = TestRecap.collect(incompleteOf(source))
|
||||
assert(collected.head.testOutput.get.events("AFailing").throwables.isEmpty)
|
||||
val original = source.testOutput.get.events("AFailing").throwables
|
||||
assert(
|
||||
original == (thrown :: Nil),
|
||||
s"collect must not strip the source exception, but its throwables became: $original"
|
||||
)
|
||||
}
|
||||
|
||||
test("collect picks up TestsFailedException payloads from the Incomplete tree") {
|
||||
val i = incompleteOf(
|
||||
failure("a / Test / test", TestResult.Failed, "AFailing"),
|
||||
failure("c / Test / test", TestResult.Error, "CErroring"),
|
||||
)
|
||||
val collected = TestRecap.collect(i)
|
||||
assert(collected.map(_.taskName).sorted == Vector("a / Test / test", "c / Test / test"))
|
||||
val resultsBy = collected.flatMap(f => f.testOutput.map(o => f.taskName -> o.overall)).toMap
|
||||
assert(resultsBy("a / Test / test") == TestResult.Failed)
|
||||
assert(resultsBy("c / Test / test") == TestResult.Error)
|
||||
}
|
||||
|
||||
test("collect retains TestsFailedException without payload as a stub entry") {
|
||||
val noDetail = new TestsFailedException // back-compat no-arg constructor
|
||||
val i = new Incomplete(
|
||||
node = None,
|
||||
causes = Seq(
|
||||
new Incomplete(node = None, directCause = Some(noDetail)),
|
||||
new Incomplete(
|
||||
node = None,
|
||||
directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail"))
|
||||
),
|
||||
)
|
||||
)
|
||||
val collected = TestRecap.collect(i)
|
||||
assert(collected.size == 2, s"expected 2 entries, got $collected")
|
||||
val stub = collected.find(_.testOutput.isEmpty)
|
||||
assert(stub.isDefined, "no-detail failure should still produce a Failure entry")
|
||||
assert(stub.get.taskName == "")
|
||||
}
|
||||
|
||||
test("collect skips exceptions that aren't TestsFailedException") {
|
||||
val i = new Incomplete(
|
||||
node = None,
|
||||
causes = Seq(
|
||||
new Incomplete(node = None, directCause = Some(new RuntimeException("nope"))),
|
||||
new Incomplete(
|
||||
node = None,
|
||||
directCause = Some(failure("ok / Test / test", TestResult.Failed, "OkFail"))
|
||||
),
|
||||
)
|
||||
)
|
||||
assert(TestRecap.collect(i).map(_.taskName) == Vector("ok / Test / test"))
|
||||
}
|
||||
|
||||
test("collect retains taskName when a TestsFailedException is rebranded with task name only") {
|
||||
// Simulates the `testFull` / `inputTests0` catch path: an upstream
|
||||
// legacy code path threw `new TestsFailedException()` (no detail), the
|
||||
// task wrapper caught it and re-threw with `(taskName, e.testOutput)`
|
||||
// to attach the project context.
|
||||
val tagged = new TestsFailedException("a / Test / test", None)
|
||||
val i = new Incomplete(node = None, directCause = Some(tagged))
|
||||
val collected = TestRecap.collect(i)
|
||||
assert(collected == Vector(TestRecap.Failure("a / Test / test", None)))
|
||||
}
|
||||
|
||||
test("collect deduplicates a TestsFailedException shared across Incomplete paths") {
|
||||
val shared = failure("a / Test / test", TestResult.Failed, "AFail")
|
||||
val i = new Incomplete(
|
||||
node = None,
|
||||
causes = Seq(
|
||||
new Incomplete(node = None, directCause = Some(shared)),
|
||||
new Incomplete(node = None, directCause = Some(shared)),
|
||||
)
|
||||
)
|
||||
val collected = TestRecap.collect(i)
|
||||
assert(
|
||||
collected.size == 1,
|
||||
s"shared failure should be reported once, got ${collected.size}: $collected"
|
||||
)
|
||||
}
|
||||
|
||||
test("render emits header, per-task counts, and indented suite names") {
|
||||
val failures = Vector(
|
||||
TestRecap.Failure(
|
||||
"a / Test / test",
|
||||
Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed)))
|
||||
),
|
||||
TestRecap.Failure(
|
||||
"c / Test / test",
|
||||
Some(output(TestResult.Error, "CErroring" -> suite(TestResult.Error)))
|
||||
),
|
||||
)
|
||||
val lines = TestRecap.render(failures)
|
||||
assert(lines.headOption.contains("Test failures recap (2 test tasks failed):"))
|
||||
assert(lines.exists(_.contains("a / Test / test:")))
|
||||
assert(lines.exists(_.contains("c / Test / test:")))
|
||||
assert(lines.exists(_.contains("AFailing")))
|
||||
assert(lines.exists(_.contains("CErroring")))
|
||||
assert(lines.contains(" Failed tests:"))
|
||||
assert(lines.contains(" Error during tests:"))
|
||||
}
|
||||
|
||||
test("render sorts failures by taskName for stable output (empty names last)") {
|
||||
val failures = Vector(
|
||||
TestRecap.Failure("zz / Test / test", None),
|
||||
TestRecap.Failure("", None),
|
||||
TestRecap.Failure("aa / Test / test", None),
|
||||
TestRecap.Failure("mm / Test / test", None),
|
||||
)
|
||||
val lines = TestRecap.render(failures)
|
||||
val headerLines = lines.filter(_.startsWith(" "))
|
||||
val order = headerLines.map(_.trim.takeWhile(_ != ':'))
|
||||
assert(
|
||||
order == Vector("aa / Test / test", "mm / Test / test", "zz / Test / test", "<unknown>"),
|
||||
s"unexpected order: $order"
|
||||
)
|
||||
}
|
||||
|
||||
test("render emits singular header when exactly one task failed") {
|
||||
val one = Vector(
|
||||
TestRecap.Failure(
|
||||
"a / Test / test",
|
||||
Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed)))
|
||||
)
|
||||
)
|
||||
assert(TestRecap.render(one).head == "Test failures recap (1 test task failed):")
|
||||
}
|
||||
|
||||
test("render shows '(no details)' for failures without a Tests.Output payload") {
|
||||
val failures = Vector(TestRecap.Failure("a / Test / test", testOutput = None))
|
||||
val lines = TestRecap.render(failures)
|
||||
assert(
|
||||
lines.exists(_.contains("a / Test / test: (no details)")),
|
||||
s"expected '(no details)' entry, got $lines"
|
||||
)
|
||||
}
|
||||
|
||||
test("render shows '<unknown>' when a failure carries no task name") {
|
||||
val failures = Vector(TestRecap.Failure(taskName = "", testOutput = None))
|
||||
val lines = TestRecap.render(failures)
|
||||
assert(
|
||||
lines.exists(_.contains("<unknown>: (no details)")),
|
||||
s"expected '<unknown>' placeholder, got $lines"
|
||||
)
|
||||
}
|
||||
|
||||
test("render is empty when there are no failures") {
|
||||
assert(TestRecap.render(Vector.empty).isEmpty)
|
||||
}
|
||||
|
||||
test("formatTo emits one error-level log line per rendered line") {
|
||||
val failures = Vector(
|
||||
TestRecap.Failure(
|
||||
"a / Test / test",
|
||||
Some(output(TestResult.Failed, "AFailing" -> suite(TestResult.Failed)))
|
||||
)
|
||||
)
|
||||
val log = new Capture
|
||||
TestRecap.formatTo(log, failures)
|
||||
val rendered = TestRecap.render(failures)
|
||||
assert(
|
||||
log.lines.size == rendered.size,
|
||||
s"expected ${rendered.size} log calls, got ${log.lines.size}"
|
||||
)
|
||||
assert(log.lines.forall(_._1 == "error"), s"all lines should be error level: ${log.lines}")
|
||||
}
|
||||
|
||||
test("formatTo is a no-op when there are no failures") {
|
||||
val log = new Capture
|
||||
TestRecap.formatTo(log, Vector.empty)
|
||||
assert(log.lines.isEmpty)
|
||||
}
|
||||
|
||||
end TestRecapTest
|
||||
|
|
@ -1227,7 +1227,9 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
testListeners :== Nil,
|
||||
testOptions :== Nil,
|
||||
testOptionDigests :== Nil,
|
||||
testResultLogger :== TestResultLogger.Default,
|
||||
testResultLogger :== TestResultLogger.SilentWhenNoTests,
|
||||
testSummary :== SysProp.testSummary,
|
||||
testSummaryLogger := TestResultLogger.Defaults.Summary(testSummary.value),
|
||||
testOnly / testFilter :== (IncrementalTest.selectedFilter),
|
||||
testSelected / testFilter :== (IncrementalTest.selectedFilter),
|
||||
extraTestDigests :== Nil,
|
||||
|
|
@ -1295,9 +1297,10 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
val taskName = Project.showContextKey(state.value).show(resolvedScoped.value)
|
||||
try
|
||||
val output = executeTests.value
|
||||
TestSummary.append(taskName, output, cached = Vector.empty, adhocOptions = Vector.empty)
|
||||
trl.run(streams.value.log, output, taskName)
|
||||
// Throw with task name + Output so the cross-project recap
|
||||
// (TestRecap.collect) can surface them. The throw lives here
|
||||
// Throw with task name + Output so the aggregation boundary
|
||||
// (Aggregation.runTasks) can signal the failure. The throw lives here
|
||||
// rather than in TestResultLogger so user-overridden loggers
|
||||
// cannot accidentally suppress the failure signal.
|
||||
output.overall match
|
||||
|
|
@ -1309,7 +1312,7 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
// Tag any no-detail TestsFailedException (legacy executeTests
|
||||
// adapters, third-party Tests.Setup actions, anything constructing
|
||||
// `new TestsFailedException()` via the back-compat ctor) with the
|
||||
// task name on its way out so the recap doesn't render <unknown>.
|
||||
// task name on its way out so error reporting has it to show.
|
||||
case e: TestsFailedException if e.taskName.isEmpty =>
|
||||
throw new TestsFailedException(taskName, e.testOutput)
|
||||
finally close(testLoader.value)
|
||||
|
|
@ -1463,8 +1466,9 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
inputTests0.mapReferenced(Def.mapScope((s) => s.rescope(key.key)))
|
||||
|
||||
private lazy val inputTests0: Initialize[InputTask[TestResult]] = {
|
||||
val parser = loadForParser(definedTestNames)((s, i) => testOnlyParser(s, i getOrElse Nil))
|
||||
ParserGen(parser).flatMapTask { (selected, frameworkOptions) =>
|
||||
val parser =
|
||||
loadForParser(definedTestNames)((s, i) => testOnlyParserWithOption(s, i getOrElse Nil))
|
||||
ParserGen(parser).flatMapTask { (selected, frameworkOptions, adhocOptions) =>
|
||||
val s = streams.value
|
||||
val filter = testFilter.value
|
||||
val config = testExecution.value
|
||||
|
|
@ -1505,11 +1509,21 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
)
|
||||
val taskName = display.show(resolvedScoped.value)
|
||||
val trl = testResultLogger.value
|
||||
val digests = definedTestDigests.value
|
||||
val cacheConfig = Def.cacheConfiguration.value
|
||||
(Def
|
||||
.value[Task[Tests.Output]] { output })
|
||||
.map: out =>
|
||||
val cached = IncrementalTest.cachedTestNames(
|
||||
digests,
|
||||
cacheConfig,
|
||||
out.events.keySet,
|
||||
selected,
|
||||
frameworkOptions,
|
||||
)
|
||||
TestSummary.append(taskName, out, cached, adhocOptions.toVector)
|
||||
try
|
||||
trl.run(s.log, out, taskName)
|
||||
trl.run(s.log, out, taskName, cached)
|
||||
out.overall match
|
||||
case TestResult.Error | TestResult.Failed =>
|
||||
throw new TestsFailedException(taskName, Some(out))
|
||||
|
|
@ -2629,6 +2643,17 @@ object Defaults extends BuildCommon with DefExtra {
|
|||
Space ~> token(NotSpace.examples(mainClasses.toSet)) ~ spaceDelimited("<arg>")
|
||||
}
|
||||
|
||||
private def testOnlyParserWithOption
|
||||
: (State, Seq[String]) => Parser[(Seq[String], Seq[String], Seq[Tests.AdhocOption])] =
|
||||
(state, tests) =>
|
||||
import DefaultParsers.*
|
||||
val selectTests = distinctParser(tests.toSet, true)
|
||||
val frameworkOpts = (token(Space) ~> token("--") ~> spaceDelimited("<option>")) ?? Nil
|
||||
val options = (token(Space) ~> Tests.AdhocOption.parser).?
|
||||
(options ~ selectTests ~ options ~ frameworkOpts).map { case o1 ~ t ~ o2 ~ f =>
|
||||
(t, f, o1.toList ::: o2.toList)
|
||||
}
|
||||
|
||||
def testOnlyParser: (State, Seq[String]) => Parser[(Seq[String], Seq[String])] = {
|
||||
(state, tests) =>
|
||||
import DefaultParsers.*
|
||||
|
|
|
|||
|
|
@ -405,6 +405,9 @@ object Keys {
|
|||
val testFilter = taskKey[Seq[String] => Seq[String => Boolean]]("Filter controlling whether the test is executed").withRank(DTask)
|
||||
@transient
|
||||
val testResultLogger = settingKey[TestResultLogger]("Logs results after a test task completes.").withRank(DTask)
|
||||
val testSummary = settingKey[TestSummary]("The style of the test summary displayed after an aggregated test run.").withRank(CSetting)
|
||||
@transient
|
||||
val testSummaryLogger = settingKey[TestResultLogger]("Logs test summary after an aggregated test completes.").withRank(DTask)
|
||||
val testGrouping = taskKey[Seq[Tests.Group]]("Collects discovered tests into groups. Whether to fork and the options for forking are configurable on a per-group basis.").withRank(BMinusTask)
|
||||
val isModule = AttributeKey[Boolean]("isModule", "True if the target is a module.", DSetting)
|
||||
val extraTestDigests = taskKey[Seq[Digest]]("Extra digests that would invalidate test caching").withRank(DTask)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ package sbt
|
|||
package internal
|
||||
|
||||
import sbt.Def.{ ScopedKey, Settings }
|
||||
import sbt.Keys.{ showSuccess, showTiming }
|
||||
import sbt.Keys.{ showSuccess, showTiming, testSummaryLogger }
|
||||
import sbt.ProjectExtra.*
|
||||
import sbt.ScopeAxis.{ Select, Zero }
|
||||
import sbt.internal.util.complete.Parser
|
||||
|
|
@ -126,6 +126,7 @@ object Aggregation {
|
|||
val config = extractedTaskConfig(extracted, structure, s)
|
||||
val start = System.currentTimeMillis
|
||||
Def.cacheEventLog.clear()
|
||||
TestSummary.clear()
|
||||
val (newS, result) = withStreams(structure, s): str =>
|
||||
val transform = nodeView(s, str, roots, extra)
|
||||
runTask(toRun, s, str, structure.index.triggers, config)(using transform)
|
||||
|
|
@ -140,16 +141,31 @@ object Aggregation {
|
|||
show: ShowConfig
|
||||
)(using display: Show[ScopedKey[?]]): State =
|
||||
val complete = timedRun[A1](s, ts, extra)
|
||||
val testEntries = TestSummary.drain()
|
||||
if testEntries.nonEmpty then
|
||||
val extracted = Project.extract(complete.state)
|
||||
val logger = (extracted.currentRef / testSummaryLogger)
|
||||
.get(extracted.structure.data)
|
||||
.getOrElse(TestResultLogger.Defaults.Summary())
|
||||
val adhocMode = testEntries.flatMap(_.options).collectFirst {
|
||||
case Tests.AdhocOption.Summary(mode) => mode
|
||||
}
|
||||
val effectiveLogger = (logger, adhocMode) match
|
||||
case (s: TestResultLogger.Defaults.Summary, Some(mode)) => s.copy(mode = mode)
|
||||
case _ => logger
|
||||
effectiveLogger.summary(
|
||||
complete.state.log,
|
||||
testEntries.map(e => (e.testOutput, e.taskName, e.cached))
|
||||
)
|
||||
showRun(complete, show)
|
||||
complete.results match
|
||||
case Result.Inc(i) =>
|
||||
val failures = sbt.internal.testing.TestRecap.collect(i)
|
||||
val afterHandle = complete.state.handleError(i)
|
||||
if failures.nonEmpty then
|
||||
sbt.internal.testing.TestRecap.formatTo(afterHandle.log, failures)
|
||||
afterHandle.put(sbt.internal.testing.TestRecap.recapKey, failures)
|
||||
if testEntries.nonEmpty then afterHandle.put(TestSummary.entriesKey, testEntries)
|
||||
else afterHandle
|
||||
case Result.Value(_) => complete.state
|
||||
case Result.Value(_) =>
|
||||
if testEntries.nonEmpty then complete.state.put(TestSummary.entriesKey, testEntries)
|
||||
else complete.state
|
||||
|
||||
def printSuccess(
|
||||
start: Long,
|
||||
|
|
|
|||
|
|
@ -96,6 +96,33 @@ object IncrementalTest:
|
|||
frameworkOptions: Seq[String]
|
||||
): (Seq[String], Digest, Digest) = (frameworkOptions, value, Digest.zero)
|
||||
|
||||
/**
|
||||
* Defined tests whose passing result was reused from the action cache in
|
||||
* the current run: selected by `selected` (same glob semantics as
|
||||
* `filterTask`), carrying a cached success for `frameworkOptions`, and not
|
||||
* re-executed (`executed` is the suite-name set of the run's
|
||||
* `Tests.Output.events`, which keeps this empty for `testOnly` /
|
||||
* `testFull` variants that rerun cached tests).
|
||||
*/
|
||||
private[sbt] def cachedTestNames(
|
||||
digests: Map[String, Digest],
|
||||
config: BuildWideCacheConfiguration,
|
||||
executed: Set[String],
|
||||
selected: Seq[String],
|
||||
frameworkOptions: Seq[String],
|
||||
): Vector[String] =
|
||||
val filters = selectedFilter(selected)
|
||||
digests.iterator
|
||||
.collect {
|
||||
case (name, ts) if !executed.contains(name) && filters.exists(_(name)) && {
|
||||
val input = cacheInput(ts, frameworkOptions)
|
||||
ActionCache.exists(input._1, input._2, input._3, config)
|
||||
} =>
|
||||
name
|
||||
}
|
||||
.toVector
|
||||
.sorted
|
||||
|
||||
end IncrementalTest
|
||||
|
||||
private[sbt] case class TestStatusReporter(
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ object LintUnused {
|
|||
serverIdleTimeout,
|
||||
shellPrompt,
|
||||
sLog,
|
||||
testSummary,
|
||||
traceLevel,
|
||||
sonaDeploymentName,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -27,14 +27,7 @@ import sbt.nio.Keys.*
|
|||
// See also LineReader.scala
|
||||
object SysProp:
|
||||
def booleanOpt(name: String): Option[Boolean] =
|
||||
sys.props.get(name) match {
|
||||
case Some(x) => parseBoolean(x)
|
||||
case _ =>
|
||||
sys.env.get(name.toUpperCase(Locale.ENGLISH).replace('.', '_')) match {
|
||||
case Some(x) => parseBoolean(x)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
strOpt(name).flatMap(parseBoolean)
|
||||
private def parseBoolean(value: String): Option[Boolean] =
|
||||
value.toLowerCase(Locale.ENGLISH) match {
|
||||
case "1" | "always" | "true" => Some(true)
|
||||
|
|
@ -46,6 +39,11 @@ object SysProp:
|
|||
def getOrFalse(name: String): Boolean = booleanOpt(name).getOrElse(false)
|
||||
def getOrTrue(name: String): Boolean = booleanOpt(name).getOrElse(true)
|
||||
|
||||
def strOpt(name: String): Option[String] =
|
||||
sys.props
|
||||
.get(name)
|
||||
.orElse(sys.env.get(name.toUpperCase(Locale.ENGLISH).replace('.', '_')))
|
||||
|
||||
def long(name: String, default: Long): Long =
|
||||
sys.props.get(name) match {
|
||||
case Some(str) =>
|
||||
|
|
@ -123,6 +121,9 @@ object SysProp:
|
|||
@deprecated("Resident compilation is no longer supported", "1.4.0")
|
||||
def residentLimit: Int = int("sbt.resident.limit", 0)
|
||||
|
||||
def testSummary: TestSummary =
|
||||
strOpt("sbt.test_summary").flatMap(Tests.parseTestSummary).getOrElse(TestSummary.default)
|
||||
|
||||
/**
|
||||
* Indicates whether formatting has been disabled in environment variables.
|
||||
* 1. -Dsbt.log.noformat=true means no formatting.
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
ThisBuild / scalaVersion := "2.13.16"
|
||||
|
||||
def junit = libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
|
||||
|
||||
lazy val a = project.settings(junit)
|
||||
lazy val b = project.settings(junit)
|
||||
lazy val c = project.settings(junit)
|
||||
|
||||
lazy val root = (project in file("."))
|
||||
.aggregate(a, b, c)
|
||||
.settings(
|
||||
commands ++= Seq(
|
||||
sbt.multifailurerecap.Checks.verifyRecap,
|
||||
sbt.multifailurerecap.Checks.verifyNoRecap,
|
||||
)
|
||||
)
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
import org.junit.Test;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class FailingTestC {
|
||||
@Test public void failure() { fail("intentional failure C"); }
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
package sbt
|
||||
package multifailurerecap
|
||||
|
||||
import sbt.internal.testing.TestRecap
|
||||
|
||||
/**
|
||||
* Lives in package `sbt` so it can access `TestRecap`, which is `private[sbt]`.
|
||||
*
|
||||
* A scripted statement that fails (`-> test`) closes the inner sbt's IPC
|
||||
* server (see `SbtHandler.onNewSbtInstance`'s catch block), which terminates
|
||||
* the inner sbt JVM. Scripted then launches a fresh JVM for the next
|
||||
* statement, so a State attribute set inside a failing statement cannot be
|
||||
* read by a follow-up `> check`. We avoid that by running `test` from
|
||||
* inside a Command (here) via `Command.process`, all within one JVM.
|
||||
*
|
||||
* `recapKey` is monotonic-latest-failure: never proactively cleared.
|
||||
* That means across CI scripted shards (which share an inner sbt across
|
||||
* tests with `reload;initialize` between them) a prior test that left a
|
||||
* recap entry is visible to this test. Both commands therefore strip
|
||||
* `recapKey` from the incoming state before doing their own assertions
|
||||
* and again on the way out, so they are hermetic w.r.t. anything other
|
||||
* scripted tests in the same shard may have left behind.
|
||||
*/
|
||||
object Checks {
|
||||
|
||||
val verifyRecap: Command = Command.command("verifyRecap") { state =>
|
||||
val cleared = state.remove(TestRecap.recapKey)
|
||||
val afterTest = Command.process("test", cleared)
|
||||
val recap = afterTest.get(TestRecap.recapKey).getOrElse {
|
||||
sys.error("TestRecap.recapKey not set on state after aggregated test failure")
|
||||
}
|
||||
val names = recap.map(_.taskName).toSet
|
||||
assert(recap.size == 2, s"expected 2 failures, got ${recap.size}: $names")
|
||||
assert(names.exists(_.startsWith("a / ")), s"recap missing project a: $names")
|
||||
assert(names.exists(_.startsWith("c / ")), s"recap missing project c: $names")
|
||||
assert(!names.exists(_.startsWith("b / ")), s"recap should not list project b: $names")
|
||||
recap.foreach { f =>
|
||||
assert(f.testOutput.isDefined, s"${f.taskName} has no Tests.Output payload")
|
||||
val failedSuites = f.testOutput.get.events.values.count: s =>
|
||||
s.result == sbt.protocol.testing.TestResult.Failed
|
||||
assert(failedSuites >= 1, s"${f.taskName} has no failed suite: ${f.testOutput.get.events}")
|
||||
// The retained output must not carry live test-thrown Throwables: their backtraces pin
|
||||
// the test classloader (and its open jar handles) for as long as the recap sits on
|
||||
// State.attributes, which is the whole session. The recap only renders names and counts.
|
||||
f.testOutput.get.events.foreach { case (suiteName, s) =>
|
||||
assert(
|
||||
s.throwables.isEmpty,
|
||||
s"${f.taskName} suite $suiteName retained ${s.throwables.size} throwable(s): " +
|
||||
s.throwables.map(_.getClass.getName).mkString(", ")
|
||||
)
|
||||
}
|
||||
}
|
||||
val lines = TestRecap.render(recap)
|
||||
assert(
|
||||
lines.head.startsWith("Test failures recap (2 test tasks failed):"),
|
||||
s"unexpected header: ${lines.head}"
|
||||
)
|
||||
// Return the original state with recapKey stripped so the next
|
||||
// scripted statement starts hermetic.
|
||||
state.remove(TestRecap.recapKey)
|
||||
}
|
||||
|
||||
val verifyNoRecap: Command = Command.command("verifyNoRecap") { state =>
|
||||
val cleared = state.remove(TestRecap.recapKey)
|
||||
val afterTest = Command.process("test", cleared)
|
||||
afterTest.get(TestRecap.recapKey) match {
|
||||
case None => state.remove(TestRecap.recapKey)
|
||||
case Some(r) =>
|
||||
sys.error(s"unexpected recap after passing test run: ${r.map(_.taskName)}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Verify the aggregated test failure recap end-to-end. verifyRecap is a
|
||||
# Command (not a Task) that internally runs `test` via Command.process,
|
||||
# reads the State attribute set by Aggregation.runTasks, and asserts the
|
||||
# recap content. The command returns the *original* state (not the
|
||||
# post-failure state) so verifyRecap itself is not marked failed -- the
|
||||
# inner `test` failure has already been inspected, and we want this
|
||||
# statement to succeed when the recap is well-formed. See
|
||||
# project/Checks.scala for details.
|
||||
> verifyRecap
|
||||
|
||||
# After removing the failing tests, the aggregated test should succeed
|
||||
# and the recap state attribute should not be set on this fresh JVM.
|
||||
$ delete a/src/test/java/FailingTestA.java
|
||||
$ delete c/src/test/java/FailingTestC.java
|
||||
> clean
|
||||
> verifyNoRecap
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.example.a;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestA {
|
||||
@Test public void success() { /* passes */ }
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.example.a;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestAA {
|
||||
@Test public void success() { /* passes */ }
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.example.b;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestB {
|
||||
@Test public void success() { /* passes */ }
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[error] Test summary (2 test tasks failed):
|
||||
[error] a / Test / testQuick
|
||||
[error] com.example.a.TestA FAIL
|
||||
[error] com.example.a.TestAA (cached) PASS
|
||||
[error] b / Test / testQuick
|
||||
[error] com.example.b.TestB FAIL
|
||||
[error] c / Test / testQuick
|
||||
[error] TestC (cached) PASS
|
||||
[error]
|
||||
[error] failed: total 4, failed 2, errors 0, passed 2, cached 2
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import sbt.internal.util.EscHelpers
|
||||
|
||||
scalaVersion := "2.13.16"
|
||||
|
||||
def junit = libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
|
||||
|
||||
Global / localCacheDirectory := baseDirectory.value / "diskcache"
|
||||
Global / testResultLogger := sbt.recaplog.Capture.captureTestResultLogger
|
||||
commands ++= Seq(sbt.recaplog.Capture.captureLog, sbt.recaplog.Capture.captureTestLog)
|
||||
|
||||
@transient
|
||||
lazy val dropSuccess = inputKey[Unit]("")
|
||||
|
||||
@transient
|
||||
lazy val dropSuccessSorted = inputKey[Unit]("")
|
||||
|
||||
lazy val a = project.settings(junit)
|
||||
lazy val b = project.settings(junit)
|
||||
lazy val c = project.settings(junit)
|
||||
|
||||
lazy val root = rootProject
|
||||
.aggregate(a, b, c)
|
||||
.settings(
|
||||
dropSuccess / aggregate := false,
|
||||
dropSuccess := {
|
||||
val fileName = Def.spaceDelimited("<log file>").parsed.head
|
||||
val log = baseDirectory.value / "target" / fileName
|
||||
val dropped = baseDirectory.value / "target" / "drop.log"
|
||||
IO.writeLines(dropped, IO.readLines(log)
|
||||
.map(EscHelpers.stripColorsAndMoves)
|
||||
.filterNot { line =>
|
||||
line.startsWith("[success]") ||
|
||||
line.contains("elapsed time:") ||
|
||||
line.contains("[info] set current project") ||
|
||||
line.contains("Defining Global / testSummary") ||
|
||||
line.contains("The new value will be used by no settings or tasks.") ||
|
||||
line.contains("Reapplying settings...")
|
||||
})
|
||||
},
|
||||
// sbt runs independent subprojects' tasks in parallel, so testQuick's
|
||||
// per-project lines don't land in a fixed order; sort them so the
|
||||
// golden comparison isn't flaky. (The other captures above are the
|
||||
// deterministic, already-ordered TestSummary rendering, so they don't
|
||||
// need this.)
|
||||
dropSuccessSorted / aggregate := false,
|
||||
dropSuccessSorted := {
|
||||
val fileName = Def.spaceDelimited("<log file>").parsed.head
|
||||
val log = baseDirectory.value / "target" / fileName
|
||||
val dropped = baseDirectory.value / "target" / "drop.log"
|
||||
IO.writeLines(dropped, IO.readLines(log)
|
||||
.map(EscHelpers.stripColorsAndMoves)
|
||||
.filterNot { line =>
|
||||
line.startsWith("[success]") || line.contains("elapsed time:")
|
||||
}
|
||||
.sorted)
|
||||
}
|
||||
)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import org.junit.Test;
|
||||
|
||||
public class PassingTestB {
|
||||
public class TestC {
|
||||
@Test public void success() { /* passes */ }
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
package com.example.a;
|
||||
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class FailingTestA {
|
||||
public class TestA {
|
||||
@Test public void failure() { fail("intentional failure A"); }
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package com.example.b;
|
||||
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class TestB {
|
||||
@Test public void failure() { fail("intentional failure B"); }
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
[error] Test summary (2 test tasks failed):
|
||||
[error] a / Test / testQuick
|
||||
[error] com.example.a.TestA FAIL
|
||||
[error] b / Test / testQuick
|
||||
[error] com.example.b.TestB FAIL
|
||||
[error]
|
||||
[error] failed: total 4, failed 2, errors 0, passed 2, cached 2
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package sbt
|
||||
package recaplog
|
||||
|
||||
import sbt.{ *, given }
|
||||
import java.io.{ File, FileWriter, PrintWriter }
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import sbt.internal.util.{ Appender, ConsoleAppender, ConsoleOut }
|
||||
import sbt.util.{ Level, Logger, LoggerContext }
|
||||
|
||||
/**
|
||||
* `captureLog <file>` attaches an appender to the global logger; the
|
||||
* output target is swappable so the command can be re-run per block.
|
||||
* `captureTestLog <file>` instead points `captureTestResultLogger`
|
||||
* (wired as `Global / testResultLogger`) at a file, since `test`'s own
|
||||
* output goes through a task-scoped logger `captureLog` can't reach.
|
||||
*/
|
||||
object Capture:
|
||||
private val appenderName = "global-capture"
|
||||
private val currentWriter = new AtomicReference[PrintWriter]
|
||||
private val currentOut = new AtomicReference[ConsoleOut]
|
||||
|
||||
private val properties: ConsoleAppender.Properties = new ConsoleAppender.Properties:
|
||||
def isAnsiSupported: Boolean = false
|
||||
def isColorEnabled: Boolean = false
|
||||
def out: ConsoleOut = currentOut.get
|
||||
|
||||
private val appender: Appender =
|
||||
new ConsoleAppender(appenderName, properties, ConsoleAppender.noSuppressedMessage)
|
||||
|
||||
val captureLog: Command = Command.single("captureLog") { (s, fileName) =>
|
||||
val f = s.baseDir / "target" / fileName
|
||||
f.getParentFile.mkdirs()
|
||||
val writer = new PrintWriter(new FileWriter(f, true))
|
||||
currentOut.set(ConsoleOut.printWriterOut(writer))
|
||||
val previous = currentWriter.getAndSet(writer)
|
||||
if previous != null then previous.close()
|
||||
|
||||
val ctx = LoggerContext.globalContext
|
||||
val loggerName = s.globalLogging.full.name
|
||||
if !ctx.appenders(loggerName).exists(_.name == appenderName) then
|
||||
ctx.addAppender(loggerName, appender -> Level.Info)
|
||||
s
|
||||
}
|
||||
|
||||
private val testWriter = new AtomicReference[PrintWriter]
|
||||
|
||||
val captureTestLog: Command = Command.single("captureTestLog") { (s, fileName) =>
|
||||
val f = s.baseDir / "target" / fileName
|
||||
f.getParentFile.mkdirs()
|
||||
val writer = new PrintWriter(new FileWriter(f, true))
|
||||
val previous = testWriter.getAndSet(writer)
|
||||
if previous != null then previous.close()
|
||||
s
|
||||
}
|
||||
|
||||
private def tee(base: Logger, w: PrintWriter): Logger = new Logger:
|
||||
def trace(t: => Throwable): Unit = base.trace(t)
|
||||
def success(message: => String): Unit = base.success(message)
|
||||
def log(level: Level.Value, message: => String): Unit =
|
||||
base.log(level, message)
|
||||
if level.compare(Level.Info) >= 0 then
|
||||
w.println(message)
|
||||
w.flush()
|
||||
|
||||
val captureTestResultLogger: TestResultLogger = TestResultLogger { (log, results, taskName, cached) =>
|
||||
val target = Option(testWriter.get).map(tee(log, _)).getOrElse(log)
|
||||
TestResultLogger.SilentWhenNoTests.run(target, results, taskName, cached)
|
||||
}
|
||||
end Capture
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[info] Test summary (3 test tasks succeeded):
|
||||
[info] a / Test / testQuick
|
||||
[info] com.example.a.TestA (cached) PASS
|
||||
[info] com.example.a.TestAA (cached) PASS
|
||||
[info] b / Test / testQuick
|
||||
[info] com.example.b.TestB (cached) PASS
|
||||
[info] c / Test / testQuick
|
||||
[info] TestC (cached) PASS
|
||||
[info]
|
||||
[info] passed: total 4, failed 0, errors 0, passed 4, cached 4
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
> captureTestLog testquick-capture.log
|
||||
> test
|
||||
> dropSuccessSorted testquick-capture.log
|
||||
$ must-mirror testquick.log.check target/drop.log
|
||||
|
||||
# Both invocations below are now fully cached, thanks to the block above.
|
||||
> captureLog recap-capture.log
|
||||
> test
|
||||
> test
|
||||
> dropSuccess recap-capture.log
|
||||
$ must-mirror test.log.check target/drop.log
|
||||
|
||||
# Test success
|
||||
> captureLog recap-capture2.log
|
||||
> test --test_summary=success
|
||||
> dropSuccess recap-capture2.log
|
||||
$ must-mirror success.log.check target/drop.log
|
||||
|
||||
# Test failures
|
||||
$ copy-file changes/BadTestA.java a/src/test/java/com/example/a/TestA.java
|
||||
$ copy-file changes/BadTestB.java b/src/test/java/com/example/b/TestB.java
|
||||
> captureLog recap-capture3.log
|
||||
-> test
|
||||
> dropSuccess recap-capture3.log
|
||||
$ must-mirror failure.log.check target/drop.log
|
||||
> dropSuccessSorted testquick-capture.log
|
||||
$ must-mirror testquick2.log.check target/drop.log
|
||||
|
||||
# Test failures and success
|
||||
> captureLog recap-capture4.log
|
||||
-> test --test_summary=success
|
||||
> dropSuccess recap-capture4.log
|
||||
$ must-mirror both.log.check target/drop.log
|
||||
|
||||
# None
|
||||
> captureLog recap-capture5.log
|
||||
-> test --test_summary=none
|
||||
> dropSuccess recap-capture5.log
|
||||
$ must-mirror none.log.check target/drop.log
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
[info] passed: total 4, failed 0, errors 0, passed 4, cached 4
|
||||
[info] passed: total 4, failed 0, errors 0, passed 4, cached 4
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
passed: total 1, failed 0, errors 0, passed 1, cached 0
|
||||
passed: total 1, failed 0, errors 0, passed 1, cached 0
|
||||
passed: total 2, failed 0, errors 0, passed 2, cached 0
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
com.example.a.TestA
|
||||
com.example.b.TestB
|
||||
failed tests:
|
||||
failed tests:
|
||||
failed: total 1, failed 1, errors 0, passed 0, cached 0
|
||||
failed: total 2, failed 1, errors 0, passed 1, cached 1
|
||||
passed: total 1, failed 0, errors 0, passed 1, cached 0
|
||||
passed: total 1, failed 0, errors 0, passed 1, cached 0
|
||||
passed: total 2, failed 0, errors 0, passed 2, cached 0
|
||||
|
|
@ -176,7 +176,7 @@ class ClientTest extends AbstractServerTest with BeforeAndAfterEach {
|
|||
)
|
||||
assert(complete("testOnly") == testOnlyExpected)
|
||||
|
||||
val testOnlyOptionsExpected = Vector("--", ";", "test.pkg.FooSpec")
|
||||
val testOnlyOptionsExpected = Vector("--", "--test_summary=", ";", "test.pkg.FooSpec")
|
||||
assert(complete("testOnly ") == testOnlyOptionsExpected)
|
||||
}
|
||||
test("quote with semi") {
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ trait TestsListener extends TestReportListener {
|
|||
* a `Class` strongly references its defining class loader. Holding a `SuiteResult` with a
|
||||
* non-empty `throwables` therefore keeps the test class loader -- and every jar handle it has
|
||||
* open -- alive. That is fine for the duration of the test task, which is where these are
|
||||
* consumed, but anything that outlives the task must drop them first. `TestRecap.collect` does
|
||||
* exactly that before stashing a copy on `State.attributes`; see the note on `TestRecap.recapKey`.
|
||||
* consumed, but anything that outlives the task must drop them first. `TestSummary.append` does
|
||||
* exactly that before stashing a copy on `State.attributes`; see the note on `TestSummary.entriesKey`.
|
||||
* On Windows a leaked handle makes the underlying jar undeletable (e.g. by `clearCaches`).
|
||||
*/
|
||||
private[sbt] final class SuiteResult(
|
||||
|
|
|
|||
Loading…
Reference in New Issue