From bfd744b2c73fa1a2e67386a694d8bde4e02e2684 Mon Sep 17 00:00:00 2001 From: Dmitrii Naumenko Date: Mon, 24 Aug 2026 19:04:28 +0200 Subject: [PATCH] [2.0.x] fix: Notify listeners before LinkageError escapes suites (#9667) TestRunner now calls endGroup(name, error) before rethrowing an otherwise escaping LinkageError. This completes the TestReportListener lifecycle while preserving the original error propagation and non-zero task result. Previously, the outer suite catch handled only NonFatal errors. A suite failing with ExceptionInInitializerError could therefore start a listener group and terminate without a terminal callback, forcing integrations to infer failure from rendered output. The direct TestRunner regression checks the error callback, absence of a normal result callback, and rethrow of the same error. Co-authored-by: Codex --- .../build.sbt | 59 ++++++++++ .../scala/example/LinkageErrorSuites.scala | 36 ++++++ .../test-report-listener-linkage-error/test | 17 +++ .../src/main/scala/sbt/TestFramework.scala | 3 + .../src/test/scala/sbt/TestRunnerSpec.scala | 111 ++++++++++++++++++ 5 files changed, 226 insertions(+) create mode 100644 sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/build.sbt create mode 100644 sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/src/test/scala/example/LinkageErrorSuites.scala create mode 100644 sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/test create mode 100644 testing/src/test/scala/sbt/TestRunnerSpec.scala diff --git a/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/build.sbt b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/build.sbt new file mode 100644 index 000000000..09957bd5f --- /dev/null +++ b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/build.sbt @@ -0,0 +1,59 @@ +import java.io.File + +import sbt.io.IO + +val implicitSuite = "example.ImplicitInitializationFailure" +val explicitSuite = "example.ExplicitInitializationFailure" + +ThisBuild / scalaVersion := "2.12.21" + +libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.20" % Test + +/** Returns the callback log file for one suite. */ +def callbackFile(base: File, suite: String): File = base / s"$suite.log" + +/** Renders a throwable callback as the stable form asserted by this fixture. */ +def throwableCallbackLine(name: String, throwable: Throwable): String = { + val causeClass = Option(throwable.getCause).fold("")(_.getClass.getName) + s"throwable|$name|${throwable.getClass.getName}|$causeClass" +} + +/** Verifies that a suite produced exactly one expected terminal callback. */ +def checkCallback(base: File, suite: String, expected: String): Unit = { + val file = callbackFile(base, suite) + val actual = if (file.exists) IO.readLines(file) else Nil + if (actual != List(expected)) { + sys.error(s"Expected only callback '$expected', but observed: ${actual.mkString(", ")}") + } +} + +// Use one listener so the fixture records only the terminal lifecycle callback under test. +Test / testListeners := { + val base = baseDirectory.value + Seq(new TestReportListener { + def startGroup(name: String): Unit = () + def testEvent(event: TestEvent): Unit = () + def endGroup(name: String, throwable: Throwable): Unit = { + IO.append(callbackFile(base, name), throwableCallbackLine(name, throwable) + "\n") + } + def endGroup(name: String, result: TestResult): Unit = { + IO.append(callbackFile(base, name), s"result|$name|$result\n") + } + }) +} + +val checkImplicitCallback = taskKey[Unit]("Check the implicit initializer error callback") +val checkExplicitCallback = taskKey[Unit]("Check the explicit initializer error callback") + +checkImplicitCallback := + checkCallback( + baseDirectory.value, + implicitSuite, + s"throwable|$implicitSuite|java.lang.ExceptionInInitializerError|java.util.NoSuchElementException" + ) +checkExplicitCallback := + checkCallback( + baseDirectory.value, + explicitSuite, + s"throwable|$explicitSuite|java.lang.ExceptionInInitializerError|java.lang.IllegalStateException" + ) diff --git a/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/src/test/scala/example/LinkageErrorSuites.scala b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/src/test/scala/example/LinkageErrorSuites.scala new file mode 100644 index 000000000..52c4d1349 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/src/test/scala/example/LinkageErrorSuites.scala @@ -0,0 +1,36 @@ +package example + +import java.util.ServiceLoader + +import org.scalatest.funsuite.AnyFunSuite + +trait RequiredCredentialProvider { + def accessToken: String +} + +object RequiredCredentialProvider { + private val provider: RequiredCredentialProvider = + ServiceLoader.load(classOf[RequiredCredentialProvider]).iterator().next() + + def current: RequiredCredentialProvider = provider +} + +/** + * The JVM wraps the singleton initializer's NoSuchElementException in + * ExceptionInInitializerError before ScalaTest can construct this suite. + */ +class ImplicitInitializationFailure extends AnyFunSuite { + private val provider = RequiredCredentialProvider.current + + test("unreachable") { + fail("suite construction must fail first") + } +} + +class ExplicitInitializationFailure extends AnyFunSuite { + test("throws ExceptionInInitializerError from user test code") { + throw new ExceptionInInitializerError( + new IllegalStateException("explicitly thrown by user test code") + ) + } +} diff --git a/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/test b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/test new file mode 100644 index 000000000..26fe6ed9a --- /dev/null +++ b/sbt-app/src/sbt-test/tests/test-report-listener-linkage-error/test @@ -0,0 +1,17 @@ +# Regression for #9666. Before rethrowing a suite's LinkageError, TestRunner must notify +# testListeners with exactly one endGroup(name, throwable) callback. `->` keeps the expected +# test command failure visible while the following check verifies the listener contract. + +# This complements testing/src/test/scala/sbt/TestRunnerSpec.scala: that unit test isolates +# TestRunner, while this fixture verifies a listener configured through the public testListeners +# build setting. + +# ScalaTest cannot construct this suite because the JVM wraps the singleton initializer's +# NoSuchElementException in ExceptionInInitializerError. The callback check asserts both types. +-> testOnly example.ImplicitInitializationFailure +> checkImplicitCallback + +# ScalaTest also lets an ExceptionInInitializerError explicitly thrown by test code escape. +# SBT must therefore close this started group with the throwable as well. +-> testOnly example.ExplicitInitializationFailure +> checkExplicitCallback diff --git a/testing/src/main/scala/sbt/TestFramework.scala b/testing/src/main/scala/sbt/TestFramework.scala index 364fbe880..84f6e5b65 100644 --- a/testing/src/main/scala/sbt/TestFramework.scala +++ b/testing/src/main/scala/sbt/TestFramework.scala @@ -171,6 +171,9 @@ private[sbt] final class TestRunner( safeListenersCall(_.endGroup(name, suiteResult.result)) (suiteResult, nestedTasks) } catch { + case e: LinkageError => + safeListenersCall(_.endGroup(name, e)) + throw e case NonFatal(e) => safeListenersCall(_.endGroup(name, e)) (SuiteResult.Error, Seq.empty[TestTask]) diff --git a/testing/src/test/scala/sbt/TestRunnerSpec.scala b/testing/src/test/scala/sbt/TestRunnerSpec.scala new file mode 100644 index 000000000..741a67add --- /dev/null +++ b/testing/src/test/scala/sbt/TestRunnerSpec.scala @@ -0,0 +1,111 @@ +/* + * sbt + * Copyright 2026, Scala center + * Copyright 2008 - 2025, Lightbend, Inc. + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import scala.collection.mutable.ArrayBuffer + +import sbt.protocol.testing.TestResult +import sbt.util.LoggerContext +import testing.{ + AnnotatedFingerprint, + EventHandler, + Logger as TestLogger, + Runner, + SuiteSelector, + Task as TestTask, + TaskDef +} +import verify.BasicTestSuite + +object TestRunnerSpec extends BasicTestSuite { + // The direct counterpart to sbt-app/src/sbt-test/tests/test-report-listener-linkage-error. + // This test isolates TestRunner's catch path; the scripted fixture additionally verifies the + // public testListeners setting with both implicitly created and explicitly thrown errors. + test("TestRunner should end the group and rethrow linkage errors") { + val suiteName = "example.InitializationFailure" + val linkageError = new ExceptionInInitializerError("suite initialization failed") + val listener = new RecordingListener + val testTaskDef = taskDef(suiteName) + + val thrown = withTestRunner(listener) { runner => + try { + runner.run(testTaskDef, new ThrowingTask(testTaskDef, linkageError)) + None + } catch { + case error: ExceptionInInitializerError => Some(error) + } + } + + assert(thrown.contains(linkageError), s"expected $linkageError to be rethrown, got $thrown") + assert( + listener.startedGroups.toSeq == Seq(suiteName), + s"unexpected started groups: ${listener.startedGroups}" + ) + assert( + listener.errorEnds.toSeq == Seq(suiteName -> linkageError), + s"unexpected error ends: ${listener.errorEnds}" + ) + assert( + listener.resultEnds.isEmpty, + s"expected no result-based group end, got ${listener.resultEnds}" + ) + } + + private final class RecordingListener extends TestReportListener { + val startedGroups: ArrayBuffer[String] = ArrayBuffer.empty + val errorEnds: ArrayBuffer[(String, Throwable)] = ArrayBuffer.empty + val resultEnds: ArrayBuffer[(String, TestResult)] = ArrayBuffer.empty + + override def startGroup(name: String): Unit = startedGroups += name + override def testEvent(event: TestEvent): Unit = () + override def endGroup(name: String, error: Throwable): Unit = errorEnds += name -> error + override def endGroup(name: String, result: TestResult): Unit = resultEnds += name -> result + } + + private final class ThrowingTask(testTaskDef: TaskDef, error: LinkageError) extends TestTask { + override def tags(): Array[String] = Array.empty + override def taskDef(): TaskDef = testTaskDef + override def execute(handler: EventHandler, loggers: Array[TestLogger]): Array[TestTask] = { + throw error + } + } + + private object NoOpRunner extends Runner { + override def tasks(taskDefs: Array[TaskDef]): Array[TestTask] = Array.empty + override def done(): String = "" + override def remoteArgs(): Array[String] = Array.empty + override def args(): Array[String] = Array.empty + } + + private def taskDef(name: String): TaskDef = { + new TaskDef( + name, + new AnnotatedFingerprint { + override def isModule(): Boolean = false + override def annotationName(): String = "example.Test" + }, + false, + Array(new SuiteSelector) + ) + } + + private def withTestRunner[T](listener: TestReportListener)(f: TestRunner => T): T = { + val loggerContext = LoggerContext() + try { + f( + new TestRunner( + NoOpRunner, + Vector(listener), + loggerContext.logger("TestRunnerSpec", None, None) + ) + ) + } finally { + loggerContext.close() + } + } +}