From c7da2b72c33c2a791ca300610b121f5e9331e120 Mon Sep 17 00:00:00 2001 From: PandaMan Date: Fri, 13 Feb 2026 12:19:05 -0500 Subject: [PATCH] [2.x] fix: Fixes explicitlySpecified and selectors for testOnly (#8727) **Problem** When the user runs testOnly with an explicit suite name (e.g. testOnly com.example.MySuite), ScalaTest suites annotated with @DoNotDiscover were not run because sbt always passed explicitlySpecified=false to the test framework. **Solution** In Tests.processOptions, when the user has specified test filters (orderedFilters.nonEmpty), mark the filtered tests as explicitlySpecified=true with SuiteSelector so frameworks can run @DoNotDiscover suites when explicitly requested. --- main-actions/src/main/scala/sbt/Tests.scala | 23 ++++++++++++-- main/src/main/scala/sbt/Defaults.scala | 3 +- .../i5609-do-not-discover-testonly/build.sbt | 9 ++++++ .../main/scala/custom/CustomReporter.scala | 30 +++++++++++++++++++ .../src/test/scala/com/test/TestSpec.scala | 12 ++++++++ .../src/test/scala/com/test/TestSpec2.scala | 13 ++++++++ .../tests/i5609-do-not-discover-testonly/test | 25 ++++++++++++++++ 7 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/build.sbt create mode 100644 sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/main/scala/custom/CustomReporter.scala create mode 100644 sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec.scala create mode 100644 sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec2.scala create mode 100644 sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/test diff --git a/main-actions/src/main/scala/sbt/Tests.scala b/main-actions/src/main/scala/sbt/Tests.scala index 61dda1875..729107d20 100644 --- a/main-actions/src/main/scala/sbt/Tests.scala +++ b/main-actions/src/main/scala/sbt/Tests.scala @@ -133,6 +133,9 @@ object Tests { /** Test execution will be ordered by the position of the matching filter. */ final case class Filters(filterTest: Seq[String => Boolean]) extends TestOption + /** Names explicitly requested (e.g. testOnly com.example.MySuite). Used to set explicitlySpecified on TaskDef. */ + final case class ExplicitlyRequestedNames(names: Seq[String]) extends TestOption + /** Defines a TestOption that passes arguments `args` to all test frameworks. */ def Argument(args: String*): Argument = Argument(None, args.toList) @@ -247,10 +250,14 @@ object Tests { val testFilters = new ListBuffer[String => Boolean] var orderedFilters = Seq[String => Boolean]() val excludeTestsSet = new HashSet[String] + var explicitlyRequestedNames = Set.empty[String] val setup, cleanup = new ListBuffer[ClassLoader => Unit] val testListeners = new ListBuffer[TestReportListener] val undefinedFrameworks = new ListBuffer[String] + def isExplicitFqn(s: String): Boolean = + !s.contains('*') && !s.contains('?') && !s.contains("...") + for (option <- config.options) { option match { case Filter(include) => testFilters += include; () @@ -258,6 +265,9 @@ object Tests { if (orderedFilters.nonEmpty) sys.error("Cannot define multiple ordered test filters.") else orderedFilters = includes () + case ExplicitlyRequestedNames(names) => + explicitlyRequestedNames = names.filter(isExplicitFqn).toSet + () case Exclude(exclude) => excludeTestsSet ++= exclude; () case Listeners(listeners) => testListeners ++= listeners; () case Setup(setupFunction, _) => setup += setupFunction; () @@ -281,8 +291,18 @@ object Tests { if (orderedFilters.isEmpty) filtered0 else orderedFilters.flatMap(f => filtered0.filter(d => f(d.name))).toList.distinct val uniqueTests = distinctBy(tests)(_.name) + // Per TaskDef: explicitlySpecified=true only when user supplied a complete FQN (e.g. testOnly com.example.MySuite), + // not for patterns (testOnly *Spec) or plain "test". So only mark when test.name is in explicitlyRequestedNames. + val testsToUse = uniqueTests.map(t => + new TestDefinition( + t.name, + t.fingerprint, + explicitlySpecified = explicitlyRequestedNames.contains(t.name), + t.selectors + ) + ) new ProcessedOptions( - uniqueTests.toVector, + testsToUse.toVector, setup.toVector, cleanup.toVector, testListeners.toVector @@ -555,7 +575,6 @@ object Tests { c.topLevel case _ => false }) - // TODO: To pass in correct explicitlySpecified and selectors val tests = for { (df, di) <- discovered diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 9cabf774a..ec048292d 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1354,7 +1354,8 @@ object Defaults extends BuildCommon { val st = state.value given display: Show[ScopedKey[?]] = Project.showContextKey(st) val modifiedOpts = - Tests.Filters(filter(selected)) +: Tests.Argument(frameworkOptions*) +: config.options + Tests.ExplicitlyRequestedNames(selected) +: Tests.Filters(filter(selected)) +: + Tests.Argument(frameworkOptions*) +: config.options val newConfig = config.copy(options = modifiedOpts) val output = allTestGroupsTask( s, diff --git a/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/build.sbt b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/build.sbt new file mode 100644 index 000000000..8cd9690fa --- /dev/null +++ b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/build.sbt @@ -0,0 +1,9 @@ +val scalatest = "org.scalatest" %% "scalatest" % "3.0.5" + +ThisBuild / scalaVersion := "2.12.21" + +lazy val root = (project in file(".")) + .settings( + libraryDependencies += scalatest, + Test / testOptions += Tests.Argument("-C", "custom.CustomReporter") + ) diff --git a/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/main/scala/custom/CustomReporter.scala b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/main/scala/custom/CustomReporter.scala new file mode 100644 index 000000000..d3f20a21b --- /dev/null +++ b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/main/scala/custom/CustomReporter.scala @@ -0,0 +1,30 @@ +package custom + +import java.io._ +import org.scalatest._ +import events._ + +class CustomReporter extends Reporter { + + private def writeFile(filePath: String, content: String): Unit = { + val file = new File(filePath) + val writer = + if (!file.exists) + new FileWriter(new File(filePath)) + else + new FileWriter(new File(filePath + "-2")) + writer.write(content) + writer.flush() + writer.close() + } + + def apply(event: Event): Unit = { + event match { + case SuiteStarting(_, suiteName, _, _, _, _, _, _, _, _) => writeFile("target/SuiteStarting-" + suiteName, suiteName) + case SuiteCompleted(_, suiteName, _, _, _, _, _, _, _, _, _) => writeFile("target/SuiteCompleted-" + suiteName, suiteName) + case TestStarting(_, _, _, _, testName, _, _, _, _, _, _, _) => writeFile("target/TestStarting-" + testName, testName) + case TestSucceeded(_, _, _, _, testName, _, _, _, _, _, _, _, _, _) => writeFile("target/TestSucceeded-" + testName, testName) + case _ => + } + } +} diff --git a/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec.scala b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec.scala new file mode 100644 index 000000000..05c155b63 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec.scala @@ -0,0 +1,12 @@ +package com.test + +import org.scalatest.Spec + +class TestSpec extends Spec { + + def `TestSpec-test-1 ` {} + + def `TestSpec-test-2 ` {} + + def `TestSpec-test-3 ` {} +} diff --git a/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec2.scala b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec2.scala new file mode 100644 index 000000000..1ff28dffc --- /dev/null +++ b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/src/test/scala/com/test/TestSpec2.scala @@ -0,0 +1,13 @@ +package com.test + +import org.scalatest._ + +@DoNotDiscover +class TestSpec2 extends Spec { + + def `TestSpec2-test-1 ` {} + + def `TestSpec2-test-2 ` {} + + def `TestSpec2-test-3 ` {} +} diff --git a/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/test b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/test new file mode 100644 index 000000000..144967b96 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/i5609-do-not-discover-testonly/test @@ -0,0 +1,25 @@ +# #5609: When explicitly requested via testOnly, @DoNotDiscover suite should run. +# First: full test run must exclude @DoNotDiscover (TestSpec2). +# Second: testOnly with explicit FQN must run TestSpec2. + +> clean +> testFull +$ exists target/SuiteStarting-TestSpec +$ exists target/SuiteCompleted-TestSpec +$ absent target/SuiteStarting-TestSpec2 +$ absent target/SuiteCompleted-TestSpec2 + +> clean +> testOnly com.test.TestSpec2 +$ exists target/SuiteStarting-TestSpec2 +$ exists target/SuiteCompleted-TestSpec2 + +$ delete target/SuiteStarting-TestSpec +$ delete target/SuiteCompleted-TestSpec +$ delete target/SuiteStarting-TestSpec2 +$ delete target/SuiteCompleted-TestSpec2 +> testOnly com.test... +$ exists target/SuiteStarting-TestSpec +$ exists target/SuiteCompleted-TestSpec +$ absent target/SuiteStarting-TestSpec2 +$ absent target/SuiteCompleted-TestSpec2