[2.x] fix: Fixes explicitlySpecified and selectors for testOnly (#8727)

**Problem**
When the user runs testOnly with an explicit suite name (e.g. testOnly com.example.MySuite),
ScalaTest suites annotated with @DoNotDiscover were not run because sbt always passed
explicitlySpecified=false to the test framework.

**Solution**
In Tests.processOptions, when the user has specified test filters (orderedFilters.nonEmpty),
mark the filtered tests as explicitlySpecified=true with SuiteSelector so frameworks
can run @DoNotDiscover suites when explicitly requested.
This commit is contained in:
PandaMan 2026-02-13 12:19:05 -05:00 committed by GitHub
parent 9ca4f186f1
commit c7da2b72c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 112 additions and 3 deletions

View File

@ -133,6 +133,9 @@ object Tests {
/** Test execution will be ordered by the position of the matching filter. */
final case class Filters(filterTest: Seq[String => Boolean]) extends TestOption
/** Names explicitly requested (e.g. testOnly com.example.MySuite). Used to set explicitlySpecified on TaskDef. */
final case class ExplicitlyRequestedNames(names: Seq[String]) extends TestOption
/** Defines a TestOption that passes arguments `args` to all test frameworks. */
def Argument(args: String*): Argument = Argument(None, args.toList)
@ -247,10 +250,14 @@ object Tests {
val testFilters = new ListBuffer[String => Boolean]
var orderedFilters = Seq[String => Boolean]()
val excludeTestsSet = new HashSet[String]
var explicitlyRequestedNames = Set.empty[String]
val setup, cleanup = new ListBuffer[ClassLoader => Unit]
val testListeners = new ListBuffer[TestReportListener]
val undefinedFrameworks = new ListBuffer[String]
def isExplicitFqn(s: String): Boolean =
!s.contains('*') && !s.contains('?') && !s.contains("...")
for (option <- config.options) {
option match {
case Filter(include) => testFilters += include; ()
@ -258,6 +265,9 @@ object Tests {
if (orderedFilters.nonEmpty) sys.error("Cannot define multiple ordered test filters.")
else orderedFilters = includes
()
case ExplicitlyRequestedNames(names) =>
explicitlyRequestedNames = names.filter(isExplicitFqn).toSet
()
case Exclude(exclude) => excludeTestsSet ++= exclude; ()
case Listeners(listeners) => testListeners ++= listeners; ()
case Setup(setupFunction, _) => setup += setupFunction; ()
@ -281,8 +291,18 @@ object Tests {
if (orderedFilters.isEmpty) filtered0
else orderedFilters.flatMap(f => filtered0.filter(d => f(d.name))).toList.distinct
val uniqueTests = distinctBy(tests)(_.name)
// Per TaskDef: explicitlySpecified=true only when user supplied a complete FQN (e.g. testOnly com.example.MySuite),
// not for patterns (testOnly *Spec) or plain "test". So only mark when test.name is in explicitlyRequestedNames.
val testsToUse = uniqueTests.map(t =>
new TestDefinition(
t.name,
t.fingerprint,
explicitlySpecified = explicitlyRequestedNames.contains(t.name),
t.selectors
)
)
new ProcessedOptions(
uniqueTests.toVector,
testsToUse.toVector,
setup.toVector,
cleanup.toVector,
testListeners.toVector
@ -555,7 +575,6 @@ object Tests {
c.topLevel
case _ => false
})
// TODO: To pass in correct explicitlySpecified and selectors
val tests =
for {
(df, di) <- discovered

View File

@ -1354,7 +1354,8 @@ object Defaults extends BuildCommon {
val st = state.value
given display: Show[ScopedKey[?]] = Project.showContextKey(st)
val modifiedOpts =
Tests.Filters(filter(selected)) +: Tests.Argument(frameworkOptions*) +: config.options
Tests.ExplicitlyRequestedNames(selected) +: Tests.Filters(filter(selected)) +:
Tests.Argument(frameworkOptions*) +: config.options
val newConfig = config.copy(options = modifiedOpts)
val output = allTestGroupsTask(
s,

View File

@ -0,0 +1,9 @@
val scalatest = "org.scalatest" %% "scalatest" % "3.0.5"
ThisBuild / scalaVersion := "2.12.21"
lazy val root = (project in file("."))
.settings(
libraryDependencies += scalatest,
Test / testOptions += Tests.Argument("-C", "custom.CustomReporter")
)

View File

@ -0,0 +1,30 @@
package custom
import java.io._
import org.scalatest._
import events._
class CustomReporter extends Reporter {
private def writeFile(filePath: String, content: String): Unit = {
val file = new File(filePath)
val writer =
if (!file.exists)
new FileWriter(new File(filePath))
else
new FileWriter(new File(filePath + "-2"))
writer.write(content)
writer.flush()
writer.close()
}
def apply(event: Event): Unit = {
event match {
case SuiteStarting(_, suiteName, _, _, _, _, _, _, _, _) => writeFile("target/SuiteStarting-" + suiteName, suiteName)
case SuiteCompleted(_, suiteName, _, _, _, _, _, _, _, _, _) => writeFile("target/SuiteCompleted-" + suiteName, suiteName)
case TestStarting(_, _, _, _, testName, _, _, _, _, _, _, _) => writeFile("target/TestStarting-" + testName, testName)
case TestSucceeded(_, _, _, _, testName, _, _, _, _, _, _, _, _, _) => writeFile("target/TestSucceeded-" + testName, testName)
case _ =>
}
}
}

View File

@ -0,0 +1,12 @@
package com.test
import org.scalatest.Spec
class TestSpec extends Spec {
def `TestSpec-test-1 ` {}
def `TestSpec-test-2 ` {}
def `TestSpec-test-3 ` {}
}

View File

@ -0,0 +1,13 @@
package com.test
import org.scalatest._
@DoNotDiscover
class TestSpec2 extends Spec {
def `TestSpec2-test-1 ` {}
def `TestSpec2-test-2 ` {}
def `TestSpec2-test-3 ` {}
}

View File

@ -0,0 +1,25 @@
# #5609: When explicitly requested via testOnly, @DoNotDiscover suite should run.
# First: full test run must exclude @DoNotDiscover (TestSpec2).
# Second: testOnly with explicit FQN must run TestSpec2.
> clean
> testFull
$ exists target/SuiteStarting-TestSpec
$ exists target/SuiteCompleted-TestSpec
$ absent target/SuiteStarting-TestSpec2
$ absent target/SuiteCompleted-TestSpec2
> clean
> testOnly com.test.TestSpec2
$ exists target/SuiteStarting-TestSpec2
$ exists target/SuiteCompleted-TestSpec2
$ delete target/SuiteStarting-TestSpec
$ delete target/SuiteCompleted-TestSpec
$ delete target/SuiteStarting-TestSpec2
$ delete target/SuiteCompleted-TestSpec2
> testOnly com.test...
$ exists target/SuiteStarting-TestSpec
$ exists target/SuiteCompleted-TestSpec
$ absent target/SuiteStarting-TestSpec2
$ absent target/SuiteCompleted-TestSpec2