[2.x] feat: testOnly as a command (#8607)

Adds `testOnly` command that fails when no tests match the specified patterns across all aggregated subprojects.

main/src/main/scala/sbt/Defaults.scala

@@ -149,7 +149,9 @@ object Defaults extends BuildCommon {
       )
     )
   private[sbt] lazy val globalCore: Seq[Setting[?]] = globalDefaults(
-    defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
+    defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(
+      testSelected
+    ) ++ defaultTestTasks(testQuick) ++ Seq(
       excludeFilter :== HiddenFileFilter,
       fileInputs :== Nil,
       fileInputIncludeFilter :== AllPassFilter.toNio,
@@ -1129,12 +1131,14 @@ object Defaults extends BuildCommon {
         testOptionDigests :== Nil,
         testResultLogger :== TestResultLogger.Default,
         testOnly / testFilter :== (IncrementalTest.selectedFilter),
+        testSelected / testFilter :== (IncrementalTest.selectedFilter),
         extraTestDigests :== Nil,
       )
     )
   lazy val testTasks: Seq[Setting[?]] = Def.settings(
     testTaskOptions(test),
     testTaskOptions(testOnly),
+    testTaskOptions(testSelected),
     testTaskOptions(testQuick),
     testDefaults,
     testLoader := Def.uncached(ClassLoaders.testTask.value),
@@ -1184,8 +1188,12 @@ object Defaults extends BuildCommon {
         output.overall
       finally close(testLoader.value)
     },
+    testSelected := {
+      try inputTests(testSelected).evaluated
+      finally close(testLoader.value)
+    },
     testOnly := {
-      try inputTests(testOnly).evaluated
+      try inputTests(testSelected).evaluated
       finally close(testLoader.value)
     },
     testQuick := {

main/src/main/scala/sbt/Keys.scala

@@ -363,6 +363,7 @@ object Keys {
   val test = inputKey[TestResult]("Executes the tests that either failed before, were not run or whose transitive dependencies changed, among those provided as arguments.").withRank(ATask)
   val testFull = taskKey[TestResult]("Executes all tests.").withRank(APlusTask)
   val testOnly = inputKey[TestResult]("Executes the tests provided as arguments or all tests if no arguments are provided.").withRank(ATask)
+  val testSelected = inputKey[TestResult]("Executes the tests provided as arguments or all tests if no arguments are provided. Used internally by testOnly command.").withRank(BMinusTask)
   val testQuick = inputKey[TestResult]("Alias for test.").withRank(CTask)
   @transient
   val testOptions = taskKey[Seq[TestOption]]("Options for running tests.").withRank(BPlusTask)

main/src/main/scala/sbt/Main.scala

@@ -359,6 +359,7 @@ object BuiltinCommands {
       NetworkChannel.disconnect,
       waitCmd,
       promptChannel,
+      TestCommand.testOnly,
     ) ++
       allBasicCommands ++
       ContinuousCommands.value ++

main/src/main/scala/sbt/internal/TestCommand.scala

@@ -0,0 +1,147 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package internal
+
+import sbt.Keys.*
+import sbt.ProjectExtra.*
+import sbt.ScopeAxis.{ Select, Zero }
+import sbt.SessionVar
+import sbt.internal.util.complete.Parser
+import sbt.librarymanagement.Configurations.{ Test as TestConfig }
+
+/**
+ * Provides commands for running tests with aggregation-aware failure semantics.
+ *
+ * The `testOnly` command fails if no tests match the patterns across ALL
+ * aggregated subprojects, instead of silently passing.
+ * See https://github.com/sbt/sbt/issues/3188
+ */
+object TestCommand:
+  val TestOnly = "testOnly"
+
+  private def testOnlyHelp = Help.more(
+    TestOnly,
+    """testOnly <test-pattern>... [-- <framework-options>]
+      |
+      |Runs tests matching the given patterns. This command will fail if no
+      |tests match the patterns across all aggregated subprojects.
+      |""".stripMargin
+  )
+
+  /**
+   * The testOnly command that fails when no tests match across all subprojects.
+   *
+   * This command:
+   * 1. Parses test patterns and framework options
+   * 2. Determines all aggregated subprojects
+   * 3. Runs definedTestNames to get all available tests (this also compiles tests)
+   * 4. Checks if any tests match the patterns across all subprojects
+   * 5. Fails if no tests match, otherwise runs the testSelected task
+   */
+  def testOnly: Command = Command(TestOnly, testOnlyHelp)(testOnlyParser)(runTestOnly)
+
+  /**
+   * Parser for testOnly command arguments.
+   * Uses the standard testOnly parser with definedTestNames for consistent completion behavior.
+   */
+  private def testOnlyParser(state: State): Parser[(Seq[String], Seq[String])] =
+    import Defaults.testOnlyParser as defaultParser
+    val tests = if Project.isProjectLoaded(state) then
+      val extracted = Project.extract(state)
+      val currentRef = extracted.currentRef
+      val scope = Scope(Select(currentRef), Select(ConfigKey(TestConfig.name)), Zero, Zero)
+      val scopedKey = (scope / definedTestNames).scopedKey
+      SessionVar.loadAndSet(scopedKey, state, false) match
+        case (_, Some(names)) => names.toList
+        case _                => Nil
+    else Nil
+    defaultParser(state, tests)
+
+  /**
+   * Get all test names from all aggregated subprojects by running definedTestNames task.
+   */
+  private def getAllTestNames(state: State): (State, Seq[String]) =
+    if !Project.isProjectLoaded(state) then (state, Nil)
+    else
+      val extracted = Project.extract(state)
+      val currentRef = extracted.currentRef
+      val structure = extracted.structure
+
+      // Get all aggregated project references (including current)
+      val aggregatedProjects = Aggregation.projectAggregates(
+        Some(currentRef),
+        structure.extra,
+        reverse = false
+      ) :+ currentRef
+
+      // Run definedTestNames for each project and collect all test names
+      var currentState = state
+      val allTestNames = aggregatedProjects.flatMap { projRef =>
+        val scope = Scope(Select(projRef), Select(ConfigKey(TestConfig.name)), Zero, Zero)
+        val scopedKey = scope / definedTestNames
+        try
+          val (newState, testNames) = extracted.runTask(scopedKey, currentState)
+          currentState = newState
+          testNames
+        catch case _: Exception => Nil
+      }.distinct
+
+      (currentState, allTestNames)
+
+  private def runTestOnly(state: State, args: (Seq[String], Seq[String])): State =
+    val (patterns, frameworkOptions) = args
+
+    if !Project.isProjectLoaded(state) then
+      state.log.error("No project is loaded.")
+      state.fail
+    else if patterns.isEmpty then
+      // No patterns specified, just run the testSelected task
+      val taskStr =
+        if frameworkOptions.isEmpty then "testSelected"
+        else s"testSelected -- ${frameworkOptions.mkString(" ")}"
+      taskStr :: state
+    else
+      // Separate include patterns from exclude patterns (prefixed with -)
+      val (excludePatterns, includePatterns) = patterns.partition(_.startsWith("-"))
+
+      // Only check for matches if there are include patterns
+      // If only exclude patterns are specified, skip the check (user wants to exclude, not include)
+      if includePatterns.nonEmpty then
+        // Get all test names by running definedTestNames (this also compiles)
+        val (newState, allTestNames) = getAllTestNames(state)
+        val filters = IncrementalTest.selectedFilter(includePatterns)
+        val matchingTests = allTestNames.filter(name => filters.exists(f => f(name)))
+
+        if matchingTests.isEmpty then
+          newState.log.error(s"No tests match the patterns: ${includePatterns.mkString(", ")}")
+          newState.log.error(
+            "The following patterns were specified but no tests were found in any subproject:"
+          )
+          includePatterns.foreach(p => newState.log.error(s"  - $p"))
+          newState.log.error("")
+          newState.log.error("Available tests:")
+          allTestNames.sorted.take(20).foreach(t => newState.log.error(s"  - $t"))
+          if allTestNames.size > 20 then
+            newState.log.error(s"  ... and ${allTestNames.size - 20} more")
+          newState.fail
+        else
+          // Build the testSelected task string
+          val testSelectedArgs =
+            patterns ++ (if frameworkOptions.nonEmpty then Seq("--") ++ frameworkOptions else Nil)
+          val taskStr = s"testSelected ${testSelectedArgs.mkString(" ")}"
+          taskStr :: newState
+      else
+        // Only exclude patterns - just run the task without validation
+        val testSelectedArgs =
+          patterns ++ (if frameworkOptions.nonEmpty then Seq("--") ++ frameworkOptions else Nil)
+        val taskStr = s"testSelected ${testSelectedArgs.mkString(" ")}"
+        taskStr :: state
+
+end TestCommand

sbt-app/src/sbt-test/tests/filter-runners/test

@@ -1,2 +1,2 @@
-> testOnly example.MunitSpec example.ScalaTestSpec
-> testOnly example.MunitSpec -- "--tests=munit"
+> testOnly example.Spec example.ScalaTestSpec
+> testOnly example.Spec -- "--tests=munit"

sbt-app/src/sbt-test/tests/testonly-no-match/build.sbt

@@ -0,0 +1,17 @@
+ThisBuild / scalaVersion := "2.13.16"
+
+lazy val root = (project in file("."))
+  .aggregate(sub1, sub2)
+  .settings(name := "root")
+
+lazy val sub1 = (project in file("sub1"))
+  .settings(
+    name := "sub1",
+    libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.19" % Test
+  )
+
+lazy val sub2 = (project in file("sub2"))
+  .settings(
+    name := "sub2",
+    libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.19" % Test
+  )

sbt-app/src/sbt-test/tests/testonly-no-match/sub1/src/test/scala/HelloTest.scala

@@ -0,0 +1,9 @@
+package example
+
+import org.scalatest.flatspec.AnyFlatSpec
+
+class HelloTest extends AnyFlatSpec {
+  "Hello" should "say hello" in {
+    assert("hello" == "hello")
+  }
+}

sbt-app/src/sbt-test/tests/testonly-no-match/sub2/src/test/scala/WorldTest.scala

@@ -0,0 +1,9 @@
+package example
+
+import org.scalatest.flatspec.AnyFlatSpec
+
+class WorldTest extends AnyFlatSpec {
+  "World" should "say world" in {
+    assert("world" == "world")
+  }
+}

sbt-app/src/sbt-test/tests/testonly-no-match/test

@@ -0,0 +1,17 @@
+# Test for issue #3188: testOnly should fail when no tests match the pattern
+# https://github.com/sbt/sbt/issues/3188
+
+# Test that testOnly succeeds when matching tests exist in sub1
+> testOnly example.HelloTest
+
+# Test that testOnly succeeds when matching tests exist in sub2
+> testOnly example.WorldTest
+
+# Test that testOnly fails when no tests match the pattern
+-> testOnly NonExistentTest
+
+# Test that testOnly fails when no tests match a glob pattern
+-> testOnly *NonExistent*
+
+# The testSelected task should still silently pass (backwards compatibility)
+> testSelected NonExistentTest