diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index c4f7468b5..7685c4093 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -149,7 +149,9 @@ object Defaults extends BuildCommon { ) ) private[sbt] lazy val globalCore: Seq[Setting[?]] = globalDefaults( - defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq( + defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks( + testSelected + ) ++ defaultTestTasks(testQuick) ++ Seq( excludeFilter :== HiddenFileFilter, fileInputs :== Nil, fileInputIncludeFilter :== AllPassFilter.toNio, @@ -1129,12 +1131,14 @@ object Defaults extends BuildCommon { testOptionDigests :== Nil, testResultLogger :== TestResultLogger.Default, testOnly / testFilter :== (IncrementalTest.selectedFilter), + testSelected / testFilter :== (IncrementalTest.selectedFilter), extraTestDigests :== Nil, ) ) lazy val testTasks: Seq[Setting[?]] = Def.settings( testTaskOptions(test), testTaskOptions(testOnly), + testTaskOptions(testSelected), testTaskOptions(testQuick), testDefaults, testLoader := Def.uncached(ClassLoaders.testTask.value), @@ -1184,8 +1188,12 @@ object Defaults extends BuildCommon { output.overall finally close(testLoader.value) }, + testSelected := { + try inputTests(testSelected).evaluated + finally close(testLoader.value) + }, testOnly := { - try inputTests(testOnly).evaluated + try inputTests(testSelected).evaluated finally close(testLoader.value) }, testQuick := { diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index e3ae4eb9d..ed7180d11 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -363,6 +363,7 @@ object Keys { val test = inputKey[TestResult]("Executes the tests that either failed before, were not run or whose transitive dependencies changed, among those provided as arguments.").withRank(ATask) val testFull = taskKey[TestResult]("Executes all tests.").withRank(APlusTask) val testOnly = inputKey[TestResult]("Executes the tests provided as arguments or all tests if no arguments are provided.").withRank(ATask) + val testSelected = inputKey[TestResult]("Executes the tests provided as arguments or all tests if no arguments are provided. Used internally by testOnly command.").withRank(BMinusTask) val testQuick = inputKey[TestResult]("Alias for test.").withRank(CTask) @transient val testOptions = taskKey[Seq[TestOption]]("Options for running tests.").withRank(BPlusTask) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index d30dc5895..fbabef52f 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -359,6 +359,7 @@ object BuiltinCommands { NetworkChannel.disconnect, waitCmd, promptChannel, + TestCommand.testOnly, ) ++ allBasicCommands ++ ContinuousCommands.value ++ diff --git a/main/src/main/scala/sbt/internal/TestCommand.scala b/main/src/main/scala/sbt/internal/TestCommand.scala new file mode 100644 index 000000000..7a927d33c --- /dev/null +++ b/main/src/main/scala/sbt/internal/TestCommand.scala @@ -0,0 +1,147 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import sbt.Keys.* +import sbt.ProjectExtra.* +import sbt.ScopeAxis.{ Select, Zero } +import sbt.SessionVar +import sbt.internal.util.complete.Parser +import sbt.librarymanagement.Configurations.{ Test as TestConfig } + +/** + * Provides commands for running tests with aggregation-aware failure semantics. + * + * The `testOnly` command fails if no tests match the patterns across ALL + * aggregated subprojects, instead of silently passing. + * See https://github.com/sbt/sbt/issues/3188 + */ +object TestCommand: + val TestOnly = "testOnly" + + private def testOnlyHelp = Help.more( + TestOnly, + """testOnly ... [-- ] + | + |Runs tests matching the given patterns. This command will fail if no + |tests match the patterns across all aggregated subprojects. + |""".stripMargin + ) + + /** + * The testOnly command that fails when no tests match across all subprojects. + * + * This command: + * 1. Parses test patterns and framework options + * 2. Determines all aggregated subprojects + * 3. Runs definedTestNames to get all available tests (this also compiles tests) + * 4. Checks if any tests match the patterns across all subprojects + * 5. Fails if no tests match, otherwise runs the testSelected task + */ + def testOnly: Command = Command(TestOnly, testOnlyHelp)(testOnlyParser)(runTestOnly) + + /** + * Parser for testOnly command arguments. + * Uses the standard testOnly parser with definedTestNames for consistent completion behavior. + */ + private def testOnlyParser(state: State): Parser[(Seq[String], Seq[String])] = + import Defaults.testOnlyParser as defaultParser + val tests = if Project.isProjectLoaded(state) then + val extracted = Project.extract(state) + val currentRef = extracted.currentRef + val scope = Scope(Select(currentRef), Select(ConfigKey(TestConfig.name)), Zero, Zero) + val scopedKey = (scope / definedTestNames).scopedKey + SessionVar.loadAndSet(scopedKey, state, false) match + case (_, Some(names)) => names.toList + case _ => Nil + else Nil + defaultParser(state, tests) + + /** + * Get all test names from all aggregated subprojects by running definedTestNames task. + */ + private def getAllTestNames(state: State): (State, Seq[String]) = + if !Project.isProjectLoaded(state) then (state, Nil) + else + val extracted = Project.extract(state) + val currentRef = extracted.currentRef + val structure = extracted.structure + + // Get all aggregated project references (including current) + val aggregatedProjects = Aggregation.projectAggregates( + Some(currentRef), + structure.extra, + reverse = false + ) :+ currentRef + + // Run definedTestNames for each project and collect all test names + var currentState = state + val allTestNames = aggregatedProjects.flatMap { projRef => + val scope = Scope(Select(projRef), Select(ConfigKey(TestConfig.name)), Zero, Zero) + val scopedKey = scope / definedTestNames + try + val (newState, testNames) = extracted.runTask(scopedKey, currentState) + currentState = newState + testNames + catch case _: Exception => Nil + }.distinct + + (currentState, allTestNames) + + private def runTestOnly(state: State, args: (Seq[String], Seq[String])): State = + val (patterns, frameworkOptions) = args + + if !Project.isProjectLoaded(state) then + state.log.error("No project is loaded.") + state.fail + else if patterns.isEmpty then + // No patterns specified, just run the testSelected task + val taskStr = + if frameworkOptions.isEmpty then "testSelected" + else s"testSelected -- ${frameworkOptions.mkString(" ")}" + taskStr :: state + else + // Separate include patterns from exclude patterns (prefixed with -) + val (excludePatterns, includePatterns) = patterns.partition(_.startsWith("-")) + + // Only check for matches if there are include patterns + // If only exclude patterns are specified, skip the check (user wants to exclude, not include) + if includePatterns.nonEmpty then + // Get all test names by running definedTestNames (this also compiles) + val (newState, allTestNames) = getAllTestNames(state) + val filters = IncrementalTest.selectedFilter(includePatterns) + val matchingTests = allTestNames.filter(name => filters.exists(f => f(name))) + + if matchingTests.isEmpty then + newState.log.error(s"No tests match the patterns: ${includePatterns.mkString(", ")}") + newState.log.error( + "The following patterns were specified but no tests were found in any subproject:" + ) + includePatterns.foreach(p => newState.log.error(s" - $p")) + newState.log.error("") + newState.log.error("Available tests:") + allTestNames.sorted.take(20).foreach(t => newState.log.error(s" - $t")) + if allTestNames.size > 20 then + newState.log.error(s" ... and ${allTestNames.size - 20} more") + newState.fail + else + // Build the testSelected task string + val testSelectedArgs = + patterns ++ (if frameworkOptions.nonEmpty then Seq("--") ++ frameworkOptions else Nil) + val taskStr = s"testSelected ${testSelectedArgs.mkString(" ")}" + taskStr :: newState + else + // Only exclude patterns - just run the task without validation + val testSelectedArgs = + patterns ++ (if frameworkOptions.nonEmpty then Seq("--") ++ frameworkOptions else Nil) + val taskStr = s"testSelected ${testSelectedArgs.mkString(" ")}" + taskStr :: state + +end TestCommand diff --git a/sbt-app/src/sbt-test/tests/filter-runners/test b/sbt-app/src/sbt-test/tests/filter-runners/test index 37b26041c..d4a100645 100644 --- a/sbt-app/src/sbt-test/tests/filter-runners/test +++ b/sbt-app/src/sbt-test/tests/filter-runners/test @@ -1,2 +1,2 @@ -> testOnly example.MunitSpec example.ScalaTestSpec -> testOnly example.MunitSpec -- "--tests=munit" +> testOnly example.Spec example.ScalaTestSpec +> testOnly example.Spec -- "--tests=munit" diff --git a/sbt-app/src/sbt-test/tests/testonly-no-match/build.sbt b/sbt-app/src/sbt-test/tests/testonly-no-match/build.sbt new file mode 100644 index 000000000..ec16d0eb6 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/testonly-no-match/build.sbt @@ -0,0 +1,17 @@ +ThisBuild / scalaVersion := "2.13.16" + +lazy val root = (project in file(".")) + .aggregate(sub1, sub2) + .settings(name := "root") + +lazy val sub1 = (project in file("sub1")) + .settings( + name := "sub1", + libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.19" % Test + ) + +lazy val sub2 = (project in file("sub2")) + .settings( + name := "sub2", + libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.19" % Test + ) diff --git a/sbt-app/src/sbt-test/tests/testonly-no-match/sub1/src/test/scala/HelloTest.scala b/sbt-app/src/sbt-test/tests/testonly-no-match/sub1/src/test/scala/HelloTest.scala new file mode 100644 index 000000000..0775a836c --- /dev/null +++ b/sbt-app/src/sbt-test/tests/testonly-no-match/sub1/src/test/scala/HelloTest.scala @@ -0,0 +1,9 @@ +package example + +import org.scalatest.flatspec.AnyFlatSpec + +class HelloTest extends AnyFlatSpec { + "Hello" should "say hello" in { + assert("hello" == "hello") + } +} diff --git a/sbt-app/src/sbt-test/tests/testonly-no-match/sub2/src/test/scala/WorldTest.scala b/sbt-app/src/sbt-test/tests/testonly-no-match/sub2/src/test/scala/WorldTest.scala new file mode 100644 index 000000000..29339aaa3 --- /dev/null +++ b/sbt-app/src/sbt-test/tests/testonly-no-match/sub2/src/test/scala/WorldTest.scala @@ -0,0 +1,9 @@ +package example + +import org.scalatest.flatspec.AnyFlatSpec + +class WorldTest extends AnyFlatSpec { + "World" should "say world" in { + assert("world" == "world") + } +} diff --git a/sbt-app/src/sbt-test/tests/testonly-no-match/test b/sbt-app/src/sbt-test/tests/testonly-no-match/test new file mode 100644 index 000000000..1e518a35d --- /dev/null +++ b/sbt-app/src/sbt-test/tests/testonly-no-match/test @@ -0,0 +1,17 @@ +# Test for issue #3188: testOnly should fail when no tests match the pattern +# https://github.com/sbt/sbt/issues/3188 + +# Test that testOnly succeeds when matching tests exist in sub1 +> testOnly example.HelloTest + +# Test that testOnly succeeds when matching tests exist in sub2 +> testOnly example.WorldTest + +# Test that testOnly fails when no tests match the pattern +-> testOnly NonExistentTest + +# Test that testOnly fails when no tests match a glob pattern +-> testOnly *NonExistent* + +# The testSelected task should still silently pass (backwards compatibility) +> testSelected NonExistentTest