[2.x] feat: Expose `scripted / excludeFilter` and `scripted / includeFilter` keys for scripted test filtering. (#9131)

* [2.x] feat: Add scripted / includeFilter and scripted / excludeFilter

* [2.x] Add scripted-exclude-filter scripted test. Remove stale item from Scripted.sbtWindowsExcludeFilter

build.sbt

@@ -1011,7 +1011,9 @@ def scriptedTask(launch: Boolean): Def.Initialize[InputTask[Unit]] = Def.inputTa
       .filterNot(_.getName.contains("scala-compiler")),
     (bundledLauncherProj / Compile / packageBin).value,
     streams.value.log,
-    scriptedKeepTempDirectory.value
+    scriptedKeepTempDirectory.value,
+    (scripted / includeFilter).value,
+    (scripted / excludeFilter).value,
   )
 }
 
@@ -1090,6 +1092,8 @@ def otherRootSettings =
     scriptedSource := (sbtProj / sourceDirectory).value / "sbt-test",
     scripted / watchTriggers += scriptedSource.value.toGlob / **,
     scriptedUnpublished / watchTriggers := (scripted / watchTriggers).value,
+    scripted / includeFilter := AllPassFilter,
+    scripted / excludeFilter := Scripted.sbtWindowsExcludeFilter,
     scriptedLaunchOpts := List("-Xmx1500M", "-Xms512M", "-server") :::
       (sys.props.get("sbt.ivy.home") match {
         case Some(home) => List(s"-Dsbt.ivy.home=$home")

main/src/main/scala/sbt/ScriptedPlugin.scala

@@ -63,6 +63,8 @@ object ScriptedPlugin extends AutoPlugin {
 
   override lazy val projectSettings: Seq[Setting[?]] = Seq(
     ivyConfigurations ++= Seq(ScriptedConf, ScriptedLaunchConf),
+    scripted / includeFilter := AllPassFilter,
+    scripted / excludeFilter := NothingFilter,
     scriptedSbt := (pluginCrossBuild / sbtVersion).value,
     sbtLauncher := Def.uncached(
       getJars(ScriptedLaunchConf)
@@ -183,7 +185,9 @@ object ScriptedPlugin extends AutoPlugin {
         scriptedLaunchOpts.value,
         new java.util.ArrayList[File](),
         scriptedParallelInstances.value,
-        scriptedKeepTempDirectory.value
+        scriptedKeepTempDirectory.value,
+        (scripted / includeFilter).value,
+        (scripted / excludeFilter).value,
       )
     }
 

main/src/main/scala/sbt/ScriptedRun.scala

@@ -8,9 +8,11 @@
 
 package sbt
 
-import java.io.File
+import java.io.{ File, FileFilter as JFileFilter }
 import java.lang.reflect.Method
 
+import sbt.io.{ AllPassFilter, NothingFilter }
+
 sealed trait ScriptedRun {
   final def run(
       resourceBaseDirectory: File,
@@ -62,6 +64,37 @@ sealed trait ScriptedRun {
     } catch { case e: java.lang.reflect.InvocationTargetException => throw e.getCause }
   }
 
+  final def run(
+      resourceBaseDirectory: File,
+      bufferLog: Boolean,
+      tests: Seq[String],
+      launcherJar: File,
+      javaCommand: String,
+      launchOpts: Seq[String],
+      prescripted: java.util.List[File],
+      instances: Int,
+      keepTempDirectory: Boolean,
+      includeFilter: JFileFilter,
+      excludeFilter: JFileFilter,
+  ): Unit = {
+    try {
+      invoke(
+        resourceBaseDirectory,
+        bufferLog,
+        tests.toArray,
+        launcherJar,
+        javaCommand,
+        launchOpts.toArray,
+        prescripted,
+        instances,
+        keepTempDirectory,
+        includeFilter,
+        excludeFilter,
+      )
+      ()
+    } catch { case e: java.lang.reflect.InvocationTargetException => throw e.getCause }
+  }
+
   protected def invoke(
       resourceBaseDirectory: File,
       bufferLog: java.lang.Boolean,
@@ -97,6 +130,33 @@ sealed trait ScriptedRun {
       keepTempDirectory: java.lang.Boolean,
   ): AnyRef
 
+  // Default drops filters and calls V3 invoke so V1/V2/V3 subclasses need not override.
+  protected def invoke(
+      resourceBaseDirectory: File,
+      bufferLog: java.lang.Boolean,
+      tests: Array[String],
+      launcherJar: File,
+      javaCommand: String,
+      launchOpts: Array[String],
+      prescripted: java.util.List[File],
+      instances: java.lang.Integer,
+      keepTempDirectory: java.lang.Boolean,
+      includeFilter: JFileFilter,
+      excludeFilter: JFileFilter,
+  ): AnyRef = {
+    invoke(
+      resourceBaseDirectory,
+      bufferLog,
+      tests,
+      launcherJar,
+      javaCommand,
+      launchOpts,
+      prescripted,
+      instances,
+      keepTempDirectory,
+    )
+  }
+
 }
 
 object ScriptedRun {
@@ -108,48 +168,92 @@ object ScriptedRun {
     val sCls = classOf[String]
     val lfCls = classOf[java.util.List[File]]
     val iCls = classOf[Int]
+    val ffCls = classOf[JFileFilter]
 
     val clazz = scriptedTests.getClass
     if (batchExecution)
       try
-        new RunInParallelV3(
+        new RunInParallelV4(
           scriptedTests,
-          clazz.getMethod("runInParallel", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, iCls, bCls)
+          clazz.getMethod(
+            "runInParallel",
+            fCls,
+            bCls,
+            asCls,
+            fCls,
+            sCls,
+            asCls,
+            lfCls,
+            iCls,
+            bCls,
+            ffCls,
+            ffCls,
+          )
         )
       catch {
         case _: NoSuchMethodException =>
           try
-            new RunInParallelV2(
+            new RunInParallelV3(
               scriptedTests,
-              clazz.getMethod("runInParallel", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, iCls)
+              clazz
+                .getMethod("runInParallel", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, iCls, bCls)
             )
           catch {
             case _: NoSuchMethodException =>
-              new RunInParallelV1(
+              try
-                scriptedTests,
+                new RunInParallelV2(
-                clazz.getMethod("runInParallel", fCls, bCls, asCls, fCls, asCls, lfCls, iCls)
+                  scriptedTests,
-              )
+                  clazz
+                    .getMethod("runInParallel", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, iCls)
+                )
+              catch {
+                case _: NoSuchMethodException =>
+                  new RunInParallelV1(
+                    scriptedTests,
+                    clazz.getMethod("runInParallel", fCls, bCls, asCls, fCls, asCls, lfCls, iCls)
+                  )
+              }
           }
       }
     else
       try
-        new RunV3(
+        new RunV4(
           scriptedTests,
-          clazz.getMethod("run", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, bCls)
+          clazz.getMethod(
+            "run",
+            fCls,
+            bCls,
+            asCls,
+            fCls,
+            sCls,
+            asCls,
+            lfCls,
+            bCls,
+            ffCls,
+            ffCls,
+          )
         )
       catch {
         case _: NoSuchMethodException =>
           try
-            new RunV2(
+            new RunV3(
               scriptedTests,
-              clazz.getMethod("run", fCls, bCls, asCls, fCls, sCls, asCls, lfCls)
+              clazz.getMethod("run", fCls, bCls, asCls, fCls, sCls, asCls, lfCls, bCls)
             )
           catch {
             case _: NoSuchMethodException =>
-              new RunV1(
+              try
-                scriptedTests,
+                new RunV2(
-                clazz.getMethod("run", fCls, bCls, asCls, fCls, asCls, lfCls)
+                  scriptedTests,
-              )
+                  clazz.getMethod("run", fCls, bCls, asCls, fCls, sCls, asCls, lfCls)
+                )
+              catch {
+                case _: NoSuchMethodException =>
+                  new RunV1(
+                    scriptedTests,
+                    clazz.getMethod("run", fCls, bCls, asCls, fCls, asCls, lfCls)
+                  )
+              }
           }
       }
   }
@@ -301,4 +405,113 @@ object ScriptedRun {
       )
   }
 
+  private class RunV4(scriptedTests: AnyRef, run: Method) extends ScriptedRun {
+    override protected def invoke(
+        resourceBaseDirectory: File,
+        bufferLog: java.lang.Boolean,
+        tests: Array[String],
+        launcherJar: File,
+        javaCommand: String,
+        launchOpts: Array[String],
+        prescripted: java.util.List[File],
+        instances: java.lang.Integer,
+        keepTempDirectory: java.lang.Boolean,
+    ): AnyRef =
+      invoke(
+        resourceBaseDirectory,
+        bufferLog,
+        tests,
+        launcherJar,
+        javaCommand,
+        launchOpts,
+        prescripted,
+        instances,
+        keepTempDirectory,
+        AllPassFilter,
+        NothingFilter,
+      )
+
+    override protected def invoke(
+        resourceBaseDirectory: File,
+        bufferLog: java.lang.Boolean,
+        tests: Array[String],
+        launcherJar: File,
+        javaCommand: String,
+        launchOpts: Array[String],
+        prescripted: java.util.List[File],
+        instances: java.lang.Integer,
+        keepTempDirectory: java.lang.Boolean,
+        includeFilter: JFileFilter,
+        excludeFilter: JFileFilter,
+    ): AnyRef =
+      run.invoke(
+        scriptedTests,
+        resourceBaseDirectory,
+        bufferLog,
+        tests,
+        launcherJar,
+        javaCommand,
+        launchOpts,
+        prescripted,
+        keepTempDirectory,
+        includeFilter,
+        excludeFilter,
+      )
+  }
+
+  private class RunInParallelV4(scriptedTests: AnyRef, runInParallel: Method) extends ScriptedRun {
+    override protected def invoke(
+        resourceBaseDirectory: File,
+        bufferLog: java.lang.Boolean,
+        tests: Array[String],
+        launcherJar: File,
+        javaCommand: String,
+        launchOpts: Array[String],
+        prescripted: java.util.List[File],
+        instances: Integer,
+        keepTempDirectory: java.lang.Boolean,
+    ): AnyRef =
+      invoke(
+        resourceBaseDirectory,
+        bufferLog,
+        tests,
+        launcherJar,
+        javaCommand,
+        launchOpts,
+        prescripted,
+        instances,
+        keepTempDirectory,
+        AllPassFilter,
+        NothingFilter,
+      )
+
+    override protected def invoke(
+        resourceBaseDirectory: File,
+        bufferLog: java.lang.Boolean,
+        tests: Array[String],
+        launcherJar: File,
+        javaCommand: String,
+        launchOpts: Array[String],
+        prescripted: java.util.List[File],
+        instances: Integer,
+        keepTempDirectory: java.lang.Boolean,
+        includeFilter: JFileFilter,
+        excludeFilter: JFileFilter,
+    ): AnyRef =
+      runInParallel.invoke(
+        scriptedTests,
+        resourceBaseDirectory,
+        bufferLog,
+        tests,
+        launcherJar,
+        javaCommand,
+        launchOpts,
+        prescripted,
+        instances,
+        keepTempDirectory,
+        includeFilter,
+        excludeFilter,
+      )
+  }
+
 }

project/Scripted.scala

@@ -32,6 +32,20 @@ object Scripted {
 
   val RepoOverrideTest = config("repoOverrideTest") extend Compile
 
+  val sbtWindowsExcludeFilter: FileFilter =
+    if (scala.util.Properties.isWin)
+      new SimpleFileFilter(f =>
+        (f.getParentFile.getName, f.getName) match {
+          case ("classloader-cache", "jni") => true // no native lib is built for windows
+          case ("classloader-cache", "spark") =>
+            true // the test spark server is unable to bind to a local socket on Visual Studio 2019
+          case ("nio", "make-clone") => true // uses gcc which isn't set up on all systems
+          case ("watch", "symlinks") => true // symlinks don't work the same on windows
+          case _                     => false
+        }
+      )
+    else NothingFilter
+
   import sbt.complete.*
 
   // Paging, 1-index based.
@@ -106,7 +120,9 @@ object Scripted {
       classpath: Seq[File],
       launcherJar: File,
       logger: Logger,
-      keepTempDirectory: Boolean
+      keepTempDirectory: Boolean,
+      includeFilter: java.io.FileFilter,
+      excludeFilter: java.io.FileFilter,
   ): Unit = {
     logger.info(s"Tests selected: ${args.mkString("\n * ", "\n * ", "\n")}")
     logger.info("")
@@ -120,18 +136,6 @@ object Scripted {
 
     // Interface to cross class loader
     type SbtScriptedRunner = {
-      // def runInParallel(
-      //     resourceBaseDirectory: File,
-      //     bufferLog: Boolean,
-      //     tests: Array[String],
-      //     launchOpts: Array[String],
-      //     prescripted: java.util.List[File],
-      //     scalaVersion: String,
-      //     sbtVersion: String,
-      //     classpath: Array[File],
-      //     instances: Int
-      // ): Unit
-
       def runInParallel(
           resourceBaseDirectory: File,
           bufferLog: Boolean,
@@ -141,7 +145,9 @@ object Scripted {
           launchOpts: Array[String],
           prescripted: java.util.List[File],
           instance: Int,
-          keepTempDirectory: Boolean
+          keepTempDirectory: Boolean,
+          includeFilter: java.io.FileFilter,
+          excludeFilter: java.io.FileFilter,
       ): Unit
     }
 
@@ -166,17 +172,6 @@ object Scripted {
         }
         import scala.language.reflectiveCalls
 
-        // bridge.runInParallel(
-        //   sourcePath,
-        //   bufferLog,
-        //   args.toArray,
-        //   launchOpts.toArray,
-        //   callback,
-        //   scalaVersion,
-        //   sbtVersion,
-        //   classpath.toArray,
-        //   instances
-        // )
         bridge.runInParallel(
           sourcePath,
           bufferLog,
@@ -186,7 +181,9 @@ object Scripted {
           launchOpts.toArray,
           callback,
           instances,
-          keepTempDirectory
+          keepTempDirectory,
+          includeFilter,
+          excludeFilter,
         )
       } catch { case ite: InvocationTargetException => throw ite.getCause }
     } finally {

sbt-app/src/sbt-test/project/scripted-exclude-filter/build.sbt

@@ -0,0 +1,5 @@
+lazy val root = (project in file("."))
+  .enablePlugins(SbtPlugin)
+  .settings(
+    scripted / excludeFilter := new SimpleFileFilter(_.getName == "skipped")
+  )

sbt-app/src/sbt-test/project/scripted-exclude-filter/changes/broken-plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("nonexistent.example" % "sbt-nonexistent" % "0.0.0")

sbt-app/src/sbt-test/project/scripted-exclude-filter/changes/ok-test

@@ -0,0 +1 @@
+> compile

sbt-app/src/sbt-test/project/scripted-exclude-filter/test

@@ -0,0 +1,23 @@
+# `passing` is a project that compiles; `skipped` has a project/plugins.sbt referencing a
+# fictitious plugin so its sbt session fails to load if actually run.
+$ copy-file changes/ok-test src/sbt-test/group/passing/test
+$ copy-file changes/ok-test src/sbt-test/group/skipped/test
+$ copy-file changes/broken-plugins.sbt src/sbt-test/group/skipped/project/plugins.sbt
+
+> scripted
+
+# Explicit selection of the un-filtered test. Succeeds.
+> scripted group/passing
+
+# Explicit selection of a filtered test yields "No tests found matching" error.
+-> scripted group/skipped
+
+# Replace the excludeFilter to let both run. `skipped` now fails.
+> set scripted / excludeFilter := NothingFilter
+-> scripted
+-> scripted group/skipped
+
+# Flip to includeFilter: only accept tests whose name is "passing".
+> set scripted / includeFilter := new SimpleFileFilter(_.getName == "passing")
+> scripted
+-> scripted group/skipped

scripted-sbt/src/main/scala/sbt/scriptedtest/ScriptedTests.scala

@@ -102,7 +102,6 @@ final class ScriptedTests(
     Map('$' -> fileHandler, '>' -> sbtHandler, '#' -> CommentHandler)
   }
 
-  /** Returns a sequence of test runners that have to be applied in the call site. */
   def batchScriptedRunner(
       testGroupAndNames: Seq[(String, String)],
       prescripted: File => Unit,
@@ -110,6 +109,28 @@ final class ScriptedTests(
       prop: RemoteSbtCreatorProp,
       log: Logger,
       keepTempDirectory: Boolean = false,
+  ): Seq[TestRunner] =
+    batchScriptedRunner(
+      testGroupAndNames,
+      prescripted,
+      sbtInstances,
+      prop,
+      log,
+      keepTempDirectory,
+      AllPassFilter,
+      NothingFilter,
+    )
+
+  /** Returns a sequence of test runners that have to be applied in the call site. */
+  def batchScriptedRunner(
+      testGroupAndNames: Seq[(String, String)],
+      prescripted: File => Unit,
+      sbtInstances: Int,
+      prop: RemoteSbtCreatorProp,
+      log: Logger,
+      keepTempDirectory: Boolean,
+      includeFilter: java.io.FileFilter,
+      excludeFilter: java.io.FileFilter,
   ): Seq[TestRunner] = {
     // Test group and names may be file filters (like '*')
     val groupAndNameDirs = {
@@ -117,12 +138,14 @@ final class ScriptedTests(
         (group, name) <- testGroupAndNames
         groupDir <- (resourceBaseDirectory * group).get()
         testDir <- (groupDir * name).get()
+        if !testDir.isFile
+        if includeFilter.accept(testDir) && !excludeFilter.accept(testDir)
       } yield (groupDir, testDir)
     }
 
     type TestInfo = ((String, String), File)
 
-    val labelsAndDirs = groupAndNameDirs.filterNot(_._2.isFile).map { (groupDir, nameDir) =>
+    val labelsAndDirs = groupAndNameDirs.map { (groupDir, nameDir) =>
       val groupName = groupDir.getName
       val testName = nameDir.getName
       val testDirectory = testResources.readOnlyResourceDirectory(groupName, testName)
@@ -137,18 +160,15 @@ final class ScriptedTests(
         case s => s
       }
 
-      val runFromSourceBasedTestsUnfiltered = labelsAndDirs
-      val runFromSourceBasedTests = runFromSourceBasedTestsUnfiltered.filterNot(windowsExclude)
-
       def logTests(size: Int, how: String) =
         log.info(
           f"Running $size / $totalSize (${size * 100d / totalSize}%3.2f%%) scripted tests with $how"
         )
-      logTests(runFromSourceBasedTests.size, prop.toString)
+      logTests(labelsAndDirs.size, prop.toString)
 
-      if (keepTempDirectory && runFromSourceBasedTests.size > 1) {
+      if (keepTempDirectory && labelsAndDirs.size > 1) {
         sys.error(
-          s"scriptedKeepTempDirectory requires exactly one test, but ${runFromSourceBasedTests.size} tests were requested"
+          s"scriptedKeepTempDirectory requires exactly one test, but ${labelsAndDirs.size} tests were requested"
         )
       }
 
@@ -170,27 +190,10 @@ final class ScriptedTests(
           .toList
       }
 
-      createTestRunners(runFromSourceBasedTests)
+      createTestRunners(labelsAndDirs)
     }
   }
 
-  private val windowsExclude: (((String, String), File)) => Boolean =
-    if (scala.util.Properties.isWin) { case (testName, _) =>
-      testName match {
-        case ("classloader-cache", "jni") => true // no native lib is built for windows
-        case ("classloader-cache", "snapshot") =>
-          true // the test overwrites a jar that is being used which is verboten in windows
-        // The test spark server is unable to bind to a local socket on Visual Studio 2019
-        case ("classloader-cache", "spark") => true
-        case ("nio", "make-clone")          => true // uses gcc which isn't set up on all systems
-        // symlinks don't work the same on windows. Symlink monitoring does work in many cases
-        // on windows but not to the same level as it does on osx and linux
-        case ("watch", "symlinks") => true
-        case _                     => false
-      }
-    }
-    else _ => false
-
   /**
    * Defines the batch execution of scripted tests.
    *
@@ -512,6 +515,37 @@ class ScriptedRunner {
     )
   }
 
+  /** Entry point with configurable include/exclude filters. */
+  def run(
+      resourceBaseDirectory: File,
+      bufferLog: Boolean,
+      tests: Array[String],
+      launcherJar: File,
+      javaCommand: String,
+      launchOpts: Array[String],
+      prescripted: java.util.List[File],
+      keepTempDirectory: Boolean,
+      includeFilter: java.io.FileFilter,
+      excludeFilter: java.io.FileFilter,
+  ): Unit = {
+    val logger = TestConsoleLogger()
+    run(
+      resourceBaseDirectory,
+      bufferLog,
+      tests,
+      logger,
+      javaCommand,
+      launchOpts,
+      prescripted,
+      LauncherBased(launcherJar),
+      Int.MaxValue,
+      parallelExecution = false,
+      keepTempDirectory,
+      includeFilter,
+      excludeFilter,
+    )
+  }
+
   /**
    * This is the entry point used by SbtPlugin in sbt 1.2.x, 1.3.x, 1.4.x etc.
    * Removing this method will break scripted and sbt plugin cross building.
@@ -596,6 +630,37 @@ class ScriptedRunner {
     )
   }
 
+  /** Entry point with configurable include/exclude filters. */
+  def runInParallel(
+      resourceBaseDirectory: File,
+      bufferLog: Boolean,
+      tests: Array[String],
+      launcherJar: File,
+      javaCommand: String,
+      launchOpts: Array[String],
+      prescripted: java.util.List[File],
+      instance: Int,
+      keepTempDirectory: Boolean,
+      includeFilter: java.io.FileFilter,
+      excludeFilter: java.io.FileFilter,
+  ): Unit = {
+    val logger = TestConsoleLogger()
+    runInParallel(
+      resourceBaseDirectory,
+      bufferLog,
+      tests,
+      logger,
+      javaCommand,
+      launchOpts,
+      prescripted,
+      LauncherBased(launcherJar),
+      instance,
+      keepTempDirectory,
+      includeFilter,
+      excludeFilter,
+    )
+  }
+
   // This is called by project/Scripted.scala
   // Using java.util.List[File] to encode File => Unit
   def runInParallel(
@@ -633,6 +698,35 @@ class ScriptedRunner {
       prop: RemoteSbtCreatorProp,
       instances: Int,
       keepTempDirectory: Boolean = false,
+  ): Unit =
+    runInParallel(
+      baseDir,
+      bufferLog,
+      tests,
+      logger,
+      javaCommand,
+      launchOpts,
+      prescripted,
+      prop,
+      instances,
+      keepTempDirectory,
+      AllPassFilter,
+      NothingFilter,
+    )
+
+  private[sbt] def runInParallel(
+      baseDir: File,
+      bufferLog: Boolean,
+      tests: Array[String],
+      logger: Logger,
+      javaCommand: String,
+      launchOpts: Array[String],
+      prescripted: java.util.List[File],
+      prop: RemoteSbtCreatorProp,
+      instances: Int,
+      keepTempDirectory: Boolean,
+      includeFilter: java.io.FileFilter,
+      excludeFilter: java.io.FileFilter,
   ): Unit =
     run(
       baseDir,
@@ -646,6 +740,8 @@ class ScriptedRunner {
       instances,
       parallelExecution = true,
       keepTempDirectory,
+      includeFilter,
+      excludeFilter,
     )
 
   private def run(
@@ -660,6 +756,8 @@ class ScriptedRunner {
       instances: Int,
       parallelExecution: Boolean,
       keepTempDirectory: Boolean = false,
+      includeFilter: java.io.FileFilter = AllPassFilter,
+      excludeFilter: java.io.FileFilter = NothingFilter,
   ): Unit = {
     val addTestFile = (f: File) => { prescripted.add(f); () }
     val runner = new ScriptedTests(baseDir, bufferLog, javaCommand, launchOpts.toIndexedSeq)
@@ -682,7 +780,9 @@ class ScriptedRunner {
         groupCount,
         prop,
         logger,
-        keepTempDirectory
+        keepTempDirectory,
+        includeFilter,
+        excludeFilter,
       )
     // Fail if user provided test patterns but none matched any existing test directories
     if (tests.nonEmpty && scriptedRunners.isEmpty) {