Add nio path filter settings

It makes sense for the new glob/nio based apis that we provide first
class support for filtering the results. Because it isn't possible to
scope a task within a task within a task, i.e.
`compile / fileInputs / includePathFilter`, I had to add four new
filter settings of type `PathFilter`:

fileInputIncludeFilter :== AllPassFilter.toNio,
fileInputExcludeFilter :== DirectoryFilter.toNio || HiddenFileFilter,
fileOutputIncludeFilter :== AllPassFilter.toNio,
fileOutputExcludeFilter :== NothingFilter.toNio,

Before I was effectively hard-coding the filter: RegularFileFilter &&
!HiddenFileFilter in the inputFileStamps and allInputFiles tasks. These
remain the defaults, as seen in the fileInputExcludeFilter definition
above, but can be overridden by the user.

It makes sense to exclude directories and hidden files for the input
files, but it doesn't necessarily make sense to apply any output filters
by default. For symmetry, it makes sense to have them, but they are
unlikely to be used often.

Apart from adding and defining the default values for these keys, the
only other changes I had to make was to remove the hard-coded filters
from the allInputFiles and inputFileStamps tasks and also add the
filtering to the allOutputFiles task. Because we don't automatically
calculate the FileAttributes for the output files, I added logic for
bypassing the path filter application if the PathFilter is effectively
AllPass, which is the case for the default values because:
AllPassFilter.toNio == AllPass
NothingFilter.toNio == NoPass
AllPass && !NoPass == AllPass && AllPass == AllPass

main/src/main/scala/sbt/Defaults.scala

@@ -16,7 +16,6 @@ import lmcoursier.CoursierDependencyResolution
 import lmcoursier.definitions.{ Configuration => CConfiguration }
 import org.apache.ivy.core.module.descriptor.ModuleDescriptor
 import org.apache.ivy.core.module.id.ModuleRevisionId
-import sbt.coursierint._
 import sbt.Def.{ Initialize, ScopedKey, Setting, SettingsDefinition }
 import sbt.Keys._
 import sbt.Project.{
@@ -28,6 +27,7 @@ import sbt.Project.{
   richTaskSessionVar
 }
 import sbt.Scope.{ GlobalScope, ThisScope, fillTaskAxis }
+import sbt.coursierint._
 import sbt.internal.CommandStrings.ExportStream
 import sbt.internal._
 import sbt.internal.classpath.AlternativeZincUtil
@@ -69,10 +69,10 @@ import sbt.librarymanagement.CrossVersion.{ binarySbtVersion, binaryScalaVersion
 import sbt.librarymanagement._
 import sbt.librarymanagement.ivy._
 import sbt.librarymanagement.syntax._
-import sbt.nio.{ FileChanges, Watch }
 import sbt.nio.Keys._
-import sbt.nio.file.{ FileTreeView, Glob, RecursiveGlob }
 import sbt.nio.file.syntax._
+import sbt.nio.file.{ FileTreeView, Glob, RecursiveGlob }
+import sbt.nio.{ FileChanges, Watch }
 import sbt.std.TaskExtra._
 import sbt.testing.{ AnnotatedFingerprint, Framework, Runner, SubclassFingerprint }
 import sbt.util.CacheImplicits._
@@ -150,6 +150,10 @@ object Defaults extends BuildCommon {
     defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq(
       excludeFilter :== HiddenFileFilter,
       fileInputs :== Nil,
+      fileInputIncludeFilter :== AllPassFilter.toNio,
+      fileInputExcludeFilter :== DirectoryFilter.toNio || HiddenFileFilter,
+      fileOutputIncludeFilter :== AllPassFilter.toNio,
+      fileOutputExcludeFilter :== NothingFilter.toNio,
       inputFileStamper :== sbt.nio.FileStamper.Hash,
       outputFileStamper :== sbt.nio.FileStamper.LastModified,
       onChangedBuildSource :== sbt.nio.Keys.WarnOnSourceChanges,

main/src/main/scala/sbt/nio/Keys.scala

@@ -17,7 +17,7 @@ import sbt.internal.DynamicInput
 import sbt.internal.nio.FileTreeRepository
 import sbt.internal.util.AttributeKey
 import sbt.internal.util.complete.Parser
-import sbt.nio.file.{ FileAttributes, FileTreeView, Glob }
+import sbt.nio.file.{ FileAttributes, FileTreeView, Glob, PathFilter }
 import sbt._
 
 import scala.concurrent.duration.FiniteDuration
@@ -34,11 +34,19 @@ object Keys {
   val fileInputs = settingKey[Seq[Glob]](
     "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution."
   )
+  val fileInputIncludeFilter =
+    settingKey[PathFilter]("A filter to apply to the input sources of a task.")
+  val fileInputExcludeFilter =
+    settingKey[PathFilter]("An exclusion filter to apply to the input sources of a task.")
   val inputFileStamper = settingKey[FileStamper](
     "Toggles the file stamping implementation used to determine whether or not a file has been modified."
   )
 
   val fileOutputs = settingKey[Seq[Glob]]("Describes the output files of a task.")
+  val fileOutputIncludeFilter =
+    settingKey[PathFilter]("A filter to apply to the outputs of a task.")
+  val fileOutputExcludeFilter =
+    settingKey[PathFilter]("An exclusion filter to apply to the outputs of a task.")
   val allOutputFiles =
     taskKey[Seq[Path]]("All of the file outputs for a task excluding directories and hidden files.")
   val changedOutputFiles =

main/src/main/scala/sbt/nio/Settings.scala

@@ -9,7 +9,7 @@ package sbt
 package nio
 
 import java.io.File
-import java.nio.file.{ Files, Path }
+import java.nio.file.Path
 import java.util.concurrent.ConcurrentHashMap
 
 import sbt.Project._
@@ -20,6 +20,7 @@ import sbt.internal.{ Clean, Continuous, DynamicInput, SettingsGraph }
 import sbt.nio.FileStamp.Formats._
 import sbt.nio.FileStamper.{ Hash, LastModified }
 import sbt.nio.Keys._
+import sbt.nio.file.{ AllPass, FileAttributes }
 import sbt.std.TaskExtra._
 import sjsonnew.JsonFormat
 
@@ -45,8 +46,8 @@ private[sbt] object Settings {
    * `File`, `Seq[File]`, `Path`, `Seq[Path`. If it does, then we inject a number of
    * task definition settings that allow the user to check if the output paths of
    * the task have changed. It also adds a custom clean task that will delete the
-   * paths returned by the task, provided that they are in the task's target directory. We also inject these tasks if the fileOutputs setting is defined
-   * for the task.
+   * paths returned by the task, provided that they are in the task's target directory. We also
+   * inject these tasks if the fileOutputs setting is defined for the task.
    *
    * @param setting the setting to possibly inject with additional settings
    * @param fileOutputScopes the set of scopes for which the fileOutputs setting is defined
@@ -154,12 +155,13 @@ private[sbt] object Settings {
    */
   private[sbt] def inputPathSettings(setting: Def.Setting[_]): Seq[Def.Setting[_]] = {
     val scopedKey = setting.key
-    setting :: (Keys.allInputPathsAndAttributes in scopedKey.scope := {
-      val view = (fileTreeView in scopedKey.scope).value
-      val inputs = (fileInputs in scopedKey.scope).value
-      val stamper = (inputFileStamper in scopedKey.scope).value
-      val forceTrigger = (watchForceTriggerOnAnyChange in scopedKey.scope).value
-      val dynamicInputs = (Continuous.dynamicInputs in scopedKey.scope).value
+    val scope = scopedKey.scope
+    setting :: (Keys.allInputPathsAndAttributes in scope := {
+      val view = (fileTreeView in scope).value
+      val inputs = (fileInputs in scope).value
+      val stamper = (inputFileStamper in scope).value
+      val forceTrigger = (watchForceTriggerOnAnyChange in scope).value
+      val dynamicInputs = (Continuous.dynamicInputs in scope).value
       // This makes watch work by ensuring that the input glob is registered with the
       // repository used by the watch process.
       sbt.Keys.state.value.get(globalFileTreeRepository).foreach { repo =>
@@ -167,8 +169,7 @@ private[sbt] object Settings {
       }
       dynamicInputs.foreach(_ ++= inputs.map(g => DynamicInput(g, stamper, forceTrigger)))
       view.list(inputs)
-    }) :: fileStamps(scopedKey) :: allFilesImpl(scopedKey) :: Nil ++
-      changedInputFilesImpl(scopedKey.scope)
+    }) :: fileStamps(scopedKey) :: allFilesImpl(scope) :: changedInputFilesImpl(scope)
   }
 
   private[this] val taskClass = classOf[Task[_]]
@@ -183,12 +184,15 @@ private[sbt] object Settings {
    * @param scopedKey the key whose file inputs we are seeking
    * @return a task definition that retrieves all of the input paths scoped to the input key.
    */
-  private[this] def allFilesImpl(scopedKey: Def.ScopedKey[_]): Def.Setting[_] =
-    addTaskDefinition(Keys.allInputFiles in scopedKey.scope := {
-      (Keys.allInputPathsAndAttributes in scopedKey.scope).value.collect {
-        case (p, a) if a.isRegularFile && !Files.isHidden(p) => p
+  private[this] def allFilesImpl(scope: Scope): Def.Setting[_] = {
+    addTaskDefinition(Keys.allInputFiles in scope := {
+      val filter =
+        (fileInputIncludeFilter in scope).value && !(fileInputExcludeFilter in scope).value
+      (Keys.allInputPathsAndAttributes in scope).value.collect {
+        case (p, a) if filter.accept(p, a) => p
       }
     })
+  }
 
   /**
    * Returns all of the regular files whose stamp has changed since the last time the
@@ -199,7 +203,7 @@ private[sbt] object Settings {
    * @param scope the scope corresponding to the task whose fileInputs we are seeking
    * @return a task definition that retrieves the changed input files scoped to the key.
    */
-  private[this] def changedInputFilesImpl(scope: Scope): Seq[Def.Setting[_]] =
+  private[this] def changedInputFilesImpl(scope: Scope): List[Def.Setting[_]] =
     changedFilesImpl(scope, changedInputFiles, inputFileStamps) ::
       (watchForceTriggerOnAnyChange in scope := {
         (watchForceTriggerOnAnyChange in scope).?.value match {
@@ -282,10 +286,11 @@ private[sbt] object Settings {
    * @return a task definition that retrieves the input files and their file stamps scoped to the
    *         input key.
    */
-  private[sbt] def fileStamps(scopedKey: Def.ScopedKey[_]): Def.Setting[_] =
-    addTaskDefinition(Keys.inputFileStamps in scopedKey.scope := {
-      val cache = (unmanagedFileStampCache in scopedKey.scope).value
-      val stamper = (Keys.inputFileStamper in scopedKey.scope).value
+  private[sbt] def fileStamps(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = {
+    val scope = scopedKey.scope
+    addTaskDefinition(Keys.inputFileStamps in scope := {
+      val cache = (unmanagedFileStampCache in scope).value
+      val stamper = (Keys.inputFileStamper in scope).value
       val stampFile: Path => Option[(Path, FileStamp)] =
         sbt.Keys.state.value.get(globalFileTreeRepository) match {
           case Some(repo: FileStampRepository) =>
@@ -299,11 +304,15 @@ private[sbt] object Settings {
           case _ =>
             (path: Path) => cache.getOrElseUpdate(path, stamper).map(path -> _)
         }
-      (Keys.allInputPathsAndAttributes in scopedKey.scope).value.flatMap {
-        case (path, a) if a.isRegularFile && !Files.isHidden(path) => stampFile(path)
-        case _                                                     => None
+      val filter =
+        (fileInputIncludeFilter in scope).value && !(fileInputExcludeFilter in scope).value
+      (Keys.allInputPathsAndAttributes in scope).value.flatMap {
+        case (path, a) if filter.accept(path, a) => stampFile(path)
+        case _                                   => None
       }
     })
+  }
+
   private[this] def outputsAndStamps[T: JsonFormat: ToSeqPath](
       taskKey: TaskKey[T],
       cleanScopes: mutable.Set[Scope]
@@ -315,10 +324,23 @@ private[sbt] object Settings {
   }
   private[this] def allOutputPathsImpl(scope: Scope): Def.Setting[_] =
     addTaskDefinition(allOutputFiles in scope := {
+      val filter =
+        (fileOutputIncludeFilter in scope).value && !(fileOutputExcludeFilter in scope).value
       val fileOutputGlobs = (fileOutputs in scope).value
-      val allFileOutputs = fileTreeView.value.list(fileOutputGlobs).map(_._1)
+      val allFileOutputs = (fileTreeView in scope).value.list(fileOutputGlobs).map(_._1)
       val dynamicOutputs = (dynamicFileOutputs in scope).value
-      allFileOutputs ++ dynamicOutputs.filterNot(p => fileOutputGlobs.exists(_.matches(p)))
+      /*
+       * We want to avoid computing the FileAttributes in the common case where nothing is
+       * being filtered (which is the case with the default filters:
+       * include = AllPass, exclude = NoPass).
+       */
+      val attributeFilter: Path => Boolean = filter match {
+        case AllPass => _ => true
+        case f       => p => FileAttributes(p).map(f.accept(p, _)).getOrElse(false)
+      }
+      allFileOutputs ++ dynamicOutputs.filterNot { p =>
+        fileOutputGlobs.exists(_.matches(p)) || !attributeFilter(p)
+      }
     })
   private[this] def outputFileStampsImpl(scope: Scope): Def.Setting[_] =
     addTaskDefinition(outputFileStamps in scope := {

sbt/src/main/scala/sbt/Import.scala

@@ -70,6 +70,8 @@ trait Import {
   val FileChanges = sbt.nio.FileChanges
   type Glob = sbt.nio.file.Glob
   val Glob = sbt.nio.file.Glob
+  type PathFilter = sbt.nio.file.PathFilter
+  val PathFilter = sbt.nio.file.PathFilter
   type RelativeGlob = sbt.nio.file.RelativeGlob
   val RelativeGlob = sbt.nio.file.RelativeGlob
   val RecursiveGlob = sbt.nio.file.RecursiveGlob

sbt/src/sbt-test/nio/input-filters/build.sbt

@@ -0,0 +1,34 @@
+import java.nio.file.{ Files, Path }
+
+val copyPaths = taskKey[Seq[Path]]("Copy paths")
+copyPaths / fileInputs += baseDirectory.value.toGlob / "inputs" / *
+copyPaths := {
+  val outFile = streams.value.cacheDirectory
+  IO.delete(outFile)
+  val out = Files.createDirectories(outFile.toPath)
+  copyPaths.inputFiles.map { path =>
+    Files.write(out / path.getFileName.toString, Files.readAllBytes(path))
+  }
+}
+
+val checkPaths = inputKey[Unit]("check paths")
+checkPaths := {
+  val expectedFileNames = Def.spaceDelimited().parsed.toSet
+  val actualFileNames = copyPaths.outputFiles.map(_.getFileName.toString).toSet
+  assert(expectedFileNames == actualFileNames)
+
+}
+
+val newFilter = settingKey[PathFilter]("Works around quotations not working in scripted")
+newFilter := HiddenFileFilter.toNio || "**/bar.txt"
+
+val fooFilter = settingKey[PathFilter]("A filter for the bar.txt file")
+fooFilter := ** / ".foo.txt"
+
+Global / onLoad := { s: State =>
+  if (scala.util.Properties.isWin) {
+    val path = s.baseDir.toPath / "inputs" / ".foo.txt"
+    Files.setAttribute(path, "dos:hidden", true)
+  }
+  s
+}

sbt/src/sbt-test/nio/input-filters/inputs/.foo.txt

@@ -0,0 +1 @@
+foo

sbt/src/sbt-test/nio/input-filters/inputs/bar.txt

@@ -0,0 +1 @@
+bar

sbt/src/sbt-test/nio/input-filters/test

@@ -0,0 +1,10 @@
+# hidden files are excluded
+> checkPaths bar.txt
+
+> set copyPaths / fileInputExcludeFilter := NothingFilter.toNio
+
+> checkPaths .foo.txt bar.txt
+
+> set copyPaths / fileInputIncludeFilter := fooFilter.value
+
+> checkPaths .foo.txt

sbt/src/sbt-test/nio/legacy-filters/build.sbt

@@ -1,4 +1,3 @@
-Compile / excludeFilter := "Bar.scala" || "Baz.scala"
 
 val checkSources = inputKey[Unit]("Check that the compile sources match the input file names")
 checkSources := {
@@ -6,3 +5,11 @@ checkSources := {
   val actual = (Compile / unmanagedSources).value.map(_.getName).toSet
   assert(sources == actual)
 }
+
+val oldExcludeFilter = settingKey[sbt.io.FileFilter]("the default exclude filter")
+oldExcludeFilter :=  "Bar.scala" || "Baz.scala"
+
+Compile / excludeFilter := oldExcludeFilter.value
+
+val newFilter = settingKey[sbt.nio.file.PathFilter]("an alternative path filter")
+newFilter := !sbt.nio.file.PathFilter(** / "{Baz,Bar}.scala")

sbt/src/sbt-test/nio/legacy-filters/test

@@ -6,4 +6,16 @@
 
 > checkSources Foo.scala Bar.scala
 
--> compile
+-> compile
+
+> set Compile / unmanagedSources / excludeFilter := oldExcludeFilter.value
+
+> compile
+
+> set Compile / unmanagedSources / excludeFilter := HiddenFileFilter
+
+-> compile
+
+> set Compile / unmanagedSources / fileInputIncludeFilter := newFilter.value
+
+> compile

sbt/src/sbt-test/nio/output-filters/build.sbt

@@ -0,0 +1,21 @@
+import java.nio.file.{ Files, Path }
+
+val outputTask = taskKey[Seq[Path]]("A task that generates outputs")
+outputTask := {
+  val dir = Files.createDirectories(streams.value.cacheDirectory.toPath)
+  Seq("foo.txt" -> "foo", "bar.txt" -> "bar").map { case (name, content) =>
+    Files.write(dir/ name, content.getBytes)
+  } :+ dir
+}
+
+val checkOutputs = inputKey[Unit]("check outputs")
+checkOutputs := {
+  val expected = Def.spaceDelimited("").parsed.map {
+    case "base" => (outputTask / streams).value.cacheDirectory.toPath
+    case f => (outputTask / streams).value.cacheDirectory.toPath / f
+  }
+  assert((outputTask / allOutputFiles).value.toSet == expected.toSet)
+}
+
+val barFilter = settingKey[PathFilter]("A filter for the bar.txt file")
+barFilter := ** / "bar.txt"

sbt/src/sbt-test/nio/output-filters/test

@@ -0,0 +1,17 @@
+> compile
+
+> checkOutputs foo.txt bar.txt base
+
+> set outputTask / fileOutputIncludeFilter := sbt.io.RegularFileFilter
+
+> checkOutputs foo.txt bar.txt
+
+> set outputTask / fileOutputIncludeFilter := sbt.io.DirectoryFilter
+
+> checkOutputs base
+
+> set outputTask / fileOutputIncludeFilter := sbt.io.RegularFileFilter
+
+> set outputTask / fileOutputExcludeFilter := barFilter.value
+
+> checkOutputs foo.txt