Add dsl for collecting globs

Right now, the sbt.internal.io.Source is something of a second class
citizen within sbt. Since sbt 0.13, there have been extension classes
defined that can convert a file to a PathFinder but no analog has been
introduced for sbt.internal.io.Source.

Given that sbt.internal.io.Source was not really intended to be part of
the public api (just look at its package), I think it makes sense to
just replace it with Glob. In this commit, I add extension
methods to Glob and Seq[Glob] that make it possible to easily
retrieve all of the files for a particular Glob within a task. The
upshot is that where previously, we'd have had to write something like:

watchSources += Source(baseDirectory.value / "src" / "main" / "proto", "*.proto", NothingFilter)

now we can write

watchGlobs += baseDirectory.value / "src" / "main" / "proto" * "*.proto"

Moreover, within a task, we can now do something like:
foo := {
  val allWatchGlobs: Seq[File] = watchGlobs.value.all
  println(allWatchSources.mkString("all watch source files:\n", "\n", ""))
}
Before we would have had to manually retrieve the files.

The implementation of the dsl uses the new GlobExtractor class which
proxies file look ups through a FileTree.Repository. This makes it so
that, by default, all file i/o using Sources will use the default
FileTree.Repository. The default is a macro that returns
`sbt.Keys.fileTreeRepository.value: @sbtUnchecked`. By doing it this
way, the default repository can only be used within a task definition
(since it delegates to `fileTreeRepository.value`). It does not,
however, prevent the user from explicitly providing a
FileTree.Repository instance which the user is free to instantiate
however they wish.

Bonus: optimize imports in Def.scala and Defaults.scala

core-macros/src/main/scala/sbt/internal/util/appmacro/MacroDefaults.scala

@@ -0,0 +1,25 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.internal.util.appmacro
+
+import scala.reflect.macros.blackbox
+
+object MacroDefaults {
+
+  /**
+   * Macro to generated default file tree repository. It must be defined as an untyped tree because
+   * sbt.Keys is not available in this project. This is meant for internal use only, but must be
+   * public because its a macro.
+   * @param c the macro context
+   * @return the tree expressing the default file tree repository.
+   */
+  def fileTreeRepository(c: blackbox.Context): c.Tree = {
+    import c.universe._
+    q"sbt.Keys.fileTreeRepository.value: @sbtUnchecked"
+  }
+}

main-command/src/test/scala/sbt/WatchedSpec.scala

@@ -25,7 +25,6 @@ import scala.concurrent.duration._
 
 class WatchedSpec extends FlatSpec with Matchers {
   object Defaults {
-    private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis)
     def config(
         globs: Seq[Glob],
         fileEventMonitor: Option[FileEventMonitor[FileCacheEntry]] = None,

main-settings/src/main/scala/sbt/Def.scala

@@ -8,12 +8,13 @@
 package sbt
 
 import sbt.internal.util.Types.const
-import sbt.internal.util.{ Attributed, AttributeKey, Init, ConsoleAppender }
+import sbt.internal.util.{ AttributeKey, Attributed, ConsoleAppender, Init }
 import sbt.util.Show
 import sbt.internal.util.complete.Parser
 import java.io.File
 import java.net.URI
-import Scope.{ ThisScope, GlobalScope }
+
+import Scope.{ GlobalScope, ThisScope }
 import KeyRanks.{ DTask, Invisible }
 
 /** A concrete settings system that uses `sbt.Scope` for the scope type. */

main/src/main/scala/sbt/Defaults.scala

@@ -47,8 +47,8 @@ import sbt.internal.util.Types._
 import sbt.internal.util._
 import sbt.internal.util.complete._
 import sbt.io.Path._
-import sbt.io.syntax._
 import sbt.io._
+import sbt.io.syntax._
 import sbt.librarymanagement.Artifact.{ DocClassifier, SourceClassifier }
 import sbt.librarymanagement.Configurations.{
   Compile,
@@ -68,8 +68,8 @@ import sbt.testing.{ AnnotatedFingerprint, Framework, Runner, SubclassFingerprin
 import sbt.util.CacheImplicits._
 import sbt.util.InterfaceUtil.{ toJavaFunction => f1 }
 import sbt.util._
-import sjsonnew.shaded.scalajson.ast.unsafe.JValue
 import sjsonnew._
+import sjsonnew.shaded.scalajson.ast.unsafe.JValue
 import xsbti.CrossValue
 import xsbti.compile.{ AnalysisContents, IncOptions, IncToolOptionsUtil }
 
@@ -80,6 +80,7 @@ import scala.xml.NodeSeq
 
 // incremental compiler
 import sbt.SlashSyntax0._
+import sbt.internal.GlobLister._
 import sbt.internal.inc.{
   Analysis,
   AnalyzingCompiler,
@@ -280,10 +281,14 @@ object Defaults extends BuildCommon {
         None
       },
       watchStartMessage := Watched.defaultStartWatch,
+      fileTreeRepository := state.value
+        .get(Keys.globalFileTreeRepository)
+        .map(FileTree.repository)
+        .getOrElse(FileTree.Repository.polling),
       externalHooks := {
-        val view = FileManagement.dataView.value
+        val repository = fileTreeRepository.value
         compileOptions =>
-          Some(ExternalHooks(compileOptions, view))
+          Some(ExternalHooks(compileOptions, repository))
       },
       watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS),
       watchLogger := streams.value.log,
@@ -373,13 +378,12 @@ object Defaults extends BuildCommon {
           crossPaths.value
         )
     },
-    unmanagedSources := FileManagement
-      .collectFiles(
-        unmanagedSourceDirectories,
-        includeFilter in unmanagedSources,
-        excludeFilter in unmanagedSources
-      )
-      .value,
+    unmanagedSources := {
+      val filter =
+        (includeFilter in unmanagedSources).value -- (excludeFilter in unmanagedSources).value
+      val baseSources = if (sourcesInBase.value) baseDirectory.value * filter :: Nil else Nil
+      (unmanagedSourceDirectories.value.map(_ ** filter) ++ baseSources).all
+    },
     watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ {
       val baseDir = baseDirectory.value
       val bases = unmanagedSourceDirectories.value
@@ -413,13 +417,11 @@ object Defaults extends BuildCommon {
     resourceDirectories := Classpaths
       .concatSettings(unmanagedResourceDirectories, managedResourceDirectories)
       .value,
-    unmanagedResources := FileManagement
-      .collectFiles(
-        unmanagedResourceDirectories,
-        includeFilter in unmanagedResources,
-        excludeFilter in unmanagedResources
-      )
-      .value,
+    unmanagedResources := {
+      val filter =
+        (includeFilter in unmanagedResources).value -- (excludeFilter in unmanagedResources).value
+      unmanagedResourceDirectories.value.map(_ ** filter).all
+    },
     watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ {
       val bases = unmanagedResourceDirectories.value
       val include = (includeFilter in unmanagedResources).value
@@ -433,7 +435,8 @@ object Defaults extends BuildCommon {
     managedResources := generate(resourceGenerators).value,
     resources := Classpaths.concat(managedResources, unmanagedResources).value
   )
-  def addBaseSources = FileManagement.appendBaseSources
+  // This exists for binary compatibility and probably never should have been public.
+  def addBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Nil
   lazy val outputConfigPaths = Seq(
     classDirectory := crossTarget.value / (prefix(configuration.value.name) + "classes"),
     semanticdbTargetRoot := crossTarget.value / (prefix(configuration.value.name) + "meta"),
@@ -1205,9 +1208,12 @@ object Defaults extends BuildCommon {
     }
   def collectFiles(
       dirs: ScopedTaskable[Seq[File]],
-      filter: ScopedTaskable[FileFilter],
-      excludes: ScopedTaskable[FileFilter]
-  ): Initialize[Task[Seq[File]]] = FileManagement.collectFiles(dirs, filter, excludes)
+      include: ScopedTaskable[FileFilter],
+      exclude: ScopedTaskable[FileFilter]
+  ): Initialize[Task[Seq[File]]] = Def.task {
+    val filter = include.toTask.value -- exclude.toTask.value
+    dirs.toTask.value.map(_ ** filter).all
+  }
   def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] =
     Def.setting {
       val f = artifactName.value
@@ -1807,8 +1813,7 @@ object Defaults extends BuildCommon {
   ) :+ (classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.RuntimeDependencies)
 
   lazy val compileSettings: Seq[Setting[_]] =
-    configSettings ++
-      (mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++
+    configSettings ++ (mainBgRunMainTask +: mainBgRunTask) ++
       Classpaths.addUnmanagedLibrary ++ runtimeLayeringSettings
 
   private val testLayeringSettings: Seq[Setting[_]] = TaskRepository.proxy(

main/src/main/scala/sbt/Keys.scala

@@ -94,6 +94,7 @@ object Keys {
   val watch = SettingKey(BasicKeys.watch)
   val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting)
   val enableGlobalCachingFileTreeRepository = settingKey[Boolean]("Toggles whether or not to create a global cache of the file system that can be used by tasks to quickly list a path").withRank(DSetting)
+  val fileTreeRepository = taskKey[FileTree.Repository]("A repository of the file system.")
   val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting)
   val pollingGlobs = settingKey[Seq[Glob]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting)
   val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting)

main/src/main/scala/sbt/internal/ExternalHooks.scala

@@ -11,8 +11,9 @@ import java.util.Optional
 
 import sbt.Stamped
 import sbt.internal.inc.ExternalLookup
+import sbt.io.FileTreeDataView.Entry
 import sbt.io.syntax._
-import sbt.io.{ AllPassFilter, FileTreeDataView, FileTreeRepository, TypedPath }
+import sbt.io.{ AllPassFilter, Glob, TypedPath }
 import xsbti.compile._
 import xsbti.compile.analysis.Stamp
 
@@ -20,10 +21,8 @@ import scala.collection.mutable
 
 private[sbt] object ExternalHooks {
   private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_))
-  def apply(
-      options: CompileOptions,
-      view: FileTreeDataView[FileCacheEntry]
-  ): DefaultExternalHooks = {
+  def apply(options: CompileOptions, repo: FileTree.Repository): DefaultExternalHooks = {
+    def listEntries(glob: Glob): Seq[Entry[FileCacheEntry]] = repo.get(glob)
     import scala.collection.JavaConverters._
     val sources = options.sources()
     val cachedSources = new java.util.HashMap[File, Stamp]
@@ -32,28 +31,19 @@ private[sbt] object ExternalHooks {
       case sf: Stamped => cachedSources.put(sf, sf.stamp)
       case f: File     => cachedSources.put(f, converter(f))
     }
-    view match {
-      case r: FileTreeRepository[FileCacheEntry] =>
-        r.register(options.classesDirectory ** AllPassFilter)
-        options.classpath.foreach {
-          case f if f.getName.endsWith(".jar") => r.register(f.toGlob)
-          case f                               => r.register(f ** AllPassFilter)
-        }
-      case _ =>
-    }
     val allBinaries = new java.util.HashMap[File, Stamp]
     options.classpath.foreach {
       case f if f.getName.endsWith(".jar") =>
         // This gives us the entry for the path itself, which is necessary if the path is a jar file
         // rather than a directory.
-        view.listEntries(f.toGlob) foreach { e =>
+        listEntries(f.toGlob) foreach { e =>
           e.value match {
             case Right(value) => allBinaries.put(e.typedPath.toPath.toFile, value.stamp)
             case _            =>
           }
         }
       case f =>
-        view.listEntries(f ** "*.jar") foreach { e =>
+        listEntries(f ** AllPassFilter) foreach { e =>
           e.value match {
             case Right(value) => allBinaries.put(e.typedPath.toPath.toFile, value.stamp)
             case _            =>

main/src/main/scala/sbt/internal/FileManagement.scala

@@ -11,9 +11,8 @@ package internal
 import sbt.BasicCommandStrings.ContinuousExecutePrefix
 import sbt.Keys._
 import sbt.internal.io.HybridPollingFileTreeRepository
-import sbt.io.FileTreeDataView.{ Entry, Observable, Observer, Observers }
-import sbt.io._
-import sbt.io.syntax._
+import sbt.io.FileTreeDataView.{ Observable, Observer, Observers }
+import sbt.io.{ FileTreeRepository, _ }
 import sbt.util.Logger
 
 import scala.concurrent.duration._
@@ -85,76 +84,8 @@ private[sbt] object FileManagement {
     }
   }
 
-  private def entryFilter(
-      include: FileFilter,
-      exclude: FileFilter
-  ): Entry[FileCacheEntry] => Boolean = { e =>
-    val tp = e.typedPath
-    /*
-     * The TypedPath has the isDirectory and isFile properties embedded. By overriding
-     * these methods in java.io.File, FileFilters may be applied without needing to
-     * stat the file (which is expensive) for isDirectory and isFile checks.
-     */
-    val file = new java.io.File(tp.toPath.toString) {
-      override def isDirectory: Boolean = tp.isDirectory
-      override def isFile: Boolean = tp.isFile
-    }
-    include.accept(file) && !exclude.accept(file)
-  }
   private[sbt] def repo: Def.Initialize[Task[FileTreeRepository[FileCacheEntry]]] = Def.task {
     lazy val msg = s"Tried to get FileTreeRepository for uninitialized state."
     state.value.get(Keys.globalFileTreeRepository).getOrElse(throw new IllegalStateException(msg))
   }
-  private[sbt] def dataView: Def.Initialize[Task[FileTreeDataView[FileCacheEntry]]] = Def.task {
-    state.value
-      .get(Keys.globalFileTreeRepository)
-      .map(toDataView)
-      .getOrElse(FileTreeView.DEFAULT.asDataView(FileCacheEntry.default))
-  }
-  private def toDataView(r: FileTreeRepository[FileCacheEntry]): FileTreeDataView[FileCacheEntry] =
-    new FileTreeDataView[FileCacheEntry] {
-      private def reg(glob: Glob): FileTreeDataView[FileCacheEntry] = { r.register(glob); r }
-      override def listEntries(glob: Glob): Seq[Entry[FileCacheEntry]] = reg(glob).listEntries(glob)
-      override def list(glob: Glob): Seq[TypedPath] = reg(glob).list(glob)
-      override def close(): Unit = {}
-    }
-  private[sbt] def collectFiles(
-      dirs: ScopedTaskable[Seq[File]],
-      filter: ScopedTaskable[FileFilter],
-      excludes: ScopedTaskable[FileFilter]
-  ): Def.Initialize[Task[Seq[File]]] =
-    Def.task {
-      val sourceDirs = dirs.toTask.value
-      val view: FileTreeDataView[FileCacheEntry] = dataView.value
-      val include = filter.toTask.value
-      val ex = excludes.toTask.value
-      val sourceFilter: Entry[FileCacheEntry] => Boolean = entryFilter(include, ex)
-      sourceDirs.flatMap { dir =>
-        view
-          .listEntries(dir.toPath ** AllPassFilter)
-          .flatMap {
-            case e if sourceFilter(e) => e.value.toOption.map(Stamped.file(e.typedPath, _))
-            case _                    => None
-          }
-      }
-    }
-
-  private[sbt] def appendBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Seq(
-    unmanagedSources := {
-      val sources = unmanagedSources.value
-      val include = (includeFilter in unmanagedSources).value
-      val excl = (excludeFilter in unmanagedSources).value
-      val baseDir = baseDirectory.value
-      val r: FileTreeDataView[FileCacheEntry] = dataView.value
-      if (sourcesInBase.value) {
-        val filter: Entry[FileCacheEntry] => Boolean = entryFilter(include, excl)
-        sources ++
-          r.listEntries(baseDir * AllPassFilter)
-            .flatMap {
-              case e if filter(e) => e.value.toOption.map(Stamped.file(e.typedPath, _))
-              case _              => None
-            }
-      } else sources
-    }
-  )
 }

main/src/main/scala/sbt/internal/FileTree.scala

@@ -0,0 +1,53 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package internal
+
+import java.nio.file.{ WatchService => _ }
+
+import sbt.internal.util.appmacro.MacroDefaults
+import sbt.io.FileTreeDataView.Entry
+import sbt.io._
+
+import scala.language.experimental.macros
+
+object FileTree {
+  trait Repository extends sbt.internal.Repository[Seq, Glob, Entry[FileCacheEntry]]
+  object Repository {
+
+    /**
+     * Provide a default [[Repository]] that works within a task definition, e.g. Def.task. It's
+     * implemented as a macro so that it can call `.value` on a TaskKey. Using a macro also allows
+     * us to use classes that aren't actually available in this project, e.g. sbt.Keys.
+     * @return a [[Repository]] instance
+     */
+    implicit def default: FileTree.Repository = macro MacroDefaults.fileTreeRepository
+    private[sbt] object polling extends Repository {
+      val view = FileTreeView.DEFAULT.asDataView(FileCacheEntry.default)
+      override def get(key: Glob): Seq[Entry[FileCacheEntry]] = view.listEntries(key)
+      override def close(): Unit = {}
+    }
+  }
+  private class ViewRepository(underlying: FileTreeDataView[FileCacheEntry]) extends Repository {
+    override def get(key: Glob): Seq[Entry[FileCacheEntry]] = underlying.listEntries(key)
+    override def close(): Unit = {}
+  }
+  private class CachingRepository(underlying: FileTreeRepository[FileCacheEntry])
+      extends Repository {
+    override def get(key: Glob): Seq[Entry[FileCacheEntry]] = {
+      underlying.register(key)
+      underlying.listEntries(key)
+    }
+    override def close(): Unit = underlying.close()
+  }
+  private[sbt] def repository(underlying: FileTreeDataView[FileCacheEntry]): Repository =
+    underlying match {
+      case r: FileTreeRepository[FileCacheEntry] => new CachingRepository(r)
+      case v                                     => new ViewRepository(v)
+    }
+}

main/src/main/scala/sbt/internal/GlobLister.scala

@@ -0,0 +1,97 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package internal
+
+import sbt.io.{ Glob, TypedPath }
+
+/**
+ * Retrieve files from a repository. This should usually be an extension class for
+ * sbt.io.internal.Glob (or a Traversable collection of source instances) that allows us to
+ * actually retrieve the files corresponding to those sources.
+ */
+sealed trait GlobLister extends Any {
+
+  /**
+   * Get the sources described this [[GlobLister]].
+   *
+   * @param repository the [[FileTree.Repository]] to delegate file i/o.
+   * @return the files described by this [[GlobLister]].
+   */
+  def all(implicit repository: FileTree.Repository): Seq[Stamped.File]
+
+  /**
+   * Get the unique sources described this [[GlobLister]].
+   *
+   * @param repository the [[FileTree.Repository]] to delegate file i/o.
+   * @return the files described by this [[GlobLister]] with any duplicates removed.
+   */
+  def unique(implicit repository: FileTree.Repository): Seq[Stamped.File]
+}
+
+/**
+ * Provides implicit definitions to provide a [[GlobLister]] given a Glob or
+ * Traversable[Glob].
+ */
+object GlobLister extends GlobListers
+
+/**
+ * Provides implicit definitions to provide a [[GlobLister]] given a Glob or
+ * Traversable[Glob].
+ */
+private[sbt] trait GlobListers {
+  import GlobListers._
+
+  /**
+   * Generate a [[GlobLister]] given a particular [[Glob]]s.
+   *
+   * @param source the input Glob
+   */
+  implicit def fromGlob(source: Glob): GlobLister = new impl(source :: Nil)
+
+  /**
+   * Generate a [[GlobLister]] given a collection of Globs. If the input collection type
+   * preserves uniqueness, e.g. `Set[Glob]`, then the output of [[GlobLister.all]] will be
+   * the unique source list. Otherwise duplicates are possible in all and it is necessary to call
+   * [[GlobLister.unique]] to de-duplicate the files.
+   *
+   * @param sources the collection of sources
+   * @tparam T the source collection type
+   */
+  implicit def fromTraversableGlob[T <: Traversable[Glob]](sources: T): GlobLister =
+    new impl(sources)
+}
+private[internal] object GlobListers {
+
+  /**
+   * Implements [[GlobLister]] given a collection of Globs. If the input collection type
+   * preserves uniqueness, e.g. `Set[Glob]`, then the output will be the unique source list.
+   * Otherwise duplicates are possible.
+   *
+   * @param globs the input globs
+   * @tparam T the collection type
+   */
+  private class impl[T <: Traversable[Glob]](val globs: T) extends AnyVal with GlobLister {
+    private def get[T0 <: Traversable[Glob]](
+        traversable: T0,
+        repository: FileTree.Repository
+    ): Seq[Stamped.File] =
+      traversable.flatMap { glob =>
+        val sourceFilter: TypedPath => Boolean = glob.toTypedPathFilter
+        repository.get(glob).flatMap {
+          case e if sourceFilter(e.typedPath) => e.value.toOption.map(Stamped.file(e.typedPath, _))
+          case _                              => None
+        }
+      }.toIndexedSeq: Seq[Stamped.File]
+
+    override def all(implicit repository: FileTree.Repository): Seq[Stamped.File] =
+      get(globs, repository)
+    override def unique(implicit repository: FileTree.Repository): Seq[Stamped.File] =
+      get(globs.toSet[Glob], repository)
+  }
+}

sbt/src/main/scala/package.scala

@@ -21,7 +21,8 @@ package object sbt
     with sbt.BuildSyntax
     with sbt.OptionSyntax
     with sbt.SlashSyntax
-    with sbt.Import {
+    with sbt.Import
+    with sbt.internal.GlobListers {
   // IO
   def uri(s: String): URI = new URI(s)
   def file(s: String): File = new File(s)