Add global file repository task

Every time that the compile task is run, there are potentially a large
number of iops that must occur in order for sbt to generate the source
file list as well as for zinc to check which files have changed since
the last build. This can lead to a noticeable delay between when a build
is started (either manually or by triggered execution) and when
compilation actually begins. To reduce this latency, I am adding a
global view of the file system that will be stored in
BasicKeys.globalFileTreeView.

To make this work, I introduce the StampedFile trait, which augments the
java.io.File class with a stamp method that returns the zinc stamp for
the file. For source files, this will be a hash of the file, while for
binaries, it is just the last modified time. In order to gain access to
the sbt.internal.inc.Stamper class, I had to append addSbtZinc to the
commandProj configurations.

This view may or may not use an in-memory cache of the file system tree
to return the results. Because there is always the risk of the cache
getting out of sync with the actual file system, I both make it optional
to use a cache and provide a mechanism for flushing the cache. Moreover,
the in-memory cache implementation in sbt.io, which is backed by a
swoval FileTreeRepository, has the property that touching a monitored
directory invalidates the entire directory within the cache, so the
flush command isn't even strictly needed in general.

Because caching is optional, the global is of a FileTreeDataView, which
doesn't specify a caching strategy. Subsequent commits will make use of
this to potentially speed up incremental compilation by caching the
Stamps of the source files so that zinc does not need to compute the
hashes itself and will allow for continuous builds to use the cache to
monitor events instead of creating a new, standalone FileEventMonitor.

build.sbt

@@ -507,7 +507,8 @@ lazy val commandProj = (project in file("main-command"))
     addSbtUtilLogging,
     addSbtCompilerInterface,
     addSbtCompilerClasspath,
-    addSbtLmCore
+    addSbtLmCore,
+    addSbtZinc
   )
 
 // The core macro project defines the main logic of the DSL, abstracted

main-command/src/main/scala/sbt/BasicCommandStrings.scala

@@ -231,4 +231,8 @@ $AliasCommand name=
   val ContinuousExecutePrefix = "~"
   def continuousDetail = "Executes the specified command whenever source files change."
   def continuousBriefHelp = (ContinuousExecutePrefix + " <command>", continuousDetail)
+  def FlushFileTreeRepository = "flushFileTreeRepository"
+  def FlushDetailed: String =
+    "Resets the global file repository in the event that the repository has become inconsistent " +
+      "with the file system."
 }

main-command/src/main/scala/sbt/BasicKeys.scala

@@ -11,6 +11,7 @@ import java.io.File
 import sbt.internal.util.AttributeKey
 import sbt.internal.inc.classpath.ClassLoaderCache
 import sbt.internal.server.ServerHandler
+import sbt.io.FileTreeDataView
 import sbt.librarymanagement.ModuleID
 import sbt.util.Level
 
@@ -100,6 +101,11 @@ object BasicKeys {
     "List of template resolver infos.",
     1000
   )
+  private[sbt] val globalFileTreeView = AttributeKey[FileTreeDataView[StampedFile]](
+    "globalFileTreeView",
+    "provides a view into the file system that may or may not cache the tree in memory",
+    1000
+  )
 }
 
 case class TemplateResolverInfo(module: ModuleID, implementationClass: String)

main-command/src/main/scala/sbt/FileTreeViewConfig.scala

@@ -6,11 +6,9 @@
  */
 
 package sbt
-import java.nio.file.Path
-
 import sbt.Watched.WatchSource
 import sbt.internal.io.{ WatchServiceBackedObservable, WatchState }
-import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView, TypedPath }
+import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView }
 import sbt.util.Logger
 
 import scala.concurrent.duration.FiniteDuration
@@ -19,21 +17,21 @@ import scala.concurrent.duration.FiniteDuration
  * Configuration for viewing and monitoring the file system.
  */
 final class FileTreeViewConfig private (
-    val newDataView: () => FileTreeDataView[Path],
+    val newDataView: () => FileTreeDataView[StampedFile],
     val newMonitor: (
-        FileTreeDataView[Path],
+        FileTreeDataView[StampedFile],
         Seq[WatchSource],
         Logger
-    ) => FileEventMonitor[Path]
+    ) => FileEventMonitor[StampedFile]
 )
 object FileTreeViewConfig {
 
   /**
    * Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded
-   * by {{{sbt.io.FileTreeDataView[Path]}}}. The reason for this is to ensure that a
+   * by {{{sbt.io.FileTreeDataView[StampedFile]}}}. The reason for this is to ensure that a
    * sbt.io.FileTreeDataView that is instantiated by [[FileTreeViewConfig.newDataView]] can be
    * passed into [[FileTreeViewConfig.newMonitor]] without constraining the type of view to be
-   * {{{sbt.io.FileTreeDataView[Path]}}}.
+   * {{{sbt.io.FileTreeDataView[StampedFile]}}}.
    * @param newDataView create a new sbt.io.FileTreeDataView. This value may be cached in a global
    *                    attribute
    * @param newMonitor create a new sbt.io.FileEventMonitor using the sbt.io.FileTreeDataView
@@ -41,13 +39,13 @@ object FileTreeViewConfig {
    * @tparam T the subtype of sbt.io.FileTreeDataView that is returned by [[FileTreeViewConfig.newDataView]]
    * @return a [[FileTreeViewConfig]] instance.
    */
-  def apply[T <: FileTreeDataView[Path]](
+  def apply[T <: FileTreeDataView[StampedFile]](
       newDataView: () => T,
-      newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[Path]
+      newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[StampedFile]
   ): FileTreeViewConfig =
     new FileTreeViewConfig(
       newDataView,
-      (view: FileTreeDataView[Path], sources: Seq[WatchSource], logger: Logger) =>
+      (view: FileTreeDataView[StampedFile], sources: Seq[WatchSource], logger: Logger) =>
         newMonitor(view.asInstanceOf[T], sources, logger)
     )
 
@@ -61,14 +59,14 @@ object FileTreeViewConfig {
    */
   def default(pollingInterval: FiniteDuration, antiEntropy: FiniteDuration): FileTreeViewConfig =
     FileTreeViewConfig(
-      () => FileTreeView.DEFAULT.asDataView(_.getPath),
-      (_: FileTreeDataView[Path], sources, logger) => {
+      () => FileTreeView.DEFAULT.asDataView(StampedFile.converter),
+      (_: FileTreeDataView[StampedFile], sources, logger) => {
         val ioLogger: sbt.io.WatchLogger = msg => logger.debug(msg.toString)
         FileEventMonitor.antiEntropy(
           new WatchServiceBackedObservable(
             WatchState.empty(Watched.createWatchService(), sources),
             pollingInterval,
-            (_: TypedPath).getPath,
+            StampedFile.converter,
             closeService = true,
             ioLogger
           ),

main-command/src/main/scala/sbt/Watched.scala

@@ -8,7 +8,7 @@
 package sbt
 
 import java.io.File
-import java.nio.file.{ FileSystems, Path }
+import java.nio.file.FileSystems
 
 import sbt.BasicCommandStrings.{
   ContinuousExecutePrefix,
@@ -18,6 +18,7 @@ import sbt.BasicCommandStrings.{
 }
 import sbt.BasicCommands.otherCommandParser
 import sbt.internal.LegacyWatched
+import sbt.internal.inc.Stamper
 import sbt.internal.io.{ EventMonitor, Source, WatchState }
 import sbt.internal.util.AttributeKey
 import sbt.internal.util.Types.const
@@ -25,6 +26,7 @@ import sbt.internal.util.complete.DefaultParsers
 import sbt.io.FileEventMonitor.Event
 import sbt.io._
 import sbt.util.{ Level, Logger }
+import xsbti.compile.analysis.Stamp
 
 import scala.annotation.tailrec
 import scala.concurrent.duration._
@@ -99,8 +101,8 @@ object Watched {
 
   /**
    * A user defined Action. It is not sealed so that the user can create custom instances. If any
-   * of the [[Config]] callbacks, e.g. [[Config.onWatchEvent]], return an instance of [[Custom]],
-   * the watch will terminate.
+   * of the [[WatchConfig]] callbacks, e.g. [[WatchConfig.onWatchEvent]], return an instance of
+   * [[Custom]], the watch will terminate.
    */
   trait Custom extends Action
 
@@ -414,7 +416,7 @@ trait WatchConfig {
    *
    * @return an sbt.io.FileEventMonitor instance.
    */
-  def fileEventMonitor: FileEventMonitor[Path]
+  def fileEventMonitor: FileEventMonitor[StampedFile]
 
   /**
    * A function that is periodically invoked to determine whether the watch should stop or
@@ -437,7 +439,7 @@ trait WatchConfig {
    * @param event the detected sbt.io.FileEventMonitor.Event.
    * @return the next [[Watched.Action Action]] to run.
    */
-  def onWatchEvent(event: Event[Path]): Watched.Action
+  def onWatchEvent(event: Event[StampedFile]): Watched.Action
 
   /**
    * Transforms the state after the watch terminates.
@@ -493,10 +495,10 @@ object WatchConfig {
    */
   def default(
       logger: Logger,
-      fileEventMonitor: FileEventMonitor[Path],
+      fileEventMonitor: FileEventMonitor[StampedFile],
       handleInput: () => Watched.Action,
       preWatch: (Int, Boolean) => Watched.Action,
-      onWatchEvent: Event[Path] => Watched.Action,
+      onWatchEvent: Event[StampedFile] => Watched.Action,
       onWatchTerminated: (Watched.Action, String, State) => State,
       triggeredMessage: (TypedPath, Int) => Option[String],
       watchingMessage: Int => Option[String]
@@ -511,11 +513,11 @@ object WatchConfig {
     val wm = watchingMessage
     new WatchConfig {
       override def logger: Logger = l
-      override def fileEventMonitor: FileEventMonitor[Path] = fem
+      override def fileEventMonitor: FileEventMonitor[StampedFile] = fem
       override def handleInput(): Watched.Action = hi()
       override def preWatch(count: Int, lastResult: Boolean): Watched.Action =
         pw(count, lastResult)
-      override def onWatchEvent(event: Event[Path]): Watched.Action = owe(event)
+      override def onWatchEvent(event: Event[StampedFile]): Watched.Action = owe(event)
       override def onWatchTerminated(action: Watched.Action, command: String, state: State): State =
         owt(action, command, state)
       override def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] =
@@ -524,3 +526,28 @@ object WatchConfig {
     }
   }
 }
+
+trait StampedFile extends File {
+  def stamp: Stamp
+}
+object StampedFile {
+  val sourceConverter: TypedPath => StampedFile =
+    new StampedFileImpl(_: TypedPath, forceLastModified = false)
+  val binaryConverter: TypedPath => StampedFile =
+    new StampedFileImpl(_: TypedPath, forceLastModified = true)
+  val converter: TypedPath => StampedFile = (tp: TypedPath) =>
+    tp.getPath.toString match {
+      case s if s.endsWith(".jar")   => binaryConverter(tp)
+      case s if s.endsWith(".class") => binaryConverter(tp)
+      case _                         => sourceConverter(tp)
+  }
+
+  private class StampedFileImpl(typedPath: TypedPath, forceLastModified: Boolean)
+      extends java.io.File(typedPath.getPath.toString)
+      with StampedFile {
+    override val stamp: Stamp =
+      if (forceLastModified || typedPath.isDirectory)
+        Stamper.forLastModified(typedPath.getPath.toFile)
+      else Stamper.forHash(typedPath.getPath.toFile)
+  }
+}

main-command/src/test/scala/sbt/WatchedSpec.scala

@@ -8,7 +8,7 @@
 package sbt
 
 import java.io.File
-import java.nio.file.{ Files, Path }
+import java.nio.file.Files
 import java.util.concurrent.atomic.AtomicBoolean
 
 import org.scalatest.{ FlatSpec, Matchers }
@@ -26,11 +26,11 @@ class WatchedSpec extends FlatSpec with Matchers {
     private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis, 50.millis)
     def config(
         sources: Seq[WatchSource],
-        fileEventMonitor: Option[FileEventMonitor[Path]] = None,
+        fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None,
         logger: Logger = NullLogger,
         handleInput: () => Action = () => Ignore,
         preWatch: (Int, Boolean) => Action = (_, _) => CancelWatch,
-        onWatchEvent: Event[Path] => Action = _ => Ignore,
+        onWatchEvent: Event[StampedFile] => Action = _ => Ignore,
         triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None,
         watchingMessage: Int => Option[String] = _ => None
     ): WatchConfig = {

main/src/main/scala/sbt/Defaults.scala

@@ -42,6 +42,7 @@ import sbt.io.{
   AllPassFilter,
   DirectoryFilter,
   FileFilter,
+  FileTreeView,
   GlobFilter,
   Hash,
   HiddenFileFilter,
@@ -271,6 +272,9 @@ object Defaults extends BuildCommon {
       },
       watchStartMessage := Watched.defaultStartWatch,
       fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value),
+      fileTreeView := state.value
+        .get(BasicKeys.globalFileTreeView)
+        .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)),
       watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS),
       watchLogger := streams.value.log,
       watchService :== { () =>
@@ -642,10 +646,9 @@ object Defaults extends BuildCommon {
         )
         .getOrElse(watchTriggeredMessage.value)
       val logger = watchLogger.value
-      val viewConfig = fileTreeViewConfig.value
       WatchConfig.default(
         logger,
-        viewConfig.newMonitor(viewConfig.newDataView(), sources, logger),
+        fileTreeViewConfig.value.newMonitor(fileTreeView.value, sources, logger),
         watchHandleInput.value,
         watchPreWatch.value,
         watchOnEvent.value,

main/src/main/scala/sbt/Keys.scala

@@ -8,7 +8,6 @@
 package sbt
 
 import java.io.File
-import java.nio.file.{ Path => JPath }
 import java.net.URL
 import scala.concurrent.duration.{ FiniteDuration, Duration }
 import Def.ScopedKey
@@ -41,7 +40,7 @@ import sbt.internal.{
   SessionSettings,
   LogManager
 }
-import sbt.io.{ FileFilter, TypedPath, WatchService }
+import sbt.io.{ FileFilter, FileTreeDataView, TypedPath, WatchService }
 import sbt.io.FileEventMonitor.Event
 import sbt.internal.io.WatchState
 import sbt.internal.server.ServerHandler
@@ -146,12 +145,13 @@ object Keys {
   @deprecated("This is no longer used for continuous execution", "1.3.0")
   val watch = SettingKey(BasicKeys.watch)
   val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting)
+  val fileTreeView = taskKey[FileTreeDataView[StampedFile]]("A view of the file system")
   val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting)
   val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting)
   val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting)
   val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting)
   val watchHandleInput = settingKey[() => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting)
-  val watchOnEvent = taskKey[Event[JPath] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting)
+  val watchOnEvent = taskKey[Event[StampedFile] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting)
   val watchOnTermination = taskKey[(Watched.Action, String, State) => State]("Transforms the input state after the continuous build completes.").withRank(BMinusSetting)
   val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting)
   val watchProjectSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources for the sbt meta project to watch to trigger a reload.").withRank(CSetting)

main/src/main/scala/sbt/Main.scala

@@ -49,7 +49,7 @@ import Project.LoadAction
 import xsbti.compile.CompilerCache
 
 import scala.annotation.tailrec
-import sbt.io.IO
+import sbt.io.{ FileTreeDataView, IO }
 import sbt.io.syntax._
 import java.io.{ File, IOException }
 import java.net.URI
@@ -242,7 +242,8 @@ object BuiltinCommands {
       boot,
       initialize,
       act,
-      continuous
+      continuous,
+      flushFileTreeRepository
     ) ++ allBasicCommands
 
   def DefaultBootCommands: Seq[String] =
@@ -858,7 +859,7 @@ object BuiltinCommands {
 
     val session = Load.initialSession(structure, eval, s0)
     SessionSettings.checkSession(session, s)
-    Project.setProject(session, structure, s)
+    registerGlobalFileRepository(Project.setProject(session, structure, s))
   }
 
   def registerCompilerCache(s: State): State = {
@@ -876,6 +877,27 @@ object BuiltinCommands {
       }
     s.put(Keys.stateCompilerCache, cache)
   }
+  def registerGlobalFileRepository(s: State): State = {
+    val extracted = Project.extract(s)
+    try {
+      val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s)
+      val view: FileTreeDataView[StampedFile] = config.newDataView()
+      val newState = s.addExitHook {
+        view.close()
+        s.attributes.remove(BasicKeys.globalFileTreeView)
+        ()
+      }
+      newState.get(BasicKeys.globalFileTreeView).foreach(_.close())
+      newState.put(BasicKeys.globalFileTreeView, view)
+    } catch {
+      case NonFatal(_) => s
+    }
+  }
+
+  def flushFileTreeRepository: Command = {
+    val help = Help.more(FlushFileTreeRepository, FlushDetailed)
+    Command.command(FlushFileTreeRepository, help)(registerGlobalFileRepository)
+  }
 
   def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 =>
     import sbt.internal.{ ConsolePromptEvent, ConsoleUnpromptEvent }