Rework FileTreeRepository configuration

The FileTreeViewConfig abstraction that I added was somewhat unwieldy
and confusing. The original intention was to provide users with a lot of
flexibility in configuring the global file tree repository used by sbt.
I don't think that flexibility is necessary and it was both conceptually
complicated and made the implementation complex. In this commit, I add a
new boolean flag enableGlobalCachingFileTreeRepository that toggles
which kind of FileTreeRepository to use globally.

There are actually three kinds of repositories that could be returned:
1) FileTreeRepository.default -- this caches the entire file system
   tree it hooks into the cache's event callbacks to create a file event
   monitor. It will be used if enableGlobalCachingFileTreeRepository is
   true and Global / pollingGlobs := Nil
2) FileTreeRepository.hybrid -- similar to FileTreeRepository.default
   except that it will not cache any files that are included in
   Global / pollingGlobs. It will be used if
   enableGlobalCachingFileTreeRepository is true and
   Global / pollingGlobs is non empty
3) FileTreeRepository.legacy -- does not cache any of the file system
   tree, but does maintain a persistent file monitoring process that is
   implemented with a WatchServiceBackedObservable. Because it doesn't
   poll, in general, it's ok to leave the monitoring on in the
   background. One reason to use this is that if there are any issues
   with the cache being unable to accurately mirror the underlying file
   system tree, this repository will always poll the file system
   whenever sbt requests the entries for a given glob. Moreover, the
   file system tree implementation is very similar to the implementation
   that was used in 1.2.x so this gives users a way to almost fully opt
   back in to the old behavior.
This commit is contained in:
Ethan Atkins 2019-02-01 11:32:49 -08:00
parent 792fb91737
commit d0310cc866
6 changed files with 109 additions and 238 deletions

View File

@ -1,185 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt
import sbt.Watched.WatchSource
import sbt.internal.FileCacheEntry
import sbt.internal.io.{ HybridPollingFileTreeRepository, WatchServiceBackedObservable, WatchState }
import sbt.io.FileTreeDataView.{ Observable, Observer }
import sbt.io._
import sbt.util.Logger
import scala.concurrent.duration._
/**
* Configuration for viewing and monitoring the file system.
*/
final class FileTreeViewConfig private (
val newDataView: () => FileTreeDataView[FileCacheEntry],
val newMonitor: (
FileTreeDataView[FileCacheEntry],
Seq[WatchSource],
Logger
) => FileEventMonitor[FileCacheEntry]
)
object FileTreeViewConfig {
private implicit class SourceOps(val s: WatchSource) extends AnyVal {
def toGlob: Glob = Glob(s.base, AllPassFilter, if (s.recursive) Integer.MAX_VALUE else 0)
}
/**
* Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded
* by {{{sbt.io.FileTreeDataView[FileCacheEntry]}}}. The reason for this is to ensure that a
* sbt.io.FileTreeDataView that is instantiated by [[FileTreeViewConfig.newDataView]] can be
* passed into [[FileTreeViewConfig.newMonitor]] without constraining the type of view to be
* {{{sbt.io.FileTreeDataView[FileCacheEntry]}}}.
* @param newDataView create a new sbt.io.FileTreeDataView. This value may be cached in a global
* attribute
* @param newMonitor create a new sbt.io.FileEventMonitor using the sbt.io.FileTreeDataView
* created by newDataView
* @tparam T the subtype of sbt.io.FileTreeDataView that is returned by [[FileTreeViewConfig.newDataView]]
* @return a [[FileTreeViewConfig]] instance.
*/
def apply[T <: FileTreeDataView[FileCacheEntry]](
newDataView: () => T,
newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[FileCacheEntry]
): FileTreeViewConfig =
new FileTreeViewConfig(
newDataView,
(view: FileTreeDataView[FileCacheEntry], sources: Seq[WatchSource], logger: Logger) =>
newMonitor(view.asInstanceOf[T], sources, logger)
)
/**
* Provides a [[FileTreeViewConfig]] with semantics as close as possible to sbt 1.2.0. This means
* that there is no file tree caching and the sbt.io.FileEventMonitor will use an
* sbt.io.WatchService for monitoring the file system.
* @param delay the maximum delay for which the background thread will poll the
* sbt.io.WatchService for file system events
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @return a [[FileTreeViewConfig]] instance.
*/
def sbt1_2_compat(
delay: FiniteDuration,
antiEntropy: FiniteDuration
): FileTreeViewConfig =
FileTreeViewConfig(
() => FileTreeView.DEFAULT.asDataView(FileCacheEntry.default),
(_: FileTreeDataView[FileCacheEntry], sources, logger) => {
val ioLogger: sbt.io.WatchLogger = msg => logger.debug(msg.toString)
FileEventMonitor.antiEntropy(
new WatchServiceBackedObservable(
WatchState.empty(sources.map(_.toGlob), Watched.createWatchService()),
delay,
FileCacheEntry.default,
closeService = true,
ioLogger
),
antiEntropy,
ioLogger,
50.milliseconds,
10.seconds
)
}
)
/**
* Provides a default [[FileTreeViewConfig]]. This view caches entries and solely relies on
* file system events from the operating system to update its internal representation of the
* file tree.
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @return a [[FileTreeViewConfig]] instance.
*/
def default(antiEntropy: FiniteDuration): FileTreeViewConfig =
FileTreeViewConfig(
() => FileTreeRepository.default(FileCacheEntry.default),
(
repository: FileTreeRepository[FileCacheEntry],
sources: Seq[WatchSource],
logger: Logger
) => {
sources.view.map(_.toGlob).foreach(repository.register)
val copied = new Observable[FileCacheEntry] {
override def addObserver(observer: Observer[FileCacheEntry]): Int =
repository.addObserver(observer)
override def removeObserver(handle: Int): Unit = repository.removeObserver(handle)
override def close(): Unit = {} // Don't close the underlying observable
}
FileEventMonitor.antiEntropy(
copied,
antiEntropy,
msg => logger.debug(msg.toString),
50.milliseconds,
10.seconds
)
}
)
/**
* Provides a default [[FileTreeViewConfig]]. When the pollingSources argument is empty, it
* returns the same config as [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]].
* Otherwise, it returns the same config as [[polling]].
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @param pollingInterval the frequency with which the sbt.io.FileEventMonitor polls the file
* system for the paths included in pollingSources
* @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that
* will be periodically polled for changes during continuous builds.
* @return
*/
def default(
antiEntropy: FiniteDuration,
pollingInterval: FiniteDuration,
pollingSources: Seq[WatchSource]
): FileTreeViewConfig = {
if (pollingSources.isEmpty) default(antiEntropy)
else polling(antiEntropy, pollingInterval, pollingSources)
}
/**
* Provides a polling [[FileTreeViewConfig]]. Unlike the view returned by newDataView in
* [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]],
* the view returned by newDataView will not cache any portion of the file system tree that is is
* covered by the pollingSources parameter. The monitor that is generated by newMonitor, will
* poll these directories for changes rather than relying on file system events from the
* operating system. Any paths that are registered with the view that are not included in the
* pollingSources will be cached and monitored using file system events from the operating system
* in the same way that they are in the default view.
*
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @param pollingInterval the frequency with which the FileEventMonitor polls the file system
* for the paths included in pollingSources
* @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that
* will be periodically polled for changes during continuous builds.
* @return a [[FileTreeViewConfig]] instance.
*/
def polling(
antiEntropy: FiniteDuration,
pollingInterval: FiniteDuration,
pollingSources: Seq[WatchSource],
): FileTreeViewConfig = FileTreeViewConfig(
() => FileTreeRepository.hybrid(FileCacheEntry.default, pollingSources.map(_.toGlob): _*),
(
repository: HybridPollingFileTreeRepository[FileCacheEntry],
sources: Seq[WatchSource],
logger: Logger
) => {
sources.view.map(_.toGlob).foreach(repository.register)
FileEventMonitor
.antiEntropy(
repository.toPollingRepository(pollingInterval, NullWatchLogger),
antiEntropy,
msg => logger.debug(msg.toString),
50.milliseconds,
10.seconds
)
}
)
}

View File

@ -64,9 +64,8 @@ object Watched {
/**
* This trait is used to communicate what the watch should do next at various points in time. It
* is heavily linked to a number of callbacks in [[WatchConfig]]. For example, when the
* sbt.io.FileEventMonitor created by [[FileTreeViewConfig.newMonitor]] detects a changed source
* file, then we expect [[WatchConfig.onWatchEvent]] to return [[Trigger]].
* is heavily linked to a number of callbacks in [[WatchConfig]]. For example, when the event
* monitor detects a changed source we expect [[WatchConfig.onWatchEvent]] to return [[Trigger]].
*/
sealed trait Action
@ -427,11 +426,11 @@ object Watched {
val Configuration =
AttributeKey[Watched]("watched-configuration", "Configures continuous execution.")
def createWatchService(): WatchService = {
def createWatchService(pollDelay: FiniteDuration): WatchService = {
def closeWatch = new MacOSXWatchService()
sys.props.get("sbt.watch.mode") match {
case Some("polling") =>
new PollingWatchService(PollDelay)
new PollingWatchService(pollDelay)
case Some("nio") =>
FileSystems.getDefault.newWatchService()
case Some("closewatch") => closeWatch
@ -440,6 +439,7 @@ object Watched {
FileSystems.getDefault.newWatchService()
}
}
def createWatchService(): WatchService = createWatchService(PollDelay)
}
/**

View File

@ -249,7 +249,7 @@ object Defaults extends BuildCommon {
extraLoggers :== { _ =>
Nil
},
pollingDirectories :== Nil,
pollingGlobs :== Nil,
watchSources :== Nil,
watchProjectSources :== Nil,
skip :== false,
@ -280,12 +280,8 @@ object Defaults extends BuildCommon {
None
},
watchStartMessage := Watched.defaultStartWatch,
fileTreeViewConfig := FileManagement.defaultFileTreeView.value,
fileTreeView := state.value
.get(Keys.globalFileTreeView)
.getOrElse(FileTreeView.DEFAULT.asDataView(FileCacheEntry.default)),
externalHooks := {
val view = fileTreeView.value
val view = FileManagement.dataView.value
compileOptions =>
Some(ExternalHooks(compileOptions, view))
},
@ -640,9 +636,12 @@ object Defaults extends BuildCommon {
)
.getOrElse(watchTriggeredMessage.value)
val logger = watchLogger.value
val repo = FileManagement.repo.value
globs.foreach(repo.register)
val monitor = FileManagement.monitor(repo, watchAntiEntropy.value, logger)
WatchConfig.default(
logger,
fileTreeViewConfig.value.newMonitor(fileTreeView.value, sources, logger),
monitor,
watchHandleInput.value,
watchPreWatch.value,
watchOnEvent.value,
@ -653,7 +652,6 @@ object Defaults extends BuildCommon {
},
watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project),
watch := watchSetting.value,
fileTreeViewConfig := FileManagement.defaultFileTreeView.value
)
def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] =

View File

@ -23,7 +23,7 @@ import sbt.internal.librarymanagement.{ CompatibilityWarningOptions, IvySbt }
import sbt.internal.server.ServerHandler
import sbt.internal.util.{ AttributeKey, SourcePosition }
import sbt.io.FileEventMonitor.Event
import sbt.io.{ FileFilter, FileTreeDataView, TypedPath, WatchService }
import sbt.io._
import sbt.librarymanagement.Configurations.CompilerPlugin
import sbt.librarymanagement.LibraryManagementCodec._
import sbt.librarymanagement._
@ -93,9 +93,9 @@ object Keys {
@deprecated("This is no longer used for continuous execution", "1.3.0")
val watch = SettingKey(BasicKeys.watch)
val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting)
val fileTreeView = taskKey[FileTreeDataView[FileCacheEntry]]("A view of the file system")
val enableGlobalCachingFileTreeRepository = settingKey[Boolean]("Toggles whether or not to create a global cache of the file system that can be used by tasks to quickly list a path").withRank(DSetting)
val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting)
val pollingDirectories = settingKey[Seq[Watched.WatchSource]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting)
val pollingGlobs = settingKey[Seq[Glob]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting)
val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting)
val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting)
val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting)
@ -114,7 +114,6 @@ object Keys {
val watchingMessage = settingKey[WatchState => String]("The message to show when triggered execution waits for sources to change.").withRank(DSetting)
@deprecated("Use watchTriggeredMessage instead", "1.3.0")
val triggeredMessage = settingKey[WatchState => String]("The message to show before triggered execution executes an action after sources change.").withRank(DSetting)
val fileTreeViewConfig = taskKey[FileTreeViewConfig]("Configures how sbt will traverse and monitor the file system.").withRank(BMinusSetting)
// Path Keys
val baseDirectory = settingKey[File]("The base directory. Depending on the scope, this is the base directory for the build, project, configuration, or task.").withRank(AMinusSetting)
@ -468,8 +467,8 @@ object Keys {
@deprecated("No longer used", "1.3.0")
private[sbt] val executeProgress = settingKey[State => TaskProgress]("Experimental task execution listener.").withRank(DTask)
private[sbt] val globalFileTreeView = AttributeKey[FileTreeDataView[FileCacheEntry]](
"globalFileTreeView",
private[sbt] val globalFileTreeRepository = AttributeKey[FileTreeRepository[FileCacheEntry]](
"global-file-tree-repository",
"Provides a view into the file system that may or may not cache the tree in memory",
1000
)

View File

@ -9,6 +9,7 @@ package sbt
import java.io.{ File, IOException }
import java.net.URI
import java.util.concurrent.atomic.AtomicBoolean
import java.util.{ Locale, Properties }
import sbt.BasicCommandStrings.{ Shell, TemplateCommand }
@ -21,8 +22,8 @@ import sbt.internal.inc.ScalaInstance
import sbt.internal.util.Types.{ const, idFun }
import sbt.internal.util._
import sbt.internal.util.complete.Parser
import sbt.io.IO
import sbt.io.syntax._
import sbt.io.{ FileTreeDataView, IO }
import sbt.util.{ Level, Logger, Show }
import xsbti.compile.CompilerCache
@ -852,27 +853,26 @@ object BuiltinCommands {
}
s.put(Keys.stateCompilerCache, cache)
}
private[sbt] def registerGlobalCaches(s: State): State = {
val extracted = Project.extract(s)
private[sbt] def registerGlobalCaches(s: State): State =
try {
val extracted = Project.extract(s)
val cleanedUp = new AtomicBoolean(false)
def cleanup(): Unit = {
s.get(Keys.globalFileTreeView).foreach(_.close())
s.attributes.remove(Keys.globalFileTreeView)
s.get(Keys.globalFileTreeRepository).foreach(_.close())
s.attributes.remove(Keys.globalFileTreeRepository)
s.get(Keys.taskRepository).foreach(_.close())
s.attributes.remove(Keys.taskRepository)
()
}
val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s)
val view: FileTreeDataView[FileCacheEntry] = config.newDataView()
val newState = s.addExitHook(cleanup())
cleanup()
val fileTreeRepository = FileManagement.defaultFileTreeRepository(s, extracted)
val newState = s.addExitHook(if (cleanedUp.compareAndSet(false, true)) cleanup())
newState
.put(Keys.globalFileTreeView, view)
.put(Keys.taskRepository, new TaskRepository.Repr)
.put(Keys.globalFileTreeRepository, fileTreeRepository)
} catch {
case NonFatal(_) => s
}
}
def clearCaches: Command = {
val help = Help.more(ClearCaches, ClearCachesDetailed)

View File

@ -5,38 +5,83 @@
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.io.IOException
package sbt
package internal
import sbt.BasicCommandStrings.ContinuousExecutePrefix
import sbt.Keys._
import sbt._
import sbt.io.FileTreeDataView.Entry
import sbt.internal.io.HybridPollingFileTreeRepository
import sbt.io.FileTreeDataView.{ Entry, Observable, Observer, Observers }
import sbt.io._
import sbt.io.syntax._
import sbt.util.Logger
import scala.concurrent.duration._
private[sbt] object FileManagement {
private[sbt] def defaultFileTreeView: Def.Initialize[Task[FileTreeViewConfig]] = Def.task {
val remaining = state.value.remainingCommands.map(_.commandLine.trim)
private[sbt] def defaultFileTreeRepository(
state: State,
extracted: Extracted
): FileTreeRepository[FileCacheEntry] = {
val pollingGlobs = extracted.getOpt(Keys.pollingGlobs).getOrElse(Nil)
val remaining = state.remainingCommands.map(_.commandLine)
// If the session is interactive or if the commands include a continuous build, then use
// the default configuration. Otherwise, use the sbt1_2_compat config, which does not cache
// anything, which makes it less likely to cause issues with CI.
val interactive = remaining.contains("shell") || remaining.lastOption.contains("iflast shell")
val interactive =
remaining.contains("shell") || remaining.lastOption.contains("iflast shell")
val scripted = remaining.contains("setUpScripted")
val continuous = remaining.lastOption.exists(_.startsWith(ContinuousExecutePrefix))
if (!scripted && (interactive || continuous)) {
FileTreeViewConfig
.default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value)
} else FileTreeViewConfig.sbt1_2_compat(pollInterval.value, watchAntiEntropy.value)
val enableCache = extracted
.getOpt(Keys.enableGlobalCachingFileTreeRepository)
.getOrElse(!scripted && (interactive || continuous))
if (enableCache) {
if (pollingGlobs.isEmpty) FileTreeRepository.default(FileCacheEntry.default)
else FileTreeRepository.hybrid(FileCacheEntry.default, pollingGlobs: _*)
} else {
FileTreeRepository.legacy(
FileCacheEntry.default,
(_: Any) => {},
Watched.createWatchService(extracted.getOpt(Keys.pollInterval).getOrElse(500.milliseconds))
)
}
}
private[sbt] implicit class FileTreeDataViewOps[+T](val fileTreeDataView: FileTreeDataView[T]) {
def register(glob: Glob): Either[IOException, Boolean] = {
fileTreeDataView match {
case r: FileTreeRepository[T] => r.register(glob)
case _ => Right(false)
private[sbt] def monitor(
repository: FileTreeRepository[FileCacheEntry],
antiEntropy: FiniteDuration,
logger: Logger
): FileEventMonitor[FileCacheEntry] = {
// Forwards callbacks to the repository. The close method removes all of these
// callbacks.
val copied: Observable[FileCacheEntry] = new Observable[FileCacheEntry] {
private[this] val observers = new Observers[FileCacheEntry]
val (underlying, needClose) = repository match {
case h: HybridPollingFileTreeRepository[FileCacheEntry] =>
(h.toPollingRepository(antiEntropy, (msg: Any) => logger.debug(msg.toString)), true)
case r => (r, false)
}
private[this] val handle = underlying.addObserver(observers)
override def addObserver(observer: Observer[FileCacheEntry]): Int =
observers.addObserver(observer)
override def removeObserver(handle: Int): Unit = observers.removeObserver(handle)
override def close(): Unit = {
underlying.removeObserver(handle)
if (needClose) underlying.close()
}
}
new FileEventMonitor[FileCacheEntry] {
val monitor =
FileEventMonitor.antiEntropy(
copied,
antiEntropy,
new WatchLogger { override def debug(msg: => Any): Unit = logger.debug(msg.toString) },
50.millis,
10.minutes
)
override def poll(duration: Duration): Seq[FileEventMonitor.Event[FileCacheEntry]] =
monitor.poll(duration)
override def close(): Unit = monitor.close()
}
}
@ -56,6 +101,23 @@ private[sbt] object FileManagement {
}
include.accept(file) && !exclude.accept(file)
}
private[sbt] def repo: Def.Initialize[Task[FileTreeRepository[FileCacheEntry]]] = Def.task {
lazy val msg = s"Tried to get FileTreeRepository for uninitialized state."
state.value.get(Keys.globalFileTreeRepository).getOrElse(throw new IllegalStateException(msg))
}
private[sbt] def dataView: Def.Initialize[Task[FileTreeDataView[FileCacheEntry]]] = Def.task {
state.value
.get(Keys.globalFileTreeRepository)
.map(toDataView)
.getOrElse(FileTreeView.DEFAULT.asDataView(FileCacheEntry.default))
}
private def toDataView(r: FileTreeRepository[FileCacheEntry]): FileTreeDataView[FileCacheEntry] =
new FileTreeDataView[FileCacheEntry] {
private def reg(glob: Glob): FileTreeDataView[FileCacheEntry] = { r.register(glob); r }
override def listEntries(glob: Glob): Seq[Entry[FileCacheEntry]] = reg(glob).listEntries(glob)
override def list(glob: Glob): Seq[TypedPath] = reg(glob).list(glob)
override def close(): Unit = {}
}
private[sbt] def collectFiles(
dirs: ScopedTaskable[Seq[File]],
filter: ScopedTaskable[FileFilter],
@ -63,12 +125,11 @@ private[sbt] object FileManagement {
): Def.Initialize[Task[Seq[File]]] =
Def.task {
val sourceDirs = dirs.toTask.value
val view = fileTreeView.value
val view: FileTreeDataView[FileCacheEntry] = dataView.value
val include = filter.toTask.value
val ex = excludes.toTask.value
val sourceFilter: Entry[FileCacheEntry] => Boolean = entryFilter(include, ex)
sourceDirs.flatMap { dir =>
view.register(dir ** AllPassFilter)
view
.listEntries(dir.toPath ** AllPassFilter)
.flatMap {
@ -84,13 +145,11 @@ private[sbt] object FileManagement {
val include = (includeFilter in unmanagedSources).value
val excl = (excludeFilter in unmanagedSources).value
val baseDir = baseDirectory.value
val view = fileTreeView.value
val r: FileTreeDataView[FileCacheEntry] = dataView.value
if (sourcesInBase.value) {
view.register(baseDir.toPath * AllPassFilter)
val filter: Entry[FileCacheEntry] => Boolean = entryFilter(include, excl)
sources ++
view
.listEntries(baseDir * AllPassFilter)
r.listEntries(baseDir * AllPassFilter)
.flatMap {
case e if filter(e) => e.value.toOption.map(Stamped.file(e.typedPath, _))
case _ => None