luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use FileTreeDataView to collect files 

Now that we have the fileTreeView task, we can generalized the process
of collecting files from the view (which may or may not actually cache
the underlying file tree). I moved the implementation of collectFiles
and addBaseSources into the new FileManagement object because Defaults
is already too large of a file. When we query the view, we also need to
register the directory we're listing because if the underlying view is a
cache, we must call register before any entries will be available.
Because FileTreeDataView doesn't have a register method, I implement
registration with a simple implicit class that pattern matches on the
underlying type and only calls register if it is actually a
FileRepository.

A side effect of this change is that the underlying files returned by
collectFiles and appendBaseSources are StampedFile instances. This is so
that in a subsequent commit, I can add a Zinc external hook that will
read these stamps from the files in the source input array rather than
compute the stamp on the fly. This leads to a substantial reduction in
Zinc startup time for projects with many source files. The file filters
also may be applied more quickly because the isDirectory property (which
we check for all source files) is read from a cached value rather than
requiring a stat.

I had to update a few of the scripted tests to use the `1.2.0`
FileTreeViewConfig because those tests would copy a file and then
immediately re-compile. The latency of cache invalidation is O(1-10ms),
but not instantaneous so it's necessary to either use a non-caching
FileTreeView or add a sleep between updates and compilation. I chose the
former.
This commit is contained in:
Ethan Atkins 2018-08-26 11:16:58 -07:00
parent d31fae59f7
commit 2b2b84f589
4 changed files with 146 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
main-command/src/main/scala/sbt/FileTreeViewConfig.scala
View File

@ -8,10 +8,11 @@
package sbt
import sbt.Watched.WatchSource
import sbt.internal.io.{ WatchServiceBackedObservable, WatchState }
import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView }
import sbt.io._
import FileTreeDataView.{ Observable, Observer }
import sbt.util.Logger
import scala.concurrent.duration.FiniteDuration
import scala.concurrent.duration._
/**
* Configuration for viewing and monitoring the file system.
@ -25,6 +26,11 @@ final class FileTreeViewConfig private (
) => FileEventMonitor[StampedFile]
)
object FileTreeViewConfig {
private implicit class RepositoryOps(val repository: FileTreeRepository[StampedFile]) {
def register(sources: Seq[WatchSource]): Unit = sources foreach { s =>
repository.register(s.base.toPath, if (s.recursive) Integer.MAX_VALUE else 0)
}
}
/**
* Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded
@ -50,14 +56,19 @@ object FileTreeViewConfig {
)
/**
* Provides a default [[FileTreeViewConfig]]. This view does not cache entries.
* @param pollingInterval the maximum duration that the sbt.internal.io.EventMonitor will poll
* the underlying sbt.io.WatchService when monitoring for file events
* Provides a [[FileTreeViewConfig]] with semantics as close as possible to sbt 1.2.0. This means
* that there is no file tree caching and the sbt.io.FileEventMonitor will use an
* sbt.io.WatchService for monitoring the file system.
* @param delay the maximum delay for which the background thread will poll the
* sbt.io.WatchService for file system events
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @return a [[FileTreeViewConfig]] instance.
*/
def default(pollingInterval: FiniteDuration, antiEntropy: FiniteDuration): FileTreeViewConfig =
def sbt1_2_compat(
delay: FiniteDuration,
antiEntropy: FiniteDuration
): FileTreeViewConfig =
FileTreeViewConfig(
() => FileTreeView.DEFAULT.asDataView(StampedFile.converter),
(_: FileTreeDataView[StampedFile], sources, logger) => {
@ -65,7 +76,7 @@ object FileTreeViewConfig {
FileEventMonitor.antiEntropy(
new WatchServiceBackedObservable(
WatchState.empty(Watched.createWatchService(), sources),
pollingInterval,
delay,
StampedFile.converter,
closeService = true,
ioLogger
@ -75,4 +86,27 @@ object FileTreeViewConfig {
)
}
)
/**
* Provides a default [[FileTreeViewConfig]]. This view caches entries and solely relies on
* file system events from the operating system to update its internal representation of the
* file tree.
* @param antiEntropy the duration of the period after a path triggers a build for which it is
* quarantined from triggering another build
* @return a [[FileTreeViewConfig]] instance.
*/
def default(antiEntropy: FiniteDuration): FileTreeViewConfig =
FileTreeViewConfig(
() => FileTreeRepository.default(StampedFile.converter),
(repository: FileTreeRepository[StampedFile], sources: Seq[WatchSource], logger: Logger) => {
repository.register(sources)
val copied = new Observable[StampedFile] {
override def addObserver(observer: Observer[StampedFile]): Int =
repository.addObserver(observer)
override def removeObserver(handle: Int): Unit = repository.removeObserver(handle)
override def close(): Unit = {} // Don't close the underlying observable
}
FileEventMonitor.antiEntropy(copied, antiEntropy, msg => logger.debug(msg.toString))
}
)
}

2
main-command/src/test/scala/sbt/WatchedSpec.scala
View File

@ -23,7 +23,7 @@ import scala.concurrent.duration._
class WatchedSpec extends FlatSpec with Matchers {
object Defaults {
private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis, 50.millis)
private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis)
def config(
sources: Seq[WatchSource],
fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None,

45
main/src/main/scala/sbt/Defaults.scala
View File

@ -271,7 +271,7 @@ object Defaults extends BuildCommon {
None
},
watchStartMessage := Watched.defaultStartWatch,
fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value),
fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value),
fileTreeView := state.value
.get(BasicKeys.globalFileTreeView)
.getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)),
@ -363,11 +363,13 @@ object Defaults extends BuildCommon {
crossPaths.value
)
},
unmanagedSources := collectFiles(
unmanagedSourceDirectories,
includeFilter in unmanagedSources,
excludeFilter in unmanagedSources
).value,
unmanagedSources := FileManagement
.collectFiles(
unmanagedSourceDirectories,
includeFilter in unmanagedSources,
excludeFilter in unmanagedSources
)
.value,
watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ {
val baseDir = baseDirectory.value
val bases = unmanagedSourceDirectories.value
@ -412,11 +414,13 @@ object Defaults extends BuildCommon {
resourceDirectories := Classpaths
.concatSettings(unmanagedResourceDirectories, managedResourceDirectories)
.value,
unmanagedResources := collectFiles(
unmanagedResourceDirectories,
includeFilter in unmanagedResources,
excludeFilter in unmanagedResources
).value,
unmanagedResources := FileManagement
.collectFiles(
unmanagedResourceDirectories,
includeFilter in unmanagedResources,
excludeFilter in unmanagedResources
)
.value,
watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ {
val bases = unmanagedResourceDirectories.value
val include = (includeFilter in unmanagedResources).value
@ -430,19 +434,11 @@ object Defaults extends BuildCommon {
managedResources := generate(resourceGenerators).value,
resources := Classpaths.concat(managedResources, unmanagedResources).value
)
def addBaseSources = FileManagement.appendBaseSources
lazy val outputConfigPaths = Seq(
classDirectory := crossTarget.value / (prefix(configuration.value.name) + "classes"),
target in doc := crossTarget.value / (prefix(configuration.value.name) + "api")
)
def addBaseSources = Seq(
unmanagedSources := {
val srcs = unmanagedSources.value
val f = (includeFilter in unmanagedSources).value
val excl = (excludeFilter in unmanagedSources).value
val baseDir = baseDirectory.value
if (sourcesInBase.value) (srcs +++ baseDir * (f -- excl)).get else srcs
}
)
// This is included into JvmPlugin.projectSettings
def compileBase = inTask(console)(compilersSetting :: Nil) ++ compileBaseGlobal ++ Seq(
@ -659,7 +655,7 @@ object Defaults extends BuildCommon {
},
watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project),
watch := watchSetting.value,
fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value),
fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value)
)
def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] =
@ -1192,10 +1188,7 @@ object Defaults extends BuildCommon {
dirs: ScopedTaskable[Seq[File]],
filter: ScopedTaskable[FileFilter],
excludes: ScopedTaskable[FileFilter]
): Initialize[Task[Seq[File]]] =
Def.task {
dirs.toTask.value.descendantsExcept(filter.toTask.value, excludes.toTask.value).get
}
): Initialize[Task[Seq[File]]] = FileManagement.collectFiles(dirs, filter, excludes)
def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] =
Def.setting {
val f = artifactName.value
@ -1767,7 +1760,7 @@ object Defaults extends BuildCommon {
lazy val compileSettings: Seq[Setting[_]] =
configSettings ++
(mainBgRunMainTask +: mainBgRunTask +: addBaseSources) ++
(mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++
Classpaths.addUnmanagedLibrary
lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks

85
main/src/main/scala/sbt/internal/FileManagement.scala Normal file
View File

@ -0,0 +1,85 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal
import java.io.IOException
import java.nio.file.Path
import sbt.Keys._
import sbt.io.FileTreeDataView.Entry
import sbt.io.syntax.File
import sbt.io.{ FileFilter, FileTreeRepository, FileTreeDataView }
import sbt.{ Def, ScopedTaskable, StampedFile, Task }
private[sbt] object FileManagement {
private[sbt] implicit class FileTreeDataViewOps[+T](val fileTreeDataView: FileTreeDataView[T]) {
def register(path: Path, maxDepth: Int): Either[IOException, Boolean] = {
fileTreeDataView match {
case r: FileTreeRepository[T] => r.register(path, maxDepth)
case _ => Right(false)
}
}
}
private[sbt] def collectFiles(
dirs: ScopedTaskable[Seq[File]],
filter: ScopedTaskable[FileFilter],
excludes: ScopedTaskable[FileFilter]
): Def.Initialize[Task[Seq[File]]] =
Def.task {
val sourceDirs = dirs.toTask.value
val view = fileTreeView.value
val include = filter.toTask.value
val ex = excludes.toTask.value
val sourceFilter: Entry[StampedFile] => Boolean = (entry: Entry[StampedFile]) => {
entry.value match {
case Right(sf) => include.accept(sf) && !ex.accept(sf)
case _ => false
}
}
sourceDirs.flatMap { dir =>
view.register(dir.toPath, maxDepth = Integer.MAX_VALUE)
view
.listEntries(dir.toPath, maxDepth = Integer.MAX_VALUE, sourceFilter)
.map(e => e.value.getOrElse(e.typedPath.getPath.toFile))
}
}
private[sbt] def appendBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Seq(
unmanagedSources := {
val sources = unmanagedSources.value
val f = (includeFilter in unmanagedSources).value
val excl = (excludeFilter in unmanagedSources).value
val baseDir = baseDirectory.value
val view = fileTreeView.value
if (sourcesInBase.value) {
view.register(baseDir.toPath, maxDepth = 0)
sources ++
view
.listEntries(
baseDir.toPath,
maxDepth = 0,
e => {
val tp = e.typedPath
/*
* The TypedPath has the isDirectory and isFile properties embedded. By overriding
* these methods in java.io.File, FileFilters may be applied without needing to
* stat the file (which is expensive) for isDirectory and isFile checks.
*/
val file = new java.io.File(tp.getPath.toString) {
override def isDirectory: Boolean = tp.isDirectory
override def isFile: Boolean = tp.isFile
}
f.accept(file) && !excl.accept(file)
}
)
.flatMap(_.value.toOption)
} else sources
}
)
}