diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/MacroDefaults.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/MacroDefaults.scala index 43526a373..08fed6fa1 100644 --- a/core-macros/src/main/scala/sbt/internal/util/appmacro/MacroDefaults.scala +++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/MacroDefaults.scala @@ -11,18 +11,6 @@ import scala.reflect.macros.blackbox object MacroDefaults { - /** - * Macro to generated default file tree repository. It must be defined as an untyped tree because - * sbt.Keys is not available in this project. This is meant for internal use only, but must be - * public because its a macro. - * @param c the macro context - * @return the tree expressing the default file tree repository. - */ - def fileTreeRepository(c: blackbox.Context): c.Tree = { - import c.universe._ - q"sbt.Keys.fileTreeRepository.value: @sbtUnchecked" - } - /** * Macro to generated default file tree repository. It must be defined as an untyped tree because * sbt.Keys is not available in this project. This is meant for internal use only, but must be diff --git a/main-actions/src/main/scala/sbt/TestResultLogger.scala b/main-actions/src/main/scala/sbt/TestResultLogger.scala index d554fe45f..01640def6 100644 --- a/main-actions/src/main/scala/sbt/TestResultLogger.scala +++ b/main-actions/src/main/scala/sbt/TestResultLogger.scala @@ -8,8 +8,8 @@ package sbt import sbt.Tests.{ Output, Summary } -import sbt.util.{ Level, Logger } import sbt.protocol.testing.TestResult +import sbt.util.{ Level, Logger } /** * Logs information about tests after they finish. diff --git a/main-command/src/main/scala/sbt/Stamped.scala b/main-command/src/main/scala/sbt/Stamped.scala deleted file mode 100644 index 668c27510..000000000 --- a/main-command/src/main/scala/sbt/Stamped.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt - -import java.io.{ File => JFile } -import java.nio.file.Path - -import sbt.internal.FileAttributes -import sbt.internal.inc.{ EmptyStamp, Stamper } -import sbt.io.TypedPath -import xsbti.compile.analysis.Stamp - -/** - * A File that has a compile analysis Stamp value associated with it. In general, the stamp method - * should be a cached value that can be read without doing any io. This can be used to improve - * performance anywhere where we need to check if files have changed before doing potentially - * expensive work. - */ -private[sbt] trait Stamped { - private[sbt] def stamp: Stamp -} - -/** - * Provides converter functions from TypedPath to [[Stamped]]. - */ -private[sbt] object Stamped { - type File = JFile with Stamped - private[sbt] val file: ((Path, FileAttributes)) => JFile with Stamped = { - case (path: Path, attributes: FileAttributes) => - new StampedFileImpl(path, attributes.stamp) - } - - /** - * Converts a TypedPath instance to a [[Stamped]] by calculating the file hash. - */ - private[sbt] val sourceConverter: TypedPath => Stamp = tp => Stamper.forHash(tp.toPath.toFile) - - /** - * Converts a TypedPath instance to a [[Stamped]] using the last modified time. - */ - private[sbt] val binaryConverter: TypedPath => Stamp = tp => - Stamper.forLastModified(tp.toPath.toFile) - - /** - * A combined convert that converts TypedPath instances representing *.jar and *.class files - * using the last modified time and all other files using the file hash. - */ - private[sbt] val converter: TypedPath => Stamp = (_: TypedPath) match { - case typedPath if !typedPath.exists => EmptyStamp - case typedPath if typedPath.isDirectory => binaryConverter(typedPath) - case typedPath => - typedPath.toPath.toString match { - case s if s.endsWith(".jar") => binaryConverter(typedPath) - case s if s.endsWith(".class") => binaryConverter(typedPath) - case _ => sourceConverter(typedPath) - } - } - - /** - * Adds a default ordering that just delegates to the java.io.File.compareTo method. - */ - private[sbt] implicit case object ordering extends Ordering[Stamped.File] { - override def compare(left: Stamped.File, right: Stamped.File): Int = left.compareTo(right) - } - - private final class StampedImpl(override val stamp: Stamp) extends Stamped - private final class StampedFileImpl(path: Path, override val stamp: Stamp) - extends java.io.File(path.toString) - with Stamped -} diff --git a/main-command/src/main/scala/sbt/internal/FileAttributes.scala b/main-command/src/main/scala/sbt/internal/FileAttributes.scala deleted file mode 100644 index 1e7abefbe..000000000 --- a/main-command/src/main/scala/sbt/internal/FileAttributes.scala +++ /dev/null @@ -1,101 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import java.lang -import java.nio.file.Path -import java.util.Optional - -import sbt.Stamped -import sbt.internal.inc.{ EmptyStamp, LastModified, Stamp } -import sbt.io.FileEventMonitor.{ Creation, Deletion, Update } -import sbt.io.{ FileEventMonitor, TypedPath } -import xsbti.compile.analysis.{ Stamp => XStamp } - -/** - * Represents the FileAttributes of a file. This will be moved to io before 1.3.0 is released. - */ -trait FileAttributes { - def hash: Option[String] - def lastModified: Option[Long] - def isRegularFile: Boolean - def isDirectory: Boolean - def isSymbolicLink: Boolean -} -object FileAttributes { - trait Event { - def path: Path - def previous: Option[FileAttributes] - def current: Option[FileAttributes] - } - private[sbt] class EventImpl(event: FileEventMonitor.Event[FileAttributes]) extends Event { - override def path: Path = event.entry.typedPath.toPath - override def previous: Option[FileAttributes] = event match { - case Deletion(entry, _) => entry.value.toOption - case Update(previous, _, _) => previous.value.toOption - case _ => None - } - override def current: Option[FileAttributes] = event match { - case Creation(entry, _) => entry.value.toOption - case Update(_, current, _) => current.value.toOption - case _ => None - } - override def equals(o: Any): Boolean = o match { - case that: Event => - this.path == that.path && this.previous == that.previous && this.current == that.current - case _ => false - } - override def hashCode(): Int = - ((path.hashCode * 31) ^ previous.hashCode() * 31) ^ current.hashCode() - override def toString: String = s"Event($path, $previous, $current)" - } - private[sbt] def default(typedPath: TypedPath): FileAttributes = - DelegateFileAttributes(Stamped.converter(typedPath), typedPath) - private[sbt] implicit class FileAttributesOps(val e: FileAttributes) extends AnyVal { - private[sbt] def stamp: XStamp = e match { - case DelegateFileAttributes(s, _) => s - case _ => - e.hash - .map(Stamp.fromString) - .orElse(e.lastModified.map(new LastModified(_))) - .getOrElse(EmptyStamp) - } - } - - private implicit class Equiv(val xstamp: XStamp) extends AnyVal { - def equiv(that: XStamp): Boolean = Stamp.equivStamp.equiv(xstamp, that) - } - private case class DelegateFileAttributes( - private val stamp: XStamp, - private val typedPath: TypedPath - ) extends FileAttributes - with XStamp { - override def getValueId: Int = stamp.getValueId - override def writeStamp(): String = stamp.writeStamp() - override def getHash: Optional[String] = stamp.getHash - override def getLastModified: Optional[lang.Long] = stamp.getLastModified - override def hash: Option[String] = getHash match { - case h if h.isPresent => Some(h.get) - case _ => None - } - override def lastModified: Option[Long] = getLastModified match { - case l if l.isPresent => Some(l.get) - case _ => None - } - override def equals(o: Any): Boolean = o match { - case DelegateFileAttributes(thatStamp, thatTypedPath) => - (this.stamp equiv thatStamp) && (this.typedPath == thatTypedPath) - case _ => false - } - override def hashCode: Int = stamp.hashCode - override def toString: String = s"FileAttributes(hash = $hash, lastModified = $lastModified)" - override def isRegularFile: Boolean = typedPath.isFile - override def isDirectory: Boolean = typedPath.isDirectory - override def isSymbolicLink: Boolean = typedPath.isSymbolicLink - } -} diff --git a/main-command/src/main/scala/sbt/internal/LegacyWatched.scala b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala index 624372efc..3a818aa9c 100644 --- a/main-command/src/main/scala/sbt/internal/LegacyWatched.scala +++ b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala @@ -8,12 +8,13 @@ package sbt.internal import sbt.BasicCommandStrings.{ ClearOnFailure, FailureWall } +import sbt.Watched.ContinuousEventMonitor import sbt.internal.io.{ EventMonitor, WatchState } +import sbt.internal.nio.{ FileEventMonitor, FileTreeRepository, WatchLogger } import sbt.{ State, Watched } import scala.annotation.tailrec -import Watched.ContinuousEventMonitor - +import scala.concurrent.duration._ import scala.util.control.NonFatal private[sbt] object LegacyWatched { @@ -22,43 +23,63 @@ private[sbt] object LegacyWatched { @tailrec def shouldTerminate: Boolean = (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) val log = s.log - val logger = new EventMonitor.Logger { - override def debug(msg: => Any): Unit = log.debug(msg.toString) - } s get ContinuousEventMonitor match { case None => + val watchState = WatchState.empty(watched.watchService(), watched.watchSources(s)) // This is the first iteration, so run the task and create a new EventMonitor + val logger: WatchLogger = (a: Any) => log.debug(a.toString) + val repo = FileTreeRepository.legacy(logger, watched.watchService()) + val fileEventMonitor = FileEventMonitor.antiEntropy( + repo, + watched.antiEntropy, + logger, + watched.antiEntropy, + 10.minutes + ) + val monitor = new EventMonitor { + override def awaitEvent(): Boolean = fileEventMonitor.poll(2.millis).nonEmpty + override def state(): WatchState = watchState + override def close(): Unit = watchState.close() + } (ClearOnFailure :: next :: FailureWall :: repeat :: s) - .put( - ContinuousEventMonitor, - EventMonitor( - WatchState.empty(watched.watchService(), watched.watchSources(s)), - watched.pollInterval, - watched.antiEntropy, - shouldTerminate, - logger - ) - ) + .put(ContinuousEventMonitor, monitor) case Some(eventMonitor) => Watched.printIfDefined(watched watchingMessage eventMonitor.state) - val triggered = try eventMonitor.awaitEvent() - catch { - case NonFatal(e) => - log.error( - "Error occurred obtaining files to watch. Terminating continuous execution..." - ) - s.handleError(e) - false - } - if (triggered) { - Watched.printIfDefined(watched triggeredMessage eventMonitor.state) - ClearOnFailure :: next :: FailureWall :: repeat :: s - } else { - while (System.in.available() > 0) System.in.read() - eventMonitor.close() - s.remove(ContinuousEventMonitor) + @tailrec def impl(): State = { + val triggered = try eventMonitor.awaitEvent() + catch { + case NonFatal(e) => + log.error( + "Error occurred obtaining files to watch. Terminating continuous execution..." + ) + s.handleError(e) + false + } + if (triggered) { + Watched.printIfDefined(watched triggeredMessage eventMonitor.state) + ClearOnFailure :: next :: FailureWall :: repeat :: s + } else if (shouldTerminate) { + while (System.in.available() > 0) System.in.read() + eventMonitor.close() + s.remove(ContinuousEventMonitor) + } else { + impl() + } } + impl() } } } + +package io { + @deprecated("No longer used", "1.3.0") + private[sbt] trait EventMonitor extends AutoCloseable { + + /** Block indefinitely until the monitor receives a file event or the user stops the watch. */ + def awaitEvent(): Boolean + + /** A snapshot of the WatchState that includes the number of build triggers and watch sources. */ + def state(): WatchState + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 4cb413ea1..8ee43779f 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -9,6 +9,7 @@ package sbt import java.io.{ File, PrintWriter } import java.net.{ URI, URL, URLClassLoader } +import java.nio.file.{ Path => NioPath } import java.util.Optional import java.util.concurrent.{ Callable, TimeUnit } @@ -27,6 +28,7 @@ import sbt.Project.{ } import sbt.Scope.{ GlobalScope, ThisScope, fillTaskAxis } import sbt.internal.CommandStrings.ExportStream +import sbt.internal.TransitiveGlobs._ import sbt.internal._ import sbt.internal.inc.JavaInterfaceUtil._ import sbt.internal.inc.{ ZincLmUtil, ZincUtil } @@ -43,7 +45,6 @@ import sbt.internal.server.{ ServerHandler } import sbt.internal.testing.TestLogger -import sbt.internal.TransitiveGlobs._ import sbt.internal.util.Attributed.data import sbt.internal.util.Types._ import sbt.internal.util._ @@ -65,6 +66,10 @@ import sbt.librarymanagement.CrossVersion.{ binarySbtVersion, binaryScalaVersion import sbt.librarymanagement._ import sbt.librarymanagement.ivy._ import sbt.librarymanagement.syntax._ +import sbt.nio.FileStamp +import sbt.nio.Keys._ +import sbt.nio.file.{ FileTreeView, Glob } +import sbt.nio.file.syntax._ import sbt.std.TaskExtra._ import sbt.testing.{ AnnotatedFingerprint, Framework, Runner, SubclassFingerprint } import sbt.util.CacheImplicits._ @@ -146,6 +151,9 @@ object Defaults extends BuildCommon { classLoaderCache := ClassLoaderCache(4), fileInputs :== Nil, watchTriggers :== Nil, + sbt.nio.Keys.fileAttributeMap := { + new java.util.HashMap[NioPath, (Option[FileStamp.Hash], Option[FileStamp.LastModified])]() + }, ) ++ TaskRepository .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore ) ++ globalSbtCore @@ -191,7 +199,7 @@ object Defaults extends BuildCommon { ps := psTask.value, bgStop := bgStopTask.evaluated, bgWaitFor := bgWaitForTask.evaluated, - bgCopyClasspath :== true + bgCopyClasspath :== true, ) private[sbt] lazy val globalIvyCore: Seq[Setting[_]] = @@ -243,9 +251,13 @@ object Defaults extends BuildCommon { settingsData / fileInputs := { val baseDir = file(".").getCanonicalFile val sourceFilter = ("*.sbt" || "*.scala" || "*.java") -- HiddenFileFilter + val projectDir = baseDir / "project" Seq( - Glob(baseDir, "*.sbt" -- HiddenFileFilter, 0), - Glob(baseDir / "project", sourceFilter, Int.MaxValue) + baseDir * ("*.sbt" -- HiddenFileFilter), + projectDir * sourceFilter, + // We only want to recursively look in source because otherwise we have to search + // the project target directories which is expensive. + projectDir / "src" ** sourceFilter, ) }, trapExit :== true, @@ -296,13 +308,10 @@ object Defaults extends BuildCommon { Previous.references :== new Previous.References, concurrentRestrictions := defaultRestrictions.value, parallelExecution :== true, - fileTreeRepository := state.value - .get(globalFileTreeRepository) - .map(FileTree.repository) - .getOrElse(FileTree.Repository.polling), + fileTreeView :== FileTreeView.default, Continuous.dynamicInputs := Continuous.dynamicInputsImpl.value, externalHooks := { - val repository = fileTreeRepository.value + val repository = fileTreeView.value compileOptions => Some(ExternalHooks(compileOptions, repository)) }, logBuffered :== false, @@ -411,7 +420,9 @@ object Defaults extends BuildCommon { val baseSources = if (sourcesInBase.value) baseDirectory.value * filter :: Nil else Nil unmanagedSourceDirectories.value.map(_ ** filter) ++ baseSources }, - unmanagedSources := (unmanagedSources / fileInputs).value.all.map(Stamped.file), + unmanagedSources := (unmanagedSources / fileInputs).value + .all(fileTreeView.value) + .map(FileStamp.stampedFile), managedSourceDirectories := Seq(sourceManaged.value), managedSources := generate(sourceGenerators).value, sourceGenerators :== Nil, @@ -434,7 +445,8 @@ object Defaults extends BuildCommon { (includeFilter in unmanagedResources).value -- (excludeFilter in unmanagedResources).value unmanagedResourceDirectories.value.map(_ ** filter) }, - unmanagedResources := (unmanagedResources / fileInputs).value.all.map(Stamped.file), + unmanagedResources := + (unmanagedResources / fileInputs).value.all(fileTreeView.value).map(FileStamp.stampedFile), resourceGenerators :== Nil, resourceGenerators += Def.task { PluginDiscovery.writeDescriptors(discoveredSbtPlugins.value, resourceManaged.value) @@ -1228,7 +1240,7 @@ object Defaults extends BuildCommon { exclude: ScopedTaskable[FileFilter] ): Initialize[Task[Seq[File]]] = Def.task { val filter = include.toTask.value -- exclude.toTask.value - dirs.toTask.value.map(_ ** filter).all.map(Stamped.file) + dirs.toTask.value.map(_ ** filter).all(fileTreeView.value).map(FileStamp.stampedFile) } def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] = Def.setting { diff --git a/main/src/main/scala/sbt/EvaluateTask.scala b/main/src/main/scala/sbt/EvaluateTask.scala index d0db3a307..494e90657 100644 --- a/main/src/main/scala/sbt/EvaluateTask.scala +++ b/main/src/main/scala/sbt/EvaluateTask.scala @@ -591,6 +591,14 @@ object EvaluateTask { (dynamicDependency in scoped.scope := { () }) :: Nil } else if (scoped.key == transitiveClasspathDependency.key) { (transitiveClasspathDependency in scoped.scope := { () }) :: Nil + } else if (scoped.key == sbt.nio.Keys.fileInputs.key) { + (sbt.nio.Keys.fileHashes in scoped.scope) := { + import GlobLister._ + val map = sbt.nio.FileStamp.fileHashMap.value + (sbt.nio.Keys.fileInputs in scoped.scope).value.all(fileTreeView.value).collect { + case (p, a) if a.isRegularFile => p -> map.get(p) + } + } } else { Nil } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b256758fd..4e314e26e 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -9,6 +9,7 @@ package sbt import java.io.{ File, InputStream } import java.net.URL +import java.nio.file.{ Path => NioPath } import org.apache.ivy.core.module.descriptor.ModuleDescriptor import org.apache.ivy.core.module.id.ModuleRevisionId @@ -20,15 +21,16 @@ import sbt.internal._ import sbt.internal.inc.ScalaInstance import sbt.internal.io.WatchState import sbt.internal.librarymanagement.{ CompatibilityWarningOptions, IvySbt } +import sbt.internal.nio.FileTreeRepository import sbt.internal.server.ServerHandler import sbt.internal.util.complete.Parser import sbt.internal.util.{ AttributeKey, SourcePosition } -import sbt.io.FileEventMonitor.Event import sbt.io._ import sbt.librarymanagement.Configurations.CompilerPlugin import sbt.librarymanagement.LibraryManagementCodec._ import sbt.librarymanagement._ import sbt.librarymanagement.ivy.{ Credentials, IvyConfiguration, IvyPaths, UpdateOptions } +import sbt.nio.file.{ FileAttributes, FileTreeView, Glob } import sbt.testing.Framework import sbt.util.{ Level, Logger } import xsbti.compile._ @@ -94,7 +96,7 @@ object Keys { val analysis = AttributeKey[CompileAnalysis]("analysis", "Analysis of compilation, including dependencies and generated outputs.", DSetting) val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) - val fileTreeRepository = taskKey[FileTree.Repository]("A repository of the file system.").withRank(DSetting) + val fileTreeView = taskKey[FileTreeView[(NioPath, FileAttributes)]]("A view of the file system.").withRank(DSetting) val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) val pollingGlobs = settingKey[Seq[Glob]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting) val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) @@ -106,12 +108,12 @@ object Keys { val watchInputParser = settingKey[Parser[Watch.Action]]("A parser of user input that can be used to trigger or exit a continuous build").withRank(DSetting) val watchOnEnter = settingKey[() => Unit]("Function to run prior to beginning a continuous build. This will run before the continuous task(s) is(are) first evaluated.").withRank(DSetting) val watchOnExit = settingKey[() => Unit]("Function to run upon exit of a continuous build. It can be used to cleanup resources used during the watch.").withRank(DSetting) - val watchOnInputEvent = settingKey[(Int, Event[FileAttributes]) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the transitive inputs. This is only invoked if watchOnEvent is not explicitly set.").withRank(DSetting) - val watchOnEvent = settingKey[Continuous.Arguments => Event[FileAttributes] => Watch.Action]("Determines how to handle a file event. The Seq[Glob] contains all of the transitive inputs for the task(s) being run by the continuous build.").withRank(DSetting) - val watchOnMetaBuildEvent = settingKey[(Int, Event[FileAttributes]) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the meta build triggers.").withRank(DSetting) + val watchOnInputEvent = settingKey[(Int, Watch.Event) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the transitive inputs. This is only invoked if watchOnEvent is not explicitly set.").withRank(DSetting) + val watchOnEvent = settingKey[Continuous.Arguments => Watch.Event => Watch.Action]("Determines how to handle a file event. The Seq[Glob] contains all of the transitive inputs for the task(s) being run by the continuous build.").withRank(DSetting) + val watchOnMetaBuildEvent = settingKey[(Int, Watch.Event) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the meta build triggers.").withRank(DSetting) val watchOnTermination = settingKey[(Watch.Action, String, Int, State) => State]("Transforms the state upon completion of a watch. The String argument is the command that was run during the watch. The Int parameter specifies how many times the command was run during the watch.").withRank(DSetting) - val watchOnTrigger = settingKey[Continuous.Arguments => Event[FileAttributes] => Unit]("Callback to invoke when a continuous build triggers. The first parameter is the number of previous watch task invocations. The second parameter is the Event that triggered this build").withRank(DSetting) - val watchOnTriggerEvent = settingKey[(Int, Event[FileAttributes]) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the transitive triggers. This is only invoked if watchOnEvent is not explicitly set.").withRank(DSetting) + val watchOnTrigger = settingKey[Continuous.Arguments => Watch.Event => Unit]("Callback to invoke when a continuous build triggers. The first parameter is the number of previous watch task invocations. The second parameter is the Event that triggered this build").withRank(DSetting) + val watchOnTriggerEvent = settingKey[(Int, Watch.Event) => Watch.Action]("Callback to invoke if an event is triggered in a continuous build by one of the transitive triggers. This is only invoked if watchOnEvent is not explicitly set.").withRank(DSetting) val watchOnIteration = settingKey[Int => Watch.Action]("Function that is invoked before waiting for file system events or user input events. This is only invoked if watchOnStart is not explicitly set.").withRank(DSetting) val watchOnStart = settingKey[Continuous.Arguments => () => Watch.Action]("Function is invoked before waiting for file system or input events. The returned Action is used to either trigger the build, terminate the watch or wait for events.").withRank(DSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting).withRank(DSetting) @@ -119,7 +121,7 @@ object Keys { // The watchTasks key should really be named watch, but that is already taken by the deprecated watch key. I'd be surprised if there are any plugins that use it so I think we should consider breaking binary compatibility to rename this task. val watchTasks = InputKey[StateTransform]("watch", "Watch a task (or multiple tasks) and rebuild when its file inputs change or user input is received. The semantics are more or less the same as the `~` command except that it cannot transform the state on exit. This means that it cannot be used to reload the build.").withRank(DSetting) val watchTrackMetaBuild = settingKey[Boolean]("Toggles whether or not changing the build files (e.g. **/*.sbt, project/**/(*.scala | *.java)) should automatically trigger a project reload").withRank(DSetting) - val watchTriggeredMessage = settingKey[(Int, Event[FileAttributes], Seq[String]) => Option[String]]("The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count.").withRank(DSetting) + val watchTriggeredMessage = settingKey[(Int, Watch.Event, Seq[String]) => Option[String]]("The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count.").withRank(DSetting) // Deprecated watch apis @deprecated("This is no longer used for continuous execution", "1.3.0") @@ -149,7 +151,6 @@ object Keys { val managedSources = taskKey[Seq[File]]("Sources generated by the build.").withRank(BTask) val sources = taskKey[Seq[File]]("All sources, both managed and unmanaged.").withRank(BTask) val sourcesInBase = settingKey[Boolean]("If true, sources from the project's base directory are included as main sources.") - val fileInputs = settingKey[Seq[Glob]]("The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution.") val watchTriggers = settingKey[Seq[Glob]]("Describes files that should trigger a new continuous build.") // Filters @@ -175,7 +176,6 @@ object Keys { val cleanKeepGlobs = settingKey[Seq[Glob]]("Globs to keep during a clean. Must be direct children of target.").withRank(CSetting) val crossPaths = settingKey[Boolean]("If true, enables cross paths, which distinguish input and output directories for cross-building.").withRank(ASetting) val taskTemporaryDirectory = settingKey[File]("Directory used for temporary files for tasks that is deleted after each task execution.").withRank(DSetting) - val fileOutputs = taskKey[Seq[Glob]]("Describes the output files of a task") // Generators val sourceGenerators = settingKey[Seq[Task[Seq[File]]]]("List of tasks that generate sources.").withRank(CSetting) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 943a17736..6324af295 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -18,14 +18,15 @@ import sbt.Project.LoadAction import sbt.compiler.EvalImports import sbt.internal.Aggregation.AnyKeys import sbt.internal.CommandStrings.BootCommand -import sbt.internal.FileManagement.CopiedFileTreeRepository import sbt.internal._ import sbt.internal.inc.ScalaInstance +import sbt.internal.nio.FileTreeRepository import sbt.internal.util.Types.{ const, idFun } import sbt.internal.util._ import sbt.internal.util.complete.Parser import sbt.io._ import sbt.io.syntax._ +import sbt.nio.file.FileAttributes import sbt.util.{ Level, Logger, Show } import xsbti.compile.CompilerCache import xsbti.{ AppMain, AppProvider, ComponentProvider, ScalaProvider } @@ -907,17 +908,14 @@ object BuiltinCommands { () } cleanup() - val fileTreeRepository = FileTreeRepository.default(FileAttributes.default) + val fileTreeRepository = FileTreeRepository.default val fileCache = System.getProperty("sbt.io.filecache", "validate") val newState = s .addExitHook(if (cleanedUp.compareAndSet(false, true)) cleanup()) .put(Keys.taskRepository, new TaskRepository.Repr) .put(rawGlobalFileTreeRepository, fileTreeRepository) if (fileCache == "false" || (fileCache != "true" && Util.isWindows)) newState - else { - val copied = new CopiedFileTreeRepository(fileTreeRepository) - newState.put(Keys.globalFileTreeRepository, copied) - } + else newState.put(Keys.globalFileTreeRepository, FileManagement.copy(fileTreeRepository)) } catch { case NonFatal(_) => s } diff --git a/main/src/main/scala/sbt/Watch.scala b/main/src/main/scala/sbt/Watch.scala index 6acebe47b..7fe6b957b 100644 --- a/main/src/main/scala/sbt/Watch.scala +++ b/main/src/main/scala/sbt/Watch.scala @@ -7,14 +7,16 @@ package sbt import java.io.InputStream +import java.nio.file.Path +import java.util.concurrent.TimeUnit import sbt.BasicCommandStrings.ContinuousExecutePrefix -import sbt.internal.FileAttributes import sbt.internal.LabeledFunctions._ -import sbt.internal.util.{ JLine, Util } +import sbt.internal.nio.FileEvent import sbt.internal.util.complete.Parser import sbt.internal.util.complete.Parser._ -import sbt.io.FileEventMonitor.{ Creation, Deletion, Event, Update } +import sbt.internal.util.{ JLine, Util } +import sbt.nio.file.FileAttributes import sbt.util.{ Level, Logger } import scala.annotation.tailrec @@ -22,6 +24,68 @@ import scala.concurrent.duration._ import scala.util.control.NonFatal object Watch { + sealed trait Event { + def path: Path + def previousAttributes: Option[FileAttributes] + def attributes: Option[FileAttributes] + def occurredAt: FiniteDuration + } + private[sbt] object Event { + private implicit class DurationOps(val d: Duration) extends AnyVal { + def finite: FiniteDuration = d match { + case f: FiniteDuration => f + case _ => new FiniteDuration(Long.MaxValue, TimeUnit.MILLISECONDS) + } + } + def fromIO(fileEvent: FileEvent[FileAttributes]): Watch.Event = fileEvent match { + case c @ FileEvent.Creation(p, a) => new Watch.Creation(p, a, c.occurredAt.value.finite) + case d @ FileEvent.Deletion(p, a) => new Watch.Deletion(p, a, d.occurredAt.value.finite) + case u @ FileEvent.Update(p, prev, attrs) => + new Watch.Update(p, prev, attrs, u.occurredAt.value.finite) + } + } + final class Deletion private[sbt] ( + override val path: Path, + private[this] val attrs: FileAttributes, + override val occurredAt: FiniteDuration + ) extends Event { + override def previousAttributes: Option[FileAttributes] = Some(attrs) + override def attributes: Option[FileAttributes] = None + } + object Deletion { + def unapply(deletion: Deletion): Option[(Path, FileAttributes)] = + deletion.previousAttributes.map(a => deletion.path -> a) + } + final class Creation private[sbt] ( + override val path: Path, + private[this] val attrs: FileAttributes, + override val occurredAt: FiniteDuration + ) extends Event { + override def attributes: Option[FileAttributes] = Some(attrs) + override def previousAttributes: Option[FileAttributes] = None + } + object Creation { + def unapply(creation: Creation): Option[(Path, FileAttributes)] = + creation.attributes.map(a => creation.path -> a) + } + final class Update private[sbt] ( + override val path: Path, + private[this] val prevAttrs: FileAttributes, + private[this] val attrs: FileAttributes, + override val occurredAt: FiniteDuration + ) extends Event { + override def previousAttributes: Option[FileAttributes] = Some(prevAttrs) + override def attributes: Option[FileAttributes] = Some(attrs) + } + object Update { + def unapply(update: Update): Option[(Path, FileAttributes, FileAttributes)] = + update.previousAttributes + .zip(update.attributes) + .map { + case (previous, current) => (update.path, previous, current) + } + .headOption + } /** * This trait is used to control the state of [[Watch.apply]]. The [[Watch.Trigger]] action @@ -227,8 +291,8 @@ object Watch { */ @inline private[sbt] def aggregate( - events: Seq[(Action, Event[FileAttributes])] - ): Option[(Action, Event[FileAttributes])] = + events: Seq[(Action, Event)] + ): Option[(Action, Event)] = if (events.isEmpty) None else Some(events.minBy(_._1)) private implicit class StringToExec(val s: String) extends AnyVal { @@ -250,17 +314,16 @@ object Watch { /** * A constant function that returns [[Trigger]]. */ - final val trigger: (Int, Event[FileAttributes]) => Watch.Action = { - (_: Int, _: Event[FileAttributes]) => - Trigger + final val trigger: (Int, Event) => Watch.Action = { (_: Int, _: Event) => + Trigger }.label("Watched.trigger") - def ifChanged(action: Action): (Int, Event[FileAttributes]) => Watch.Action = - (_: Int, event: Event[FileAttributes]) => + def ifChanged(action: Action): (Int, Event) => Watch.Action = + (_: Int, event: Event) => event match { - case Update(prev, cur, _) if prev.value != cur.value => action - case _: Creation[_] | _: Deletion[_] => action - case _ => Ignore + case Update(_, previousAttributes, attributes) if previousAttributes != attributes => action + case _: Creation | _: Deletion => action + case _ => Ignore } /** @@ -369,14 +432,14 @@ object Watch { * `Keys.watchTriggeredMessage := Watched.defaultOnTriggerMessage`, then nothing is logged when * a build is triggered. */ - final val defaultOnTriggerMessage: (Int, Event[FileAttributes], Seq[String]) => Option[String] = - ((_: Int, e: Event[FileAttributes], commands: Seq[String]) => { - val msg = s"Build triggered by ${e.entry.typedPath.toPath}. " + + final val defaultOnTriggerMessage: (Int, Event, Seq[String]) => Option[String] = + ((_: Int, e: Event, commands: Seq[String]) => { + val msg = s"Build triggered by ${e.path}. " + s"Running ${commands.mkString("'", "; ", "'")}." Some(msg) }).label("Watched.defaultOnTriggerMessage") - final val noTriggerMessage: (Int, Event[FileAttributes], Seq[String]) => Option[String] = + final val noTriggerMessage: (Int, Event, Seq[String]) => Option[String] = (_, _, _) => None /** diff --git a/main/src/main/scala/sbt/internal/Clean.scala b/main/src/main/scala/sbt/internal/Clean.scala index 03ec356e3..d487e5170 100644 --- a/main/src/main/scala/sbt/internal/Clean.scala +++ b/main/src/main/scala/sbt/internal/Clean.scala @@ -9,43 +9,64 @@ package sbt package internal import java.io.IOException -import java.nio.file.{ DirectoryNotEmptyException, Files } +import java.nio.file.{ DirectoryNotEmptyException, Files, Path } import sbt.Def._ import sbt.Keys._ import sbt.Project.richInitializeTask +import sbt.io.AllPassFilter import sbt.io.syntax._ -import sbt.io.{ AllPassFilter, FileTreeView, TypedPath } +import sbt.nio.Keys._ +import sbt.nio.file.{ AnyPath, FileAttributes, FileTreeView, Glob } import sbt.util.Level object Clean { - def deleteContents(file: File, exclude: TypedPath => Boolean): Unit = - deleteContents(file, exclude, FileTreeView.DEFAULT, tryDelete((_: String) => {})) - def deleteContents( - file: File, - exclude: TypedPath => Boolean, - view: FileTreeView, - delete: File => Unit + def deleteContents(file: File, exclude: File => Boolean): Unit = + deleteContents( + file.toPath, + path => exclude(path.toFile), + FileTreeView.default, + tryDelete((_: String) => {}) + ) + private[sbt] def deleteContents( + path: Path, + exclude: Path => Boolean, + view: FileTreeView.Nio[FileAttributes], + delete: Path => Unit ): Unit = { - def deleteRecursive(file: File): Unit = { - view.list(file * AllPassFilter).filterNot(exclude).foreach { - case dir if dir.isDirectory => - deleteRecursive(dir.toPath.toFile) - delete(dir.toPath.toFile) - case f => delete(f.toPath.toFile) - } + def deleteRecursive(path: Path): Unit = { + view + .list(Glob(path, AnyPath)) + .filterNot { case (p, _) => exclude(p) } + .foreach { + case (dir, attrs) if attrs.isDirectory => + deleteRecursive(dir) + delete(dir) + case (file, _) => delete(file) + } } - deleteRecursive(file) + deleteRecursive(path) } - /** - * Provides an implementation for the clean task. It delegates to [[taskIn]] using the - * resolvedScoped key to set the scope. - * @return the clean task definition. - */ - def task: Def.Initialize[Task[Unit]] = - Def.taskDyn(taskIn(Keys.resolvedScoped.value.scope)) tag Tags.Clean + private[this] def cleanFilter(scope: Scope): Def.Initialize[Task[Path => Boolean]] = Def.task { + val excludes = (cleanKeepFiles in scope).value.map { + // This mimics the legacy behavior of cleanFilesTask + case f if f.isDirectory => f * AllPassFilter + case f => f.toGlob + } ++ (cleanKeepGlobs in scope).value + p: Path => excludes.exists(_.matches(p)) + } + private[this] def cleanDelete(scope: Scope): Def.Initialize[Task[Path => Unit]] = Def.task { + // Don't use a regular logger because the logger actually writes to the target directory. + val debug = (logLevel in scope).?.value.orElse(state.value.get(logLevel.key)) match { + case Some(Level.Debug) => + (string: String) => println(s"[debug] $string") + case _ => + (_: String) => {} + } + tryDelete(debug) + } /** * Implements the clean task in a given scope. It uses the outputs task value in the provided @@ -58,9 +79,9 @@ object Clean { val excludes = cleanKeepFiles.value.map { // This mimics the legacy behavior of cleanFilesTask case f if f.isDirectory => f * AllPassFilter - case f => f.toGlob + case f => f.glob } ++ cleanKeepGlobs.value - val excludeFilter: TypedPath => Boolean = excludes.toTypedPathFilter + val excludeFilter: Path => Boolean = p => excludes.exists(_.matches(p)) // Don't use a regular logger because the logger actually writes to the target directory. val debug = (logLevel in scope).?.value.orElse(state.value.get(logLevel.key)) match { case Some(Level.Debug) => @@ -69,26 +90,25 @@ object Clean { (_: String) => {} } val delete = tryDelete(debug) - cleanFiles.value.sorted.reverseIterator.foreach(delete) + cleanFiles.value.sorted.reverseIterator.foreach(f => delete(f.toPath)) (fileOutputs in scope).value.foreach { g => - val filter: TypedPath => Boolean = { - val globFilter = g.toTypedPathFilter - tp => !globFilter(tp) || excludeFilter(tp) + val filter: Path => Boolean = { path => + !g.matches(path) || excludeFilter(path) } - deleteContents(g.base.toFile, filter, FileTreeView.DEFAULT, delete) - delete(g.base.toFile) + deleteContents(g.base, filter, FileTreeView.default, delete) + delete(g.base) } } tag Tags.Clean - private def tryDelete(debug: String => Unit): File => Unit = file => { + private def tryDelete(debug: String => Unit): Path => Unit = path => { try { - debug(s"clean -- deleting file $file") - Files.deleteIfExists(file.toPath) + debug(s"clean -- deleting file $path") + Files.deleteIfExists(path) () } catch { case _: DirectoryNotEmptyException => - debug(s"clean -- unable to delete non-empty directory $file") + debug(s"clean -- unable to delete non-empty directory $path") case e: IOException => - debug(s"Caught unexpected exception $e deleting $file") + debug(s"Caught unexpected exception $e deleting $path") } } } diff --git a/main/src/main/scala/sbt/internal/Continuous.scala b/main/src/main/scala/sbt/internal/Continuous.scala index f5e5ef2e3..d8570a265 100644 --- a/main/src/main/scala/sbt/internal/Continuous.scala +++ b/main/src/main/scala/sbt/internal/Continuous.scala @@ -8,7 +8,8 @@ package sbt package internal -import java.io.{ ByteArrayInputStream, InputStream } +import java.io.{ ByteArrayInputStream, InputStream, File => _, _ } +import java.nio.file.Path import java.util.concurrent.atomic.AtomicInteger import sbt.BasicCommandStrings.{ @@ -20,13 +21,14 @@ import sbt.BasicCommandStrings.{ import sbt.BasicCommands.otherCommandParser import sbt.Def._ import sbt.Scope.Global -import sbt.internal.FileManagement.CopiedFileTreeRepository import sbt.internal.LabeledFunctions._ import sbt.internal.io.WatchState +import sbt.internal.nio._ import sbt.internal.util.complete.Parser._ import sbt.internal.util.complete.{ Parser, Parsers } -import sbt.internal.util.{ AttributeKey, Util } -import sbt.io._ +import sbt.internal.util.{ AttributeKey, JLine, Util } +import sbt.nio.Keys.fileInputs +import sbt.nio.file.{ FileAttributes, Glob } import sbt.util.{ Level, _ } import scala.annotation.tailrec @@ -61,6 +63,7 @@ import scala.util.Try * */ object Continuous extends DeprecatedContinuous { + private type Event = FileEvent[FileAttributes] /** * Provides the dynamic inputs to the continuous build callbacks that cannot be stored as @@ -276,12 +279,23 @@ object Continuous extends DeprecatedContinuous { f(commands, s, valid, invalid) } + private[this] def withCharBufferedStdIn[R](f: InputStream => R): R = { + val unwrapped = new FileInputStream(FileDescriptor.in) { + override def close(): Unit = { + getChannel.close() // We don't want to close the System.in file descriptor + } + } + val in = if (Util.isWindows) unwrapped else JLine.terminal.wrapInIfNeeded(unwrapped) + try f(in) + finally in.close() + } + private[sbt] def runToTermination( state: State, command: String, count: Int, isCommand: Boolean - ): State = Watch.withCharBufferedStdIn { in => + ): State = withCharBufferedStdIn { in => val duped = new DupedInputStream(in) implicit val extracted: Extracted = Project.extract(state) val (stateWithRepo, repo) = state.get(Keys.globalFileTreeRepository) match { @@ -290,12 +304,13 @@ object Continuous extends DeprecatedContinuous { val repo = if ("polling" == System.getProperty("sbt.watch.mode")) { val service = new PollingWatchService(extracted.getOpt(Keys.pollInterval).getOrElse(500.millis)) - FileTreeRepository.legacy(FileAttributes.default _, (_: Any) => {}, service) + FileTreeRepository + .legacy((_: Any) => {}, service) } else { state .get(BuiltinCommands.rawGlobalFileTreeRepository) - .map(new CopiedFileTreeRepository(_)) - .getOrElse(FileTreeRepository.default(FileAttributes.default)) + .map(FileManagement.copy) + .getOrElse(FileTreeRepository.default) } (state.put(Keys.globalFileTreeRepository, repo), repo) } @@ -372,8 +387,8 @@ object Continuous extends DeprecatedContinuous { * Aggregates a collection of [[Config]] instances into a single instance of [[Callbacks]]. * This allows us to monitor and respond to changes for all of * the inputs and triggers for each of the tasks that we are monitoring in the continuous build. - * To monitor all of the inputs and triggers, it creates a [[FileEventMonitor]] for each task - * and then aggregates each of the individual [[FileEventMonitor]] instances into an aggregated + * To monitor all of the inputs and triggers, it creates a monitor for each task + * and then aggregates each of the individual monitor instances into an aggregated * instance. It aggregates all of the event callbacks into a single callback that delegates * to each of the individual callbacks. For the callbacks that return a [[Watch.Action]], * the aggregated callback will select the minimum [[Watch.Action]] returned where the ordering @@ -405,7 +420,7 @@ object Continuous extends DeprecatedContinuous { val onEnter = () => configs.foreach(_.watchSettings.onEnter()) val onStart: () => Watch.Action = getOnStart(project, commands, configs, rawLogger, count) val nextInputEvent: () => Watch.Action = parseInputEvents(configs, state, inputStream, logger) - val (nextFileEvent, cleanupFileMonitor): (() => Option[(Event, Watch.Action)], () => Unit) = + val (nextFileEvent, cleanupFileMonitor): (() => Option[(Watch.Event, Watch.Action)], () => Unit) = getFileEvents(configs, rawLogger, state, count, commands) val nextEvent: () => Watch.Action = combineInputAndFileEvents(nextInputEvent, nextFileEvent, logger) @@ -460,25 +475,32 @@ object Continuous extends DeprecatedContinuous { val res = f.view.map(_()).min // Print the default watch message if there are multiple tasks if (configs.size > 1) - Watch.defaultStartWatch(count.get(), project, commands).foreach(logger.info(_)) + Watch + .defaultStartWatch(count.get(), project, commands) + .foreach(logger.info(_)) res } } + private implicit class TraversableGlobOps(val t: Traversable[Glob]) extends AnyVal { + def toFilter: Path => Boolean = p => t.exists(_.matches(p)) + } private def getFileEvents( configs: Seq[Config], logger: Logger, state: State, count: AtomicInteger, commands: Seq[String] - )(implicit extracted: Extracted): (() => Option[(Event, Watch.Action)], () => Unit) = { + )(implicit extracted: Extracted): (() => Option[(Watch.Event, Watch.Action)], () => Unit) = { val trackMetaBuild = configs.forall(_.watchSettings.trackMetaBuild) val buildGlobs = - if (trackMetaBuild) extracted.getOpt(Keys.fileInputs in Keys.settingsData).getOrElse(Nil) + if (trackMetaBuild) extracted.getOpt(fileInputs in Keys.settingsData).getOrElse(Nil) else Nil - val buildFilter = buildGlobs.toEntryFilter + val buildFilter: Path => Boolean = buildGlobs.toFilter val defaultTrigger = if (Util.isWindows) Watch.ifChanged(Watch.Trigger) else Watch.trigger - val onEvent: Event => (Event, Watch.Action) = { + val retentionPeriod = configs.map(_.watchSettings.antiEntropyRetentionPeriod).max + val quarantinePeriod = configs.map(_.watchSettings.deletionQuarantinePeriod).max + val onEvent: Event => (Watch.Event, Watch.Action) = { val f = configs.map { params => val ws = params.watchSettings val oe = ws.onEvent @@ -487,23 +509,25 @@ object Continuous extends DeprecatedContinuous { val onInputEvent = ws.onInputEvent.getOrElse(defaultTrigger) val onTriggerEvent = ws.onTriggerEvent.getOrElse(defaultTrigger) val onMetaBuildEvent = ws.onMetaBuildEvent.getOrElse(Watch.ifChanged(Watch.Reload)) - val triggerFilter = params.triggers.toEntryFilter + val triggerFilter = params.triggers.toFilter val excludedBuildFilter = buildFilter - event: Event => - val inputFilter = params.inputs().toEntryFilter + event: Watch.Event => + val inputFilter = params.inputs().toFilter val c = count.get() - val entry = event.entry Seq[Watch.Action]( - if (inputFilter(entry)) onInputEvent(c, event) else Watch.Ignore, - if (triggerFilter(entry)) onTriggerEvent(c, event) else Watch.Ignore, - if (excludedBuildFilter(entry)) onMetaBuildEvent(c, event) else Watch.Ignore + if (inputFilter(event.path)) onInputEvent(c, event) else Watch.Ignore, + if (triggerFilter(event.path)) onTriggerEvent(c, event) else Watch.Ignore, + if (excludedBuildFilter(event.path)) onMetaBuildEvent(c, event) + else Watch.Ignore ).min } - event: Event => event -> oe(event) + event: Event => + val watchEvent = Watch.Event.fromIO(event) + watchEvent -> oe(watchEvent) } event: Event => f.view.map(_.apply(event)).minBy(_._2) } - val monitor: FileEventMonitor[FileAttributes] = new FileEventMonitor[FileAttributes] { + val monitor: FileEventMonitor[Event] = new FileEventMonitor[Event] { /** * Create a filtered monitor that only accepts globs that have been registered for the @@ -514,44 +538,60 @@ object Continuous extends DeprecatedContinuous { * @return the filtered FileEventMonitor. */ private def filter( - monitor: FileEventMonitor[FileAttributes], + monitor: FileEventMonitor[Event], globs: () => Seq[Glob] - ): FileEventMonitor[FileAttributes] = { - new FileEventMonitor[FileAttributes] { - override def poll(duration: Duration): Seq[FileEventMonitor.Event[FileAttributes]] = - monitor.poll(duration).filter(e => globs().toEntryFilter(e.entry)) + ): FileEventMonitor[Event] = { + new FileEventMonitor[Event] { + override def poll( + duration: Duration, + filter: Event => Boolean + ): Seq[Event] = monitor.poll(duration, filter).filter(e => globs().toFilter(e.path)) override def close(): Unit = monitor.close() } } + private implicit class WatchLogger(val l: Logger) extends sbt.internal.nio.WatchLogger { + override def debug(msg: Any): Unit = l.debug(msg.toString) + } // TODO make this a normal monitor - private[this] val monitors: Seq[FileEventMonitor[FileAttributes]] = + private[this] val monitors: Seq[FileEventMonitor[Event]] = configs.map { config => // Create a logger with a scoped key prefix so that we can tell from which // monitor events occurred. val l = logger.withPrefix(config.key.show) - val monitor: FileEventMonitor[FileAttributes] = - FileManagement.monitor(config.repository, config.watchSettings.antiEntropy, l) - val allGlobs: () => Seq[Glob] = () => (config.inputs() ++ config.triggers).distinct.sorted + val monitor: FileEventMonitor[Event] = + FileEventMonitor.antiEntropy( + config.repository, + config.watchSettings.antiEntropy, + l, + config.watchSettings.deletionQuarantinePeriod, + config.watchSettings.antiEntropyRetentionPeriod + ) + val allGlobs: () => Seq[Glob] = + () => (config.inputs() ++ config.triggers).distinct.sorted filter(monitor, allGlobs) } ++ (if (trackMetaBuild) { val l = logger.withPrefix("meta-build") val antiEntropy = configs.map(_.watchSettings.antiEntropy).max val repo = getRepository(state) buildGlobs.foreach(repo.register) - val monitor = FileManagement.monitor(repo, antiEntropy, l) + val monitor = FileEventMonitor.antiEntropy( + repo, + antiEntropy, + l, + quarantinePeriod, + retentionPeriod + ) filter(monitor, () => buildGlobs) :: Nil } else Nil) - override def poll(duration: Duration): Seq[FileEventMonitor.Event[FileAttributes]] = { - val res = monitors.flatMap(_.poll(0.millis)).toSet.toVector + override def poll(duration: Duration, filter: Event => Boolean): Seq[Event] = { + val res = monitors.flatMap(_.poll(0.millis, filter)).toSet.toVector if (res.isEmpty) Thread.sleep(duration.toMillis) res } override def close(): Unit = monitors.foreach(_.close()) } val watchLogger: WatchLogger = msg => logger.debug(msg.toString) - val retentionPeriod = configs.map(_.watchSettings.antiEntropyRetentionPeriod).max val antiEntropy = configs.map(_.watchSettings.antiEntropy).max - val quarantinePeriod = configs.map(_.watchSettings.deletionQuarantinePeriod).max val antiEntropyMonitor = FileEventMonitor.antiEntropy( monitor, antiEntropy, @@ -564,7 +604,7 @@ object Continuous extends DeprecatedContinuous { * motivation is to allow the user to specify this callback via setting so that, for example, * they can clear the screen when the build triggers. */ - val onTrigger: Event => Unit = { event: Event => + val onTrigger: Watch.Event => Unit = { event: Watch.Event => configs.foreach { params => params.watchSettings.onTrigger.foreach(ot => ot(params.arguments(logger))(event)) } @@ -586,7 +626,7 @@ object Continuous extends DeprecatedContinuous { val min = actions.minBy { case (e, a) => if (builder.nonEmpty) builder.append(", ") - val path = e.entry.typedPath.toPath.toString + val path = e.path builder.append(path) builder.append(" -> ") builder.append(a.toString) @@ -672,10 +712,10 @@ object Continuous extends DeprecatedContinuous { private def combineInputAndFileEvents( nextInputAction: () => Watch.Action, - nextFileEvent: () => Option[(Event, Watch.Action)], + nextFileEvent: () => Option[(Watch.Event, Watch.Action)], logger: Logger ): () => Watch.Action = () => { - val (inputAction: Watch.Action, fileEvent: Option[(Event, Watch.Action)] @unchecked) = + val (inputAction: Watch.Action, fileEvent: Option[(Watch.Event, Watch.Action)] @unchecked) = Seq(nextInputAction, nextFileEvent).map(_.apply()).toIndexedSeq match { case Seq(ia: Watch.Action, fe @ Some(_)) => (ia, fe) case Seq(ia: Watch.Action, None) => (ia, None) @@ -688,7 +728,7 @@ object Continuous extends DeprecatedContinuous { fileEvent .collect { case (event, action) if action != Watch.Ignore => - s"Received file event $action for ${event.entry.typedPath.toPath}." + + s"Received file event $action for ${event.path}." + (if (action != min) s" Dropping in favor of input event: $min" else "") } .foreach(logger.debug(_)) @@ -738,7 +778,7 @@ object Continuous extends DeprecatedContinuous { } } - private type WatchOnEvent = (Int, Event) => Watch.Action + private type WatchOnEvent = (Int, Watch.Event) => Watch.Action /** * Contains all of the user defined settings that will be used to build a [[Callbacks]] @@ -781,7 +821,7 @@ object Continuous extends DeprecatedContinuous { key.get(Keys.watchInputParser).getOrElse(Watch.defaultInputParser) val logLevel: Level.Value = key.get(Keys.watchLogLevel).getOrElse(Level.Info) val onEnter: () => Unit = key.get(Keys.watchOnEnter).getOrElse(() => {}) - val onEvent: Option[Arguments => Event => Watch.Action] = key.get(Keys.watchOnEvent) + val onEvent: Option[Arguments => Watch.Event => Watch.Action] = key.get(Keys.watchOnEvent) val onExit: () => Unit = key.get(Keys.watchOnExit).getOrElse(() => {}) val onInputEvent: Option[WatchOnEvent] = key.get(Keys.watchOnInputEvent) val onIteration: Option[Int => Watch.Action] = key.get(Keys.watchOnIteration) @@ -789,11 +829,11 @@ object Continuous extends DeprecatedContinuous { val onStart: Option[Arguments => () => Watch.Action] = key.get(Keys.watchOnStart) val onTermination: Option[(Watch.Action, String, Int, State) => State] = key.get(Keys.watchOnTermination) - val onTrigger: Option[Arguments => Event => Unit] = key.get(Keys.watchOnTrigger) + val onTrigger: Option[Arguments => Watch.Event => Unit] = key.get(Keys.watchOnTrigger) val onTriggerEvent: Option[WatchOnEvent] = key.get(Keys.watchOnTriggerEvent) val startMessage: StartMessage = getStartMessage(key) val trackMetaBuild: Boolean = key.get(Keys.watchTrackMetaBuild).getOrElse(true) - val triggerMessage: TriggerMessage = getTriggerMessage(key) + val triggerMessage: TriggerMessage[Watch.Event] = getTriggerMessage(key) // Unlike the rest of the settings, InputStream is a TaskKey which means that if it is set, // we have to use Extracted.runTask to get the value. The reason for this is because it is @@ -827,7 +867,9 @@ object Continuous extends DeprecatedContinuous { lazy val default = key.get(Keys.watchStartMessage).getOrElse(Watch.defaultStartWatch) key.get(deprecatedWatchingMessage).map(Left(_)).getOrElse(Right(default)) } - private def getTriggerMessage(key: ScopedKey[_])(implicit e: Extracted): TriggerMessage = { + private def getTriggerMessage( + key: ScopedKey[_] + )(implicit e: Extracted): TriggerMessage[Watch.Event] = { lazy val default = key.get(Keys.watchTriggeredMessage).getOrElse(Watch.defaultOnTriggerMessage) key.get(deprecatedWatchingMessage).map(Left(_)).getOrElse(Right(default)) diff --git a/main/src/main/scala/sbt/internal/DeprecatedContinuous.scala b/main/src/main/scala/sbt/internal/DeprecatedContinuous.scala index 4cea72eb0..6cdff6e19 100644 --- a/main/src/main/scala/sbt/internal/DeprecatedContinuous.scala +++ b/main/src/main/scala/sbt/internal/DeprecatedContinuous.scala @@ -10,10 +10,10 @@ package sbt.internal import sbt.internal.io.{ WatchState => WS } private[internal] trait DeprecatedContinuous { - protected type Event = sbt.io.FileEventMonitor.Event[FileAttributes] protected type StartMessage = Option[Either[WS => String, (Int, String, Seq[String]) => Option[String]]] - protected type TriggerMessage = Either[WS => String, (Int, Event, Seq[String]) => Option[String]] + protected type TriggerMessage[Event] = + Either[WS => String, (Int, Event, Seq[String]) => Option[String]] protected type DeprecatedWatchState = WS protected val deprecatedWatchingMessage = sbt.Keys.watchingMessage protected val deprecatedTriggeredMessage = sbt.Keys.triggeredMessage diff --git a/main/src/main/scala/sbt/internal/ExternalHooks.scala b/main/src/main/scala/sbt/internal/ExternalHooks.scala index 4ee753be5..87cdc32e3 100644 --- a/main/src/main/scala/sbt/internal/ExternalHooks.scala +++ b/main/src/main/scala/sbt/internal/ExternalHooks.scala @@ -10,10 +10,13 @@ package sbt.internal import java.nio.file.Paths import java.util.Optional -import sbt.Stamped import sbt.internal.inc.ExternalLookup +import sbt.io.AllPassFilter import sbt.io.syntax._ -import sbt.io.{ AllPassFilter, TypedPath } +import sbt.nio.FileStamp +import sbt.nio.FileStamp.StampedFile +import sbt.nio.file.syntax._ +import sbt.nio.file.{ FileAttributes, FileTreeView } import xsbti.compile._ import xsbti.compile.analysis.Stamp @@ -21,21 +24,27 @@ import scala.collection.mutable private[sbt] object ExternalHooks { private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_)) - def apply(options: CompileOptions, repo: FileTree.Repository): DefaultExternalHooks = { + def apply( + options: CompileOptions, + view: FileTreeView.Nio[FileAttributes] + ): DefaultExternalHooks = { import scala.collection.JavaConverters._ val sources = options.sources() val cachedSources = new java.util.HashMap[File, Stamp] - val converter: File => Stamp = f => Stamped.sourceConverter(TypedPath(f.toPath)) sources.foreach { - case sf: Stamped => cachedSources.put(sf, sf.stamp) - case f: File => cachedSources.put(f, converter(f)) + case sf: StampedFile => cachedSources.put(sf, sf.stamp) + case f: File => cachedSources.put(f, FileStamp.stamped(f)) } val allBinaries = new java.util.HashMap[File, Stamp] options.classpath.foreach { case f if f.getName.endsWith(".jar") => - repo.get(f.toGlob) foreach { case (p, a) => allBinaries.put(p.toFile, a.stamp) } + view.list(f.toGlob) foreach { + case (p, a) => allBinaries.put(p.toFile, FileStamp(p, a).stamp) + } case f => - repo.get(f ** AllPassFilter) foreach { case (p, a) => allBinaries.put(p.toFile, a.stamp) } + view.list(f ** AllPassFilter) foreach { + case (p, a) => allBinaries.put(p.toFile, FileStamp(p, a).stamp) + } } val lookup = new ExternalLookup { diff --git a/main/src/main/scala/sbt/internal/FileManagement.scala b/main/src/main/scala/sbt/internal/FileManagement.scala index 35734c5be..2be42891d 100644 --- a/main/src/main/scala/sbt/internal/FileManagement.scala +++ b/main/src/main/scala/sbt/internal/FileManagement.scala @@ -9,61 +9,22 @@ package sbt package internal import java.io.IOException +import java.nio.file.Path -import sbt.internal.io.HybridPollingFileTreeRepository -import sbt.io.FileTreeDataView.{ Entry, Observable, Observer, Observers } -import sbt.io.{ FileTreeRepository, _ } -import sbt.util.Logger - -import scala.concurrent.duration._ +import sbt.internal.nio.{ FileEvent, FileTreeRepository, Observable, Observer } +import sbt.nio.file.Glob private[sbt] object FileManagement { - private[sbt] def monitor( - repository: FileTreeRepository[FileAttributes], - antiEntropy: FiniteDuration, - logger: Logger - ): FileEventMonitor[FileAttributes] = { - // Forwards callbacks to the repository. The close method removes all of these - // callbacks. - val copied: Observable[FileAttributes] = new Observable[FileAttributes] { - private[this] val observers = new Observers[FileAttributes] - val underlying = repository match { - case h: HybridPollingFileTreeRepository[FileAttributes] => - h.toPollingRepository(antiEntropy, (msg: Any) => logger.debug(msg.toString)) - case r => r - } - private[this] val handle = underlying.addObserver(observers) - override def addObserver(observer: Observer[FileAttributes]): Int = - observers.addObserver(observer) - override def removeObserver(handle: Int): Unit = observers.removeObserver(handle) - override def close(): Unit = { - underlying.removeObserver(handle) - underlying.close() - } - } - new FileEventMonitor[FileAttributes] { - val monitor = - FileEventMonitor.antiEntropy( - copied, - antiEntropy, - new WatchLogger { override def debug(msg: => Any): Unit = logger.debug(msg.toString) }, - 50.millis, - 10.minutes - ) - override def poll(duration: Duration): Seq[FileEventMonitor.Event[FileAttributes]] = - monitor.poll(duration) - override def close(): Unit = monitor.close() - } - } - - private[sbt] class CopiedFileTreeRepository[T](underlying: FileTreeRepository[T]) + private[sbt] def copy[T](fileTreeRepository: FileTreeRepository[T]): FileTreeRepository[T] = + new CopiedFileTreeRepository[T](fileTreeRepository) + private[this] class CopiedFileTreeRepository[T](underlying: FileTreeRepository[T]) extends FileTreeRepository[T] { - def addObserver(observer: Observer[T]) = underlying.addObserver(observer) - def close(): Unit = {} // Don't close the underlying observable - def list(glob: Glob): Seq[TypedPath] = underlying.list(glob) - def listEntries(glob: Glob): Seq[Entry[T]] = underlying.listEntries(glob) - def removeObserver(handle: Int): Unit = underlying.removeObserver(handle) - def register(glob: Glob): Either[IOException, Boolean] = underlying.register(glob) - def unregister(glob: Glob): Unit = underlying.unregister(glob) + override def list(path: Path): Seq[(Path, T)] = underlying.list(path) + override def close(): Unit = {} + override def register(glob: Glob): Either[IOException, Observable[FileEvent[T]]] = + underlying.register(glob) + override def addObserver(observer: Observer[FileEvent[T]]): AutoCloseable = + underlying.addObserver(observer) + override def toString: String = s"CopiedFileTreeRepository($underlying)" } } diff --git a/main/src/main/scala/sbt/internal/FileTree.scala b/main/src/main/scala/sbt/internal/FileTree.scala index a76e8bb77..3669a1fce 100644 --- a/main/src/main/scala/sbt/internal/FileTree.scala +++ b/main/src/main/scala/sbt/internal/FileTree.scala @@ -8,25 +8,15 @@ package sbt package internal -import java.nio.file.{ Path, WatchService => _ } +import java.nio.file.{ WatchService => _ } import sbt.internal.util.appmacro.MacroDefaults -import sbt.io.FileTreeDataView.Entry -import sbt.io._ +import sbt.nio.file.Glob import scala.collection.mutable import scala.language.experimental.macros object FileTree { - private sealed trait CacheOptions - private case object NoCache extends CacheOptions - private case object UseCache extends CacheOptions - private case object Validate extends CacheOptions - private def toPair( - filter: Entry[FileAttributes] => Boolean - )(e: Entry[FileAttributes]): Option[(Path, FileAttributes)] = - e.value.toOption.flatMap(a => if (filter(e)) Some(e.typedPath.toPath -> a) else None) - trait Repository extends sbt.internal.Repository[Seq, Glob, (Path, FileAttributes)] private[sbt] trait DynamicInputs { def value: Option[mutable.Set[Glob]] } @@ -36,69 +26,4 @@ object FileTree { private final class impl(override val value: Option[mutable.Set[Glob]]) extends DynamicInputs implicit def default: DynamicInputs = macro MacroDefaults.dynamicInputs } - private[sbt] object Repository { - - /** - * Provide a default [[Repository]] that works within a task definition, e.g. Def.task. It's - * implemented as a macro so that it can call `.value` on a TaskKey. Using a macro also allows - * us to use classes that aren't actually available in this project, e.g. sbt.Keys. - * @return a [[Repository]] instance - */ - implicit def default: FileTree.Repository = macro MacroDefaults.fileTreeRepository - private[sbt] object polling extends Repository { - val view = FileTreeView.DEFAULT.asDataView(FileAttributes.default) - override def get(key: Glob): Seq[(Path, FileAttributes)] = - view.listEntries(key).flatMap(toPair(key.toEntryFilter)) - override def close(): Unit = {} - } - } - private class CachingRepository(underlying: FileTreeRepository[FileAttributes]) - extends Repository { - lazy val cacheOptions = System.getProperty("sbt.io.filecache") match { - case "true" => UseCache - case "validate" => Validate - case _ => NoCache - } - override def get(key: Glob): Seq[(Path, FileAttributes)] = { - underlying.register(key) - cacheOptions match { - case Validate => - val res = Repository.polling.get(key) - val filter = key.toEntryFilter - val cacheRes = underlying - .listEntries(key) - .flatMap(e => if (filter(e)) Some(e.typedPath.toPath) else None) - .toSet - val resSet = res.map(_._1).toSet - if (cacheRes != resSet) { - val msg = "Warning: got different files when using the internal file cache compared " + - s"to polling the file system for key: $key.\n" - val fileDiff = cacheRes diff resSet match { - case d if d.nonEmpty => - new Exception("hmm").printStackTrace() - s"Cache had files not found in the file system:\n${d.mkString("\n")}.\n" - case _ => "" - } - val cacheDiff = resSet diff cacheRes match { - case d if d.nonEmpty => - (if (fileDiff.isEmpty) "" else " ") + - s"File system had files not in the cache:\n${d.mkString("\n")}.\n" - case _ => "" - } - val diff = fileDiff + cacheDiff - val instructions = "Please open an issue at https://github.com/sbt/sbt. To disable " + - "this warning, run sbt with -Dsbt.io.filecache=false" - System.err.println(msg + diff + instructions) - } - res - case UseCache => - underlying.listEntries(key).flatMap(toPair(key.toEntryFilter)) - case NoCache => - Repository.polling.get(key) - } - } - override def close(): Unit = underlying.close() - } - private[sbt] def repository(underlying: FileTreeRepository[FileAttributes]): Repository = - new CachingRepository(underlying) } diff --git a/main/src/main/scala/sbt/internal/GlobLister.scala b/main/src/main/scala/sbt/internal/GlobLister.scala index 20970778a..3533e1d6e 100644 --- a/main/src/main/scala/sbt/internal/GlobLister.scala +++ b/main/src/main/scala/sbt/internal/GlobLister.scala @@ -8,14 +8,9 @@ package sbt package internal -import java.io.File import java.nio.file.Path -import java.util.concurrent.ConcurrentSkipListMap -import sbt.io.{ FileFilter, Glob, SimpleFileFilter } - -import scala.collection.JavaConverters._ -import scala.collection.mutable +import sbt.nio.file.{ FileAttributes, FileTreeView, Glob } /** * Retrieve files from a repository. This should usually be an extension class for @@ -24,19 +19,20 @@ import scala.collection.mutable */ private[sbt] sealed trait GlobLister extends Any { - final def all(repository: FileTree.Repository): Seq[(Path, FileAttributes)] = - all(repository, FileTree.DynamicInputs.empty) + final def all(view: FileTreeView.Nio[FileAttributes]): Seq[(Path, FileAttributes)] = { + all(view, FileTree.DynamicInputs.empty) + } /** * Get the sources described this `GlobLister`. The results should not return any duplicate * entries for each path in the result set. * - * @param repository the file tree repository for retrieving the files for a given glob. + * @param view the file tree view * @param dynamicInputs the task dynamic inputs to track for watch. * @return the files described by this `GlobLister`. */ def all( - implicit repository: FileTree.Repository, + implicit view: FileTreeView.Nio[FileAttributes], dynamicInputs: FileTree.DynamicInputs ): Seq[(Path, FileAttributes)] } @@ -55,7 +51,7 @@ private[sbt] trait GlobListers { import GlobListers._ /** - * Generate a GlobLister given a particular [[Glob]]s. + * Generate a GlobLister given a particular `Glob`s. * * @param source the input Glob */ @@ -71,34 +67,6 @@ private[sbt] trait GlobListers { new impl(sources) } private[internal] object GlobListers { - private def covers(left: Glob, right: Glob): Boolean = { - right.base.startsWith(left.base) && { - left.depth == Int.MaxValue || { - val depth = left.base.relativize(right.base).getNameCount - 1 - depth <= left.depth - right.depth - } - } - } - private def aggregate(globs: Traversable[Glob]): Seq[(Glob, Traversable[Glob])] = { - val sorted = globs.toSeq.sorted - val map = new ConcurrentSkipListMap[Path, (Glob, mutable.Set[Glob])] - if (sorted.size > 1) { - sorted.foreach { glob => - map.subMap(glob.base.getRoot, glob.base.resolve(Char.MaxValue.toString)).asScala.find { - case (_, (g, _)) => covers(g, glob) - } match { - case Some((_, (_, globs))) => globs += glob - case None => - val globs = mutable.Set(glob) - val filter: FileFilter = new SimpleFileFilter((file: File) => { - globs.exists(_.toFileFilter.accept(file)) - }) - map.put(glob.base, (Glob(glob.base, filter, glob.depth), globs)) - } - } - map.asScala.values.toIndexedSeq - } else sorted.map(g => g -> (g :: Nil)) - } /** * Implements `GlobLister` given a collection of Globs. If the input collection type @@ -110,14 +78,11 @@ private[internal] object GlobListers { */ private class impl[T <: Traversable[Glob]](val globs: T) extends AnyVal with GlobLister { override def all( - implicit repository: FileTree.Repository, + implicit view: FileTreeView.Nio[FileAttributes], dynamicInputs: FileTree.DynamicInputs ): Seq[(Path, FileAttributes)] = { - aggregate(globs).flatMap { - case (glob, allGlobs) => - dynamicInputs.value.foreach(_ ++= allGlobs) - repository.get(glob) - }.toIndexedSeq + dynamicInputs.value.foreach(_ ++= globs) + view.list(globs) } } } diff --git a/main/src/main/scala/sbt/internal/InputGraph.scala b/main/src/main/scala/sbt/internal/InputGraph.scala index 0620a5d00..1fc39aaaf 100644 --- a/main/src/main/scala/sbt/internal/InputGraph.scala +++ b/main/src/main/scala/sbt/internal/InputGraph.scala @@ -14,7 +14,9 @@ import sbt._ import sbt.internal.io.Source import sbt.internal.util.AttributeMap import sbt.internal.util.complete.Parser -import sbt.io.Glob +import sbt.io.syntax._ +import sbt.nio.Keys._ +import sbt.nio.file.Glob import scala.annotation.tailrec @@ -25,14 +27,11 @@ object TransitiveGlobs { Def.taskKey[(Seq[Glob], Seq[Glob])]("The transitive inputs and triggers for a key") } private[sbt] object InputGraph { - @deprecated("Source is also deprecated.", "1.3.0") private implicit class SourceOps(val source: Source) { - def toGlob: Glob = - Glob( - source.base, - source.includeFilter -- source.excludeFilter, - if (source.recursive) Int.MaxValue else 0 - ) + def toGlob: Glob = { + val filter = source.includeFilter -- source.excludeFilter + if (source.recursive) source.base ** filter else source.base * filter + } } private[sbt] def inputsTask: Def.Initialize[Task[Seq[Glob]]] = Def.task(transitiveGlobs(arguments.value)._1.sorted) diff --git a/main/src/main/scala/sbt/internal/Repository.scala b/main/src/main/scala/sbt/internal/Repository.scala index 63f9f230e..51195f9a2 100644 --- a/main/src/main/scala/sbt/internal/Repository.scala +++ b/main/src/main/scala/sbt/internal/Repository.scala @@ -35,6 +35,7 @@ import scala.collection.JavaConverters._ */ trait Repository[M[_], K, V] extends AutoCloseable { def get(key: K): M[V] + override def close(): Unit = {} } private[sbt] final class MutableRepository[K, V] extends Repository[Option, K, V] { diff --git a/main/src/main/scala/sbt/nio/FileStamp.scala b/main/src/main/scala/sbt/nio/FileStamp.scala new file mode 100644 index 000000000..5ed4c59af --- /dev/null +++ b/main/src/main/scala/sbt/nio/FileStamp.scala @@ -0,0 +1,83 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.nio + +import java.io.{ File, IOException } +import java.nio.file.Path +import java.util + +import sbt.internal.Repository +import sbt.internal.inc.{ EmptyStamp, Stamper, LastModified => IncLastModified } +import sbt.internal.util.AttributeKey +import sbt.io.IO +import sbt.nio.file.FileAttributes +import sbt.{ Def, Task } +import xsbti.compile.analysis.Stamp + +import scala.util.Try + +sealed trait FileStamp +object FileStamp { + private[nio] type Id[T] = T + private[nio] val attributeMapKey = + AttributeKey[util.HashMap[Path, (Option[Hash], Option[LastModified])]]("task-attribute-map") + private[sbt] def fileHashMap: Def.Initialize[Task[Repository[Id, Path, Hash]]] = Def.task { + val attributeMap = Keys.fileAttributeMap.value + path: Path => + attributeMap.get(path) match { + case null => + val h = hash(path) + attributeMap.put(path, (Some(h), None)) + h + case (Some(h), _) => h + case (None, lm) => + val h = hash(path) + attributeMap.put(path, (Some(h), lm)) + h + } + } + private[sbt] final class StampedFile(path: Path, val stamp: Stamp) + extends java.io.File(path.toString) + private[sbt] val stampedFile: ((Path, FileAttributes)) => File = { + case (p: Path, a: FileAttributes) => new StampedFile(p, apply(p, a).stamp) + } + private[sbt] val stamped: File => Stamp = file => { + val path = file.toPath + FileAttributes(path).map(apply(path, _).stamp).getOrElse(EmptyStamp) + } + + private[sbt] implicit class Ops(val fileStamp: FileStamp) { + private[sbt] def stamp: Stamp = fileStamp match { + case f: FileHashImpl => f.xstamp + case LastModified(time) => new IncLastModified(time) + case _ => EmptyStamp + } + } + + private[sbt] val extractor: Try[FileStamp] => FileStamp = (_: Try[FileStamp]).getOrElse(Empty) + private[sbt] val converter: (Path, FileAttributes) => Try[FileStamp] = (p, a) => Try(apply(p, a)) + def apply(path: Path, fileAttributes: FileAttributes): FileStamp = + try { + if (fileAttributes.isDirectory) lastModified(path) + else + path.toString match { + case s if s.endsWith(".jar") => lastModified(path) + case s if s.endsWith(".class") => lastModified(path) + case _ => hash(path) + } + } catch { + case e: IOException => Error(e) + } + def hash(path: Path): Hash = new FileHashImpl(Stamper.forHash(path.toFile)) + def lastModified(path: Path): LastModified = LastModified(IO.getModifiedTimeOrZero(path.toFile)) + private[this] class FileHashImpl(val xstamp: Stamp) extends Hash(xstamp.getHash.orElse("")) + sealed abstract case class Hash private[sbt] (hex: String) extends FileStamp + case class LastModified private[sbt] (time: Long) extends FileStamp + case class Error(exception: IOException) extends FileStamp + case object Empty extends FileStamp +} diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala new file mode 100644 index 000000000..426e55df7 --- /dev/null +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -0,0 +1,28 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.nio + +import java.nio.file.Path + +import sbt.BuildSyntax.{ settingKey, taskKey } +import sbt.nio.file.Glob + +object Keys { + val fileInputs = settingKey[Seq[Glob]]( + "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution." + ) + val fileOutputs = taskKey[Seq[Glob]]("Describes the output files of a task") + val fileHashes = taskKey[Seq[(Path, FileStamp.Hash)]]("Retrieves the hashes for a set of files") + val fileLastModifiedTimes = taskKey[Seq[(Path, FileStamp.LastModified)]]( + "Retrieves the last modified times for a set of files" + ) + private[sbt] val fileAttributeMap = + taskKey[java.util.HashMap[Path, (Option[FileStamp.Hash], Option[FileStamp.LastModified])]]( + "Map of file stamps that may be cleared between task evaluation runs." + ) +} diff --git a/main/src/test/scala/sbt/WatchSpec.scala b/main/src/test/scala/sbt/WatchSpec.scala index c5139a228..1e04f3f20 100644 --- a/main/src/test/scala/sbt/WatchSpec.scala +++ b/main/src/test/scala/sbt/WatchSpec.scala @@ -14,10 +14,10 @@ import java.util.concurrent.atomic.{ AtomicBoolean, AtomicInteger } import org.scalatest.{ FlatSpec, Matchers } import sbt.Watch.{ NullLogger, _ } import sbt.WatchSpec._ -import sbt.internal.FileAttributes -import sbt.io.FileEventMonitor.Event +import sbt.internal.nio.{ FileEvent, FileEventMonitor, FileTreeRepository } import sbt.io._ import sbt.io.syntax._ +import sbt.nio.file.{ FileAttributes, Glob } import sbt.util.Logger import scala.collection.mutable @@ -30,31 +30,26 @@ class WatchSpec extends FlatSpec with Matchers { object TestDefaults { def callbacks( inputs: Seq[Glob], - fileEventMonitor: Option[FileEventMonitor[FileAttributes]] = None, + fileEventMonitor: Option[FileEventMonitor[FileEvent[FileAttributes]]] = None, logger: Logger = NullLogger, parseEvent: () => Watch.Action = () => Ignore, onStartWatch: () => Watch.Action = () => CancelWatch: Watch.Action, - onWatchEvent: Event[FileAttributes] => Watch.Action = _ => Ignore, - triggeredMessage: Event[FileAttributes] => Option[String] = _ => None, + onWatchEvent: FileEvent[FileAttributes] => Watch.Action = _ => Ignore, + triggeredMessage: FileEvent[FileAttributes] => Option[String] = _ => None, watchingMessage: () => Option[String] = () => None ): (NextAction, NextAction) = { - val monitor = fileEventMonitor.getOrElse { - val fileTreeRepository = FileTreeRepository.default(FileAttributes.default) + val monitor: FileEventMonitor[FileEvent[FileAttributes]] = fileEventMonitor.getOrElse { + val fileTreeRepository = FileTreeRepository.default inputs.foreach(fileTreeRepository.register) - val m = - FileEventMonitor.antiEntropy( - fileTreeRepository, - 50.millis, - m => logger.debug(m.toString), - 50.millis, - 10.minutes - ) - new FileEventMonitor[FileAttributes] { - override def poll(duration: Duration): Seq[Event[FileAttributes]] = m.poll(duration) - override def close(): Unit = m.close() - } + FileEventMonitor.antiEntropy( + fileTreeRepository, + 50.millis, + m => logger.debug(m.toString), + 50.millis, + 10.minutes + ) } - val onTrigger: Event[FileAttributes] => Unit = event => { + val onTrigger: FileEvent[FileAttributes] => Unit = event => { triggeredMessage(event).foreach(logger.info(_)) } val onStart: () => Watch.Action = () => { @@ -63,7 +58,7 @@ class WatchSpec extends FlatSpec with Matchers { } val nextAction: NextAction = () => { val inputAction = parseEvent() - val fileActions = monitor.poll(10.millis).map { e: Event[FileAttributes] => + val fileActions = monitor.poll(10.millis).map { e: FileEvent[FileAttributes] => onWatchEvent(e) match { case Trigger => onTrigger(e); Trigger case action => action @@ -113,8 +108,8 @@ class WatchSpec extends FlatSpec with Matchers { val callbacks = TestDefaults.callbacks( inputs = Seq(realDir ** AllPassFilter), onStartWatch = () => if (task.getCount == 2) CancelWatch else Ignore, - onWatchEvent = e => if (e.entry.typedPath.toPath == foo) Trigger else Ignore, - triggeredMessage = e => { queue += e.entry.typedPath.toPath; None }, + onWatchEvent = e => if (e.path == foo) Trigger else Ignore, + triggeredMessage = e => { queue += e.path; None }, watchingMessage = () => { IO.touch(bar.toFile); Thread.sleep(5); IO.touch(foo.toFile) None @@ -132,8 +127,8 @@ class WatchSpec extends FlatSpec with Matchers { val callbacks = TestDefaults.callbacks( inputs = Seq(realDir ** AllPassFilter), onStartWatch = () => if (task.getCount == 3) CancelWatch else Ignore, - onWatchEvent = _ => Trigger, - triggeredMessage = e => { queue += e.entry.typedPath.toPath; None }, + onWatchEvent = e => if (e.path != realDir.toPath) Trigger else Ignore, + triggeredMessage = e => { queue += e.path; None }, watchingMessage = () => { task.getCount match { case 1 => Files.createFile(bar) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4c817fe33..f7f86d7e3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { val baseScalaVersion = scala212 // sbt modules - private val ioVersion = "1.3.0-M7" + private val ioVersion = "1.3.0-M9" private val utilVersion = "1.3.0-M6" private val lmVersion = sys.props.get("sbt.build.lm.version") match { diff --git a/sbt/src/main/scala/sbt/Import.scala b/sbt/src/main/scala/sbt/Import.scala index 03146de4d..e790c863d 100644 --- a/sbt/src/main/scala/sbt/Import.scala +++ b/sbt/src/main/scala/sbt/Import.scala @@ -42,7 +42,6 @@ trait Import { val ExistsFileFilter = sbt.io.ExistsFileFilter val FileFilter = sbt.io.FileFilter type FileFilter = sbt.io.FileFilter - type Glob = sbt.io.Glob val GlobFilter = sbt.io.GlobFilter val Hash = sbt.io.Hash val HiddenFileFilter = sbt.io.HiddenFileFilter diff --git a/sbt/src/sbt-test/actions/clean-keep/build.sbt b/sbt/src/sbt-test/actions/clean-keep/build.sbt index 38f36e178..938067d41 100644 --- a/sbt/src/sbt-test/actions/clean-keep/build.sbt +++ b/sbt/src/sbt-test/actions/clean-keep/build.sbt @@ -1,6 +1,10 @@ +import sbt.nio.file.Glob + cleanKeepFiles ++= Seq( target.value / "keep", target.value / "keepfile" ) cleanKeepGlobs += target.value / "keepdir" ** AllPassFilter +// This is necessary because recursive globs do not include the base directory. +cleanKeepGlobs += Glob(target.value / "keepdir") diff --git a/sbt/src/sbt-test/actions/clean-managed/build.sbt b/sbt/src/sbt-test/actions/clean-managed/build.sbt index ffa46f97d..53e9055be 100644 --- a/sbt/src/sbt-test/actions/clean-managed/build.sbt +++ b/sbt/src/sbt-test/actions/clean-managed/build.sbt @@ -1,3 +1,5 @@ +import sbt.nio.file.syntax._ + Compile / sourceGenerators += Def.task { val files = Seq(sourceManaged.value / "foo.txt", sourceManaged.value / "bar.txt") files.foreach(IO.touch(_)) diff --git a/sbt/src/sbt-test/actions/compile-clean/build.sbt b/sbt/src/sbt-test/actions/compile-clean/build.sbt index 2e29f6de6..e0cccde9b 100644 --- a/sbt/src/sbt-test/actions/compile-clean/build.sbt +++ b/sbt/src/sbt-test/actions/compile-clean/build.sbt @@ -1,2 +1,4 @@ +import sbt.nio.file.syntax._ + cleanKeepGlobs in Compile += ((classDirectory in Compile in compile).value / "X.class").toGlob diff --git a/sbt/src/sbt-test/classloader-cache/snapshot/build.sbt b/sbt/src/sbt-test/classloader-cache/snapshot/build.sbt index a54646805..a43dd3048 100644 --- a/sbt/src/sbt-test/classloader-cache/snapshot/build.sbt +++ b/sbt/src/sbt-test/classloader-cache/snapshot/build.sbt @@ -16,8 +16,8 @@ val snapshot = (project in file(".")).settings( libraryDependencies += "sbt" %% "foo-lib" % "0.1.0-SNAPSHOT", rewriteIvy := { val dir = Def.spaceDelimited().parsed.head - sbt.IO.delete(file("ivy")) - sbt.IO.copyDirectory(file(s"libraries/library-$dir/ivy"), file("ivy")) + sbt.IO.delete(baseDirectory.value / "ivy") + sbt.IO.copyDirectory(baseDirectory.value / s"libraries/library-$dir/ivy", baseDirectory.value / "ivy") Files.walk(file("ivy").getCanonicalFile.toPath).iterator.asScala.foreach { f => Files.setLastModifiedTime(f, FileTime.fromMillis(System.currentTimeMillis + 3000)) } diff --git a/sbt/src/sbt-test/tests/glob-dsl/build.sbt b/sbt/src/sbt-test/tests/glob-dsl/build.sbt index b2f8c1c76..d725e55e7 100644 --- a/sbt/src/sbt-test/tests/glob-dsl/build.sbt +++ b/sbt/src/sbt-test/tests/glob-dsl/build.sbt @@ -1,3 +1,8 @@ +import java.nio.file._ + +import sbt.nio.Keys._ +import sbt.nio.file._ + // The project contains two files: { Foo.txt, Bar.md } in the subdirector base/subdir/nested-subdir // Check that we can correctly extract Foo.txt with a recursive source @@ -5,7 +10,7 @@ val foo = taskKey[Seq[File]]("Retrieve Foo.txt") foo / fileInputs += baseDirectory.value ** "*.txt" -foo := (foo / fileInputs).value.all.map(_._1.toFile) +foo := (foo / fileInputs).value.all(fileTreeView.value).map(_._1.toFile) val checkFoo = taskKey[Unit]("Check that the Foo.txt file is retrieved") @@ -16,7 +21,7 @@ val bar = taskKey[Seq[File]]("Retrieve Bar.md") bar / fileInputs += baseDirectory.value / "base/subdir/nested-subdir" * "*.md" -bar := (bar / fileInputs).value.all.map(_._1.toFile) +bar := (bar / fileInputs).value.all(fileTreeView.value).map(_._1.toFile) val checkBar = taskKey[Unit]("Check that the Bar.md file is retrieved") @@ -32,7 +37,8 @@ val checkAll = taskKey[Unit]("Check that the Bar.md file is retrieved") checkAll := { import sbt.dsl.LinterLevel.Ignore val expected = Set("Foo.txt", "Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) - assert((all / fileInputs).value.all.map(_._1.toFile).toSet == expected) + val actual = (all / fileInputs).value.all(fileTreeView.value).filter(_._2.isRegularFile).map(_._1.toFile).toSet + assert(actual == expected) } val set = taskKey[Seq[File]]("Specify redundant sources in a set") @@ -45,10 +51,10 @@ set / fileInputs ++= Seq( val checkSet = taskKey[Unit]("Verify that redundant sources are handled") checkSet := { - val redundant = (set / fileInputs).value.all.map(_._1.toFile) + val redundant = (set / fileInputs).value.all(fileTreeView.value).map(_._1.toFile) assert(redundant.size == 2) - val deduped = (set / fileInputs).value.toSet[Glob].all.map(_._1.toFile) + val deduped = (set / fileInputs).value.toSet[Glob].all(fileTreeView.value).map(_._1.toFile) val expected = Seq("Bar.md", "Foo.txt").map(baseDirectory.value / "base/subdir/nested-subdir" / _) assert(deduped.sorted == expected) } @@ -56,16 +62,15 @@ checkSet := { val depth = taskKey[Seq[File]]("Specify redundant sources with limited depth") val checkDepth = taskKey[Unit]("Check that the Bar.md file is retrieved") -depth / fileInputs ++= Seq( - sbt.io.Glob(baseDirectory.value / "base", -DirectoryFilter, 2), - sbt.io.Glob(baseDirectory.value / "base" / "subdir", -DirectoryFilter, 1) -) +depth / fileInputs ++= { + Seq( + Glob(baseDirectory.value / "base", AnyPath / AnyPath / "*.md"), + Glob(baseDirectory.value / "base" / "subdir", AnyPath / "*.md"), + ) +} checkDepth := { - val redundant = (depth / fileInputs).value.all.map(_._1.toFile) - assert(redundant.size == 2) - - val deduped = (depth / fileInputs).value.toSet[Glob].all.map(_._1.toFile) - val expected = Seq("Bar.md", "Foo.txt").map(baseDirectory.value / "base/subdir/nested-subdir" / _) - assert(deduped.sorted == expected) + val expected = Seq("Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) + val actual = (depth / fileInputs).value.all(fileTreeView.value).map(_._1.toFile) + assert(actual == expected) } diff --git a/sbt/src/sbt-test/tests/inputs/build.sbt b/sbt/src/sbt-test/tests/inputs/build.sbt index 54e1ce2fb..244ffc7a2 100644 --- a/sbt/src/sbt-test/tests/inputs/build.sbt +++ b/sbt/src/sbt-test/tests/inputs/build.sbt @@ -1,6 +1,8 @@ import java.nio.file.Path -import sbt.internal.{FileAttributes, FileTree} +import sbt.Keys._ +import sbt.nio.file._ +import sbt.nio.Keys._ val allInputs = taskKey[Seq[File]]("") val allInputsExplicit = taskKey[Seq[File]]("") @@ -8,7 +10,7 @@ val allInputsExplicit = taskKey[Seq[File]]("") val checkInputs = inputKey[Unit]("") val checkInputsExplicit = inputKey[Unit]("") -allInputs := (Compile / unmanagedSources / fileInputs).value.all.map(_._1.toFile) +allInputs := (Compile / unmanagedSources / fileInputs).value.all(fileTreeView.value).map(_._1.toFile) checkInputs := { val res = allInputs.value @@ -20,17 +22,16 @@ checkInputs := { // In this test we override the FileTree.Repository used by the all method. allInputsExplicit := { val files = scala.collection.mutable.Set.empty[File] - val underlying = implicitly[FileTree.Repository] - val repo = new FileTree.Repository { - override def get(glob: Glob): Seq[(Path, FileAttributes)] = { - val res = underlying.get(glob) + val underlying = fileTreeView.value + val view = new FileTreeView[(Path, FileAttributes)] { + override def list(path: Path): Seq[(Path, FileAttributes)] = { + val res = underlying.list(path) files ++= res.map(_._1.toFile) res } - override def close(): Unit = {} } val include = (Compile / unmanagedSources / includeFilter).value - val _ = (Compile / unmanagedSources / fileInputs).value.all(repo).map(_._1.toFile).toSet + val _ = (Compile / unmanagedSources / fileInputs).value.all(view).map(_._1.toFile).toSet files.filter(include.accept).toSeq } diff --git a/sbt/src/sbt-test/tests/interproject-inputs/build.sbt b/sbt/src/sbt-test/tests/interproject-inputs/build.sbt index 61996f5be..52d7e283b 100644 --- a/sbt/src/sbt-test/tests/interproject-inputs/build.sbt +++ b/sbt/src/sbt-test/tests/interproject-inputs/build.sbt @@ -1,4 +1,7 @@ import sbt.internal.TransitiveGlobs._ + +import sbt.nio.Keys._ + val cached = settingKey[Unit]("") val newInputs = settingKey[Unit]("") Compile / cached / fileInputs := (Compile / unmanagedSources / fileInputs).value ++ diff --git a/sbt/src/sbt-test/watch/custom-config/project/Build.scala b/sbt/src/sbt-test/watch/custom-config/project/Build.scala index 4dc816d21..35f46e9c9 100644 --- a/sbt/src/sbt-test/watch/custom-config/project/Build.scala +++ b/sbt/src/sbt-test/watch/custom-config/project/Build.scala @@ -2,6 +2,7 @@ package sbt.input.aggregation import sbt._ import Keys._ +import sbt.nio.Keys._ object Build { val setStringValue = inputKey[Unit]("set a global string to a value") diff --git a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala index bc2ecaea4..616bec212 100644 --- a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala +++ b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala @@ -2,7 +2,7 @@ package sbt.watch.task import sbt._ import Keys._ -import sbt.internal.FileTree +import sbt.nio.Keys._ object Build { val reloadFile = settingKey[File]("file to toggle whether or not to reload") @@ -25,18 +25,23 @@ object Build { setStringValue := Def.taskDyn { // This hides foo / fileInputs from the input graph Def.taskDyn { - val _ = (foo / fileInputs).value.all + val _ = (foo / fileInputs).value + .all(fileTreeView.value, sbt.internal.Continuous.dynamicInputs.value) // By putting setStringValueImpl.value inside a Def.task, we ensure that // (foo / fileInputs).value is registered with the file repository before modifying the file. Def.task(setStringValueImpl.value) } }.value, checkStringValue := checkStringValueImpl.evaluated, - watchOnInputEvent := { (_, _) => Watch.CancelWatch }, - watchOnTriggerEvent := { (_, _) => Watch.CancelWatch }, + watchOnInputEvent := { (_, _) => + Watch.CancelWatch + }, + watchOnTriggerEvent := { (_, _) => + Watch.CancelWatch + }, watchTasks := Def.inputTask { val prev = watchTasks.evaluated new StateTransform(prev.state.fail) }.evaluated ) -} \ No newline at end of file +} diff --git a/sbt/src/sbt-test/watch/input-aggregation/project/Build.scala b/sbt/src/sbt-test/watch/input-aggregation/project/Build.scala index d99c46b54..0c434f315 100644 --- a/sbt/src/sbt-test/watch/input-aggregation/project/Build.scala +++ b/sbt/src/sbt-test/watch/input-aggregation/project/Build.scala @@ -1,8 +1,10 @@ package sbt.input.aggregation +import sbt.Keys._ import sbt._ -import Keys._ import sbt.internal.TransitiveGlobs._ +import sbt.nio.Keys._ +import sbt.nio.file._ object Build { val setStringValue = inputKey[Unit]("set a global string to a value")