From 3e62b983a2996e7ef843483098cf5d0e960675ab Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Mon, 20 Aug 2018 13:26:52 -0700 Subject: [PATCH 01/17] Remove intellij warnings --- main-command/src/main/scala/sbt/Watched.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index 1331fd97d..f107a09d7 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -89,12 +89,13 @@ object Watched { def multi(base: Watched, paths: Seq[Watched]): Watched = new AWatched { - override def watchSources(s: State) = (base.watchSources(s) /: paths)(_ ++ _.watchSources(s)) + override def watchSources(s: State): Seq[Watched.WatchSource] = + (base.watchSources(s) /: paths)(_ ++ _.watchSources(s)) override def terminateWatch(key: Int): Boolean = base.terminateWatch(key) - override val pollInterval = (base +: paths).map(_.pollInterval).min - override val antiEntropy = (base +: paths).map(_.antiEntropy).min - override def watchingMessage(s: WatchState) = base.watchingMessage(s) - override def triggeredMessage(s: WatchState) = base.triggeredMessage(s) + override val pollInterval: FiniteDuration = (base +: paths).map(_.pollInterval).min + override val antiEntropy: FiniteDuration = (base +: paths).map(_.antiEntropy).min + override def watchingMessage(s: WatchState): String = base.watchingMessage(s) + override def triggeredMessage(s: WatchState): String = base.triggeredMessage(s) } def empty: Watched = new AWatched From da54e2fbd36e3fd5c7201a8a55855cf83acd136f Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 17 Aug 2018 17:03:25 -0700 Subject: [PATCH 02/17] Deprecate unused method --- main-command/src/main/scala/sbt/Watched.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index f107a09d7..00566febc 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -87,6 +87,7 @@ object Watched { private[this] class AWatched extends Watched + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") def multi(base: Watched, paths: Seq[Watched]): Watched = new AWatched { override def watchSources(s: State): Seq[Watched.WatchSource] = From 6ffb4108e7b9df83a2a07e7e1d2d7b11eba3f82e Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 24 Aug 2018 13:06:16 -0700 Subject: [PATCH 03/17] Set swoval.tmpdir property on startup This change makes the temporary shared library that is created by the swoval file-tree-views library to be extracted into the sbt global base directory rather than the temp file. This way if there is a leak of shared libraries, they can easily be found in ~/.sbt rather than in, say, /tmp (or the osx/windows equivalent location). The extracted shared library objects will be in the path ~/.sbt/swoval-jni. There is a shutdown hook that removes them as well as a garbage collection process that runs in the background whenever the swoval library is loaded, so these shouldn't leak uncontrollably. --- main/src/main/scala/sbt/internal/Load.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index afa0125f4..cf62b1fc6 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -113,6 +113,7 @@ private[sbt] object Load { val delegates = defaultDelegates val pluginMgmt = PluginManagement(loader) val inject = InjectSettings(injectGlobal(state), Nil, const(Nil)) + System.setProperty("swoval.tmpdir", System.getProperty("swoval.tmpdir", globalBase.toString)) LoadBuildConfiguration( stagingDirectory, classpath, From 97598efceaae1cbdf9d47796fe481a3b044a16c4 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 18 Aug 2018 10:04:50 -0700 Subject: [PATCH 04/17] Bump watch anti-entropy There have been reports that often a new build will be triggered immediately after the previous build even when none of the files have been modified since the start of the last build. This can happen when, for example, a program implements save with a rename. When that occurs, a deletion watch event may trigger the build but the corresponding creation event may be detected outside of the current 40ms window. By bumping this value to 500ms, we hopefully prevent the majority of these false triggers. For unusual workflows in which this longer quarantine period is an issue, the setting can be overridden. --- main/src/main/scala/sbt/Defaults.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 8d721792d..c410b17c6 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -264,7 +264,7 @@ object Defaults extends BuildCommon { concurrentRestrictions := defaultRestrictions.value, parallelExecution :== true, pollInterval :== new FiniteDuration(500, TimeUnit.MILLISECONDS), - watchAntiEntropy :== new FiniteDuration(40, TimeUnit.MILLISECONDS), + watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), watchService :== { () => Watched.createWatchService() }, From 5e3f72ad8a37d1d1318e23b84773bb6f32f1063c Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Mon, 20 Aug 2018 12:55:17 -0700 Subject: [PATCH 05/17] Do not append default watch sources with ++= Using ++= prevents these from being overridden by the user without doing some nasty runtime manipulations. --- main/src/main/scala/sbt/Defaults.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index c410b17c6..9bdbfd755 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -356,7 +356,7 @@ object Defaults extends BuildCommon { includeFilter in unmanagedSources, excludeFilter in unmanagedSources ).value, - watchSources in ConfigGlobal ++= { + watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val baseDir = baseDirectory.value val bases = unmanagedSourceDirectories.value val include = (includeFilter in unmanagedSources).value @@ -398,7 +398,7 @@ object Defaults extends BuildCommon { includeFilter in unmanagedResources, excludeFilter in unmanagedResources ).value, - watchSources in ConfigGlobal ++= { + watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val bases = unmanagedResourceDirectories.value val include = (includeFilter in unmanagedResources).value val exclude = (excludeFilter in unmanagedResources).value From 28aa1de32a1108e8b42cf7606c962df6e5dc6f52 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 18 Aug 2018 10:56:43 -0700 Subject: [PATCH 06/17] Refactor continuous execution This commit makes watch event logging work in the '~' command. The previous design of the command made this difficult, so there is a significant re-design of the implementation of '~'. I believe that this redesign will allow the feature to be maintained and improved more easily moving forward. With the redesign, it is now possible to test the business logic of the watch command (and I add a rudimentary test that I will build upon in subsequent commits). A bonus of this redesign is that now if the user tries to watch an invalid command, the watch will immediately terminate with an error rather than get stuck waiting for events when the task can never possibly succeed. The previous implementation of the '~' command makes it difficult to dynamically control the implementation arguments because it is implemented in the command project which makes it unable to depend on any task keys that are defined in the build. It works around this by putting all of it's configuration in the Watched attribute which is stored globally. This would not have been necessary if the function had been defined in the main project where it could just extract the value of the watched task rather than relying on the global attribute value. Moreover, because it cannot depend on tasks, it makes it nigh impossible to use the logging framework within the '~' command. Another issue with the previous implementation is that it's somewhat difficult to reason about. The executeContinuously has effectively two entry points: one for the first time the command is run and one for each subsequent invocation when a new build is triggered. The successive invocations are triggered by prepending commands to run to the previous state. This is made recursive by prepending the initial command (that was prefixed with '~'. Which branch we're in is determined by checking for the existence of a temporary attribute, that we must ensure that we remove when the build is stopped. This makes a lot of behavior non-local and difficult for an outsider who is less familiar with sbt to understand. Broadly, this refactor does two things: 1) Move the definition of continuous from BasicCommands to BuiltInCommands 2) Re-work the implementation to be executed in code rather than using the sbt dsl. The first part is simple. We just add an implementation of continuous to BuiltInCommands and remove it from the list of BasicCommands. We need to leave in the legacy implementation for binary compatibility. I also moved all of the actual implementation logic into Watched, which makes maintenance easier since most of the logic is in one place. The second part is more complicated. Rather than rely on the sbt dsl (e.g. `(ClearOnFailure :: next :: FailureWall :: repeat :: s)`) to parse and run the command. We manually parse the command and generate a task of type `() => State`. We don't actually need to do anything with the generated state because we're going to return the original state at the end of the command no matter what. With this task, we can then create a tail recursive function that repeatedly executes the task until the watch is terminated. The parsing is handled in the Watch.command method (which is where I moved the refactored BasicCommands.continuous implementation). The actual task running and monitoring is handled in Watched.watch. This method has no reference to the sbt state, which makes it testable. It sets up an event monitor and then delegates the recursive monitoring to a small nested function, Watched.watch.impl. One nice thing about this approach is that it is very easy to reason about the life cycle of the EventMonitor. The recursive call is within a try { } finally { } where the monitor and stdin are guaranteed to be cleared at the end. Adding support for a custom (and default) watch logger is trivial with the new infrastructure and is done via the watchLogger TaskKey. There was a small reporting race condition that was introduced by the change to (2). Because the new implementation is able to bypass command parsing for triggered builds, the watch message would usually end up being printed before the task outcome was fully logged. To work around this, I made the watch and triggered messages be logged rather than printed directly to stdout. As a result, the only user visible result of this change should be that instead of seeing: "1. Waiting for source changes in project foo... (press enter to interrupt)", users will now see: "[info] 1. Waiting for source changes in project foo... (press enter to interrupt)". --- .../src/main/scala/sbt/BasicCommands.scala | 2 +- main-command/src/main/scala/sbt/Watched.scala | 236 +++++++++++++++++- .../src/test/scala/sbt/WatchedSpec.scala | 42 ++++ main/src/main/scala/sbt/Defaults.scala | 1 + main/src/main/scala/sbt/Keys.scala | 3 +- main/src/main/scala/sbt/Main.scala | 11 +- main/src/main/scala/sbt/MainLoop.scala | 11 +- 7 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 main-command/src/test/scala/sbt/WatchedSpec.scala diff --git a/main-command/src/main/scala/sbt/BasicCommands.scala b/main-command/src/main/scala/sbt/BasicCommands.scala index 2351b185b..0e8c51a41 100644 --- a/main-command/src/main/scala/sbt/BasicCommands.scala +++ b/main-command/src/main/scala/sbt/BasicCommands.scala @@ -50,7 +50,6 @@ object BasicCommands { call, early, exit, - continuous, history, oldshell, client, @@ -254,6 +253,7 @@ object BasicCommands { def exit: Command = Command.command(TerminateAction, exitBrief, exitBrief)(_ exit true) + @deprecated("Replaced by BuiltInCommands.continuous", "1.3.0") def continuous: Command = Command(ContinuousExecutePrefix, continuousBriefHelp, continuousDetail)(otherCommandParser) { (s, arg) => diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index 00566febc..dcddda9d6 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -10,12 +10,15 @@ package sbt import java.io.File import java.nio.file.FileSystems -import sbt.BasicCommandStrings.ClearOnFailure -import sbt.State.FailureWall +import sbt.BasicCommandStrings._ +import sbt.BasicCommands.otherCommandParser +import sbt.CommandUtil.withAttribute import sbt.internal.io.{ EventMonitor, Source, WatchState } import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const +import sbt.internal.util.complete.DefaultParsers import sbt.io._ +import sbt.util.{ Level, Logger } import scala.annotation.tailrec import scala.concurrent.duration._ @@ -103,8 +106,127 @@ object Watched { val PollDelay: FiniteDuration = 500.milliseconds val AntiEntropy: FiniteDuration = 40.milliseconds def isEnter(key: Int): Boolean = key == 10 || key == 13 - def printIfDefined(msg: String) = if (!msg.isEmpty) System.out.println(msg) + def printIfDefined(msg: String): Unit = if (!msg.isEmpty) System.out.println(msg) + type Task = () => State + type Setup = (State, Watched, String) => (State, Logger, Task => State) + + /** + * Provides the '~' continuous execution command. + * @param setup a function that provides a logger and a function from (() => State) => State. + * @return the '~' command. + */ + def continuous(setup: Setup): Command = + Command(ContinuousExecutePrefix, continuousBriefHelp, continuousDetail)(otherCommandParser) { + (state, command) => + Watched.command(state, command, setup) + } + + /** + * Implements continuous execution. It works by first parsing the command and generating a task to + * run with each build. It can run multiple commands that are separated by ";" in the command + * input. If any of these commands are invalid, the watch will immmediately exit. + * @param state the initial state + * @param command the command(s) to repeatedly apply + * @param setup function to generate a logger and a transformation of the resultant state. The + * purpose of the transformation is to preserve the logging semantics that existed + * in the legacy version of this function in which the task would be run through + * MainLoop.processCommand, which is unavailable in the main-command project + * @return the initial state if all of the input commands are valid. Otherwise, returns the + * initial state with the failure transformation. + */ + private[sbt] def command( + state: State, + command: String, + setup: Setup, + ): State = + withAttribute(state, Watched.Configuration, "Continuous execution not configured.") { w => + val (s0, logger, process) = setup(state, w, command) + val s = FailureWall :: s0 + val parser = Command.combine(s.definedCommands)(s) + val commands = command.split(";") match { + case Array("", rest @ _*) => rest + case Array(cmd) => Seq(cmd) + } + val tasks = commands.foldLeft(Nil: Seq[Either[String, () => Either[Exception, Boolean]]]) { + case (t, cmd) => + t :+ (DefaultParsers.parse(cmd, parser) match { + case Right(task) => + Right { () => + try { + process(task) + Right(true) + } catch { case e: Exception => Left(e) } + } + case Left(_) => Left(cmd) + }) + } + val (valid, invalid) = tasks.partition(_.isRight) + if (invalid.isEmpty) { + val task = () => + valid.foldLeft(Right(true): Either[Exception, Boolean]) { + case (status, Right(t)) => if (status.getOrElse(true)) t() else status + case _ => throw new IllegalStateException("Should be unreachable") + } + @tailrec def shouldTerminate: Boolean = + (System.in.available > 0) && (w.terminateWatch(System.in.read()) || shouldTerminate) + val watchState = WatchState.empty(w.watchService(), w.watchSources(s)) + val config = WatchConfig.default( + logger, + () => shouldTerminate, + count => Some(w.triggeredMessage(watchState.withCount(count))).filter(_.nonEmpty), + count => Some(w.watchingMessage(watchState.withCount(count))).filter(_.nonEmpty), + watchState, + w.pollInterval, + w.antiEntropy + ) + watch(task, config) + state + } else { + logger.error( + s"Terminating watch due to invalid command(s): ${invalid.mkString("'", "', '", "'")}" + ) + state.fail + } + } + + private[sbt] def watch( + task: () => Either[Exception, _], + config: WatchConfig, + ): Unit = { + val eventLogger = new EventMonitor.Logger { + override def debug(msg: => Any): Unit = config.logger.debug(msg.toString) + } + def debug(msg: String): Unit = if (msg.nonEmpty) config.logger.debug(msg) + val monitor = EventMonitor( + config.watchState, + config.pollInterval, + config.antiEntropy, + config.shouldTerminate(), + eventLogger + ) + + @tailrec + def impl(count: Int): Unit = { + task() match { + case _: Right[Exception, _] => + config.watchingMessage(count).foreach(debug) + if (monitor.awaitEvent()) { + config.triggeredMessage(count).foreach(debug) + impl(count + 1) + } + case Left(e) => config.logger.error(s"Terminating watch due to Unexpected error: $e") + } + } + try { + impl(count = 1) + } finally { + monitor.close() + while (System.in.available() > 0) System.in.read() + } + } + + @deprecated("Replaced by Watched.command", "1.3.0") def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = { @tailrec def shouldTerminate: Boolean = (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) @@ -148,6 +270,13 @@ object Watched { } } + private[sbt] object NullLogger extends Logger { + override def trace(t: => Throwable): Unit = {} + override def success(message: => String): Unit = {} + override def log(level: Level.Value, message: => String): Unit = {} + } + + @deprecated("ContinuousEventMonitor attribute is not used by Watched.command", "1.3.0") val ContinuousEventMonitor = AttributeKey[EventMonitor]( "watch event monitor", @@ -180,3 +309,104 @@ object Watched { } } } + +/** + * Provides a number of configuration options for continuous execution. + */ +trait WatchConfig { + + /** + * A logger. + * @return a logger + */ + def logger: Logger + + /** + * Returns true if the continuous execution should stop. + * @return true if the contiuous execution should stop. + */ + def shouldTerminate(): Boolean + + /** + * The message to print when a build is triggered. + * @param count the current continous iteration count + * @return an optional string to log + */ + def triggeredMessage(count: Int): Option[String] + + /** + * The message to print at the beginning of each watch iteration. + * @param count the current watch iteration + * @return an optional string to log before each watch iteration. + */ + def watchingMessage(count: Int): Option[String] + + /** + * The WatchState that provides the WatchService that will be used to monitor events. + * @return the WatchState. + */ + def watchState: WatchState + + /** + * The maximum duration that the EventMonitor background thread will poll the underlying + * [[sbt.io.WatchService]] for events. + * @return + */ + def pollInterval: FiniteDuration + + /** + * The period for which files that trigger a build are quarantined from triggering a new build + * if they are modified. + * @return the anti-entropy period. + */ + def antiEntropy: FiniteDuration +} + +/** + * Provides a default implementation of [[WatchConfig]]. + */ +object WatchConfig { + + /** + * Generate an instance of [[WatchConfig]]. + * + * @param logger an [[sbt.util.Logger]] instance + * @param shouldStop returns true if the watch should stop + * @param triggeredMessage function to generate an optional message to print when a build is + + * @param watchingMessage function to generate an optional message to print before each watch + * iteration + * @param watchState the [[WatchState]] which provides an [[sbt.io.WatchService]] to monitor + * file system vents + * @param pollInterval the maximum polling time of the [[sbt.io.WatchService]] + * @param antiEntropy the period for which a file that triggered a build is quarantined so that + * any events detected during this period do not trigger a build. + * @return an instance of [[WatchConfig]]. + */ + def default( + logger: Logger, + shouldStop: () => Boolean, + triggeredMessage: Int => Option[String], + watchingMessage: Int => Option[String], + watchState: WatchState, + pollInterval: FiniteDuration, + antiEntropy: FiniteDuration, + ): WatchConfig = { + val l = logger + val ss = shouldStop + val tm = triggeredMessage + val wm = watchingMessage + val ws = watchState + val pi = pollInterval + val ae = antiEntropy + new WatchConfig { + override def logger: Logger = l + override def shouldTerminate(): Boolean = ss() + override def triggeredMessage(count: Int): Option[String] = tm(count) + override def watchingMessage(count: Int): Option[String] = wm(count) + override def watchState: WatchState = ws + override def pollInterval: FiniteDuration = pi + override def antiEntropy: FiniteDuration = ae + } + } +} diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala new file mode 100644 index 000000000..6f47f8880 --- /dev/null +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -0,0 +1,42 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import java.io.File +import java.util.concurrent.CountDownLatch + +import org.scalatest.{ FlatSpec, Matchers } +import sbt.Watched.{ WatchConfig, NullLogger, WatchSource } +import sbt.internal.io.WatchState +import sbt.io.IO + +import scala.concurrent.duration._ +import WatchedSpec._ + +class WatchedSpec extends FlatSpec with Matchers { + "Watched" should "stop" in IO.withTemporaryDirectory { dir => + val latch = new CountDownLatch(1) + val config = WatchConfig.default( + NullLogger, + () => latch.getCount == 0, + triggeredMessage = _ => { latch.countDown(); None }, + watchingMessage = _ => { new File(dir, "foo").createNewFile(); None }, + watchState = WatchState.empty(Watched.createWatchService, WatchSource(dir.toRealPath) :: Nil), + pollInterval = 5.millis, + antiEntropy = 5.millis + ) + Watched.watch(() => Right(true), config) + assert(latch.getCount == 0) + } +} + +object WatchedSpec { + implicit class FileOps(val f: File) { + def toRealPath: File = f.toPath.toRealPath().toFile + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 9bdbfd755..469886edd 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -265,6 +265,7 @@ object Defaults extends BuildCommon { parallelExecution :== true, pollInterval :== new FiniteDuration(500, TimeUnit.MILLISECONDS), watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), + watchLogger := streams.value.log, watchService :== { () => Watched.createWatchService() }, diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b8e5ff33e..e47d19bca 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -143,8 +143,9 @@ object Keys { val analysis = AttributeKey[CompileAnalysis]("analysis", "Analysis of compilation, including dependencies and generated outputs.", DSetting) val watch = SettingKey(BasicKeys.watch) val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) - val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) + val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) + val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) val watchSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in this project for continuous execution to watch for changes.").withRank(BMinusSetting) val watchTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for continuous execution to watch.").withRank(CSetting) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index ed669cb75..7c3002d5e 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -241,7 +241,8 @@ object BuiltinCommands { export, boot, initialize, - act + act, + continuous ) ++ allBasicCommands def DefaultBootCommands: Seq[String] = @@ -446,6 +447,14 @@ object BuiltinCommands { s } + def continuous: Command = Watched.continuous { (state: State, _: Watched, command: String) => + val extracted = Project.extract(state) + val (s, logger) = extracted.runTask(Keys.watchLogger, state) + val process: (() => State) => State = + (f: () => State) => MainLoop.processCommand(Exec(command, None), s, f) + (s, logger, process) + } + private[this] def loadedEval(s: State, arg: String): Unit = { val extracted = Project extract s import extracted._ diff --git a/main/src/main/scala/sbt/MainLoop.scala b/main/src/main/scala/sbt/MainLoop.scala index c9f4d171c..eb32d333f 100644 --- a/main/src/main/scala/sbt/MainLoop.scala +++ b/main/src/main/scala/sbt/MainLoop.scala @@ -144,13 +144,20 @@ object MainLoop { } /** This is the main function State transfer function of the sbt command processing. */ - def processCommand(exec: Exec, state: State): State = { + def processCommand(exec: Exec, state: State): State = + processCommand(exec, state, () => Command.process(exec.commandLine, state)) + + private[sbt] def processCommand( + exec: Exec, + state: State, + runCommand: () => State + ): State = { val channelName = exec.source map (_.channelName) StandardMain.exchange publishEventMessage ExecStatusEvent("Processing", channelName, exec.execId, Vector()) try { - val newState = Command.process(exec.commandLine, state) + val newState = runCommand() val doneEvent = ExecStatusEvent( "Done", channelName, From 7764dc42ae1b94323d53a4e6edf9ea64a3c30b8a Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Thu, 23 Aug 2018 18:43:07 -0700 Subject: [PATCH 07/17] Move executeContinuously into LegacyWatched This helps keep Watched.scala more manageable until we can potentially remove this code. --- main-command/src/main/scala/sbt/Watched.scala | 52 +++------------ .../scala/sbt/internal/LegacyWatched.scala | 64 +++++++++++++++++++ .../src/test/scala/sbt/WatchedSpec.scala | 5 +- 3 files changed, 76 insertions(+), 45 deletions(-) create mode 100644 main-command/src/main/scala/sbt/internal/LegacyWatched.scala diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index dcddda9d6..3d80f64ca 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -10,9 +10,15 @@ package sbt import java.io.File import java.nio.file.FileSystems -import sbt.BasicCommandStrings._ +import sbt.BasicCommandStrings.{ + ContinuousExecutePrefix, + FailureWall, + continuousBriefHelp, + continuousDetail +} import sbt.BasicCommands.otherCommandParser import sbt.CommandUtil.withAttribute +import sbt.internal.LegacyWatched import sbt.internal.io.{ EventMonitor, Source, WatchState } import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const @@ -227,48 +233,8 @@ object Watched { } @deprecated("Replaced by Watched.command", "1.3.0") - def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = { - @tailrec def shouldTerminate: Boolean = - (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) - val log = s.log - val logger = new EventMonitor.Logger { - override def debug(msg: => Any): Unit = log.debug(msg.toString) - } - s get ContinuousEventMonitor match { - case None => - // This is the first iteration, so run the task and create a new EventMonitor - (ClearOnFailure :: next :: FailureWall :: repeat :: s) - .put( - ContinuousEventMonitor, - EventMonitor( - WatchState.empty(watched.watchService(), watched.watchSources(s)), - watched.pollInterval, - watched.antiEntropy, - shouldTerminate, - logger - ) - ) - case Some(eventMonitor) => - printIfDefined(watched watchingMessage eventMonitor.state) - val triggered = try eventMonitor.awaitEvent() - catch { - case e: Exception => - log.error( - "Error occurred obtaining files to watch. Terminating continuous execution..." - ) - s.handleError(e) - false - } - if (triggered) { - printIfDefined(watched triggeredMessage eventMonitor.state) - ClearOnFailure :: next :: FailureWall :: repeat :: s - } else { - while (System.in.available() > 0) System.in.read() - eventMonitor.close() - s.remove(ContinuousEventMonitor) - } - } - } + def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = + LegacyWatched.executeContinuously(watched, s, next, repeat) private[sbt] object NullLogger extends Logger { override def trace(t: => Throwable): Unit = {} diff --git a/main-command/src/main/scala/sbt/internal/LegacyWatched.scala b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala new file mode 100644 index 000000000..624372efc --- /dev/null +++ b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala @@ -0,0 +1,64 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import sbt.BasicCommandStrings.{ ClearOnFailure, FailureWall } +import sbt.internal.io.{ EventMonitor, WatchState } +import sbt.{ State, Watched } + +import scala.annotation.tailrec +import Watched.ContinuousEventMonitor + +import scala.util.control.NonFatal + +private[sbt] object LegacyWatched { + @deprecated("Replaced by Watched.command", "1.3.0") + def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = { + @tailrec def shouldTerminate: Boolean = + (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) + val log = s.log + val logger = new EventMonitor.Logger { + override def debug(msg: => Any): Unit = log.debug(msg.toString) + } + s get ContinuousEventMonitor match { + case None => + // This is the first iteration, so run the task and create a new EventMonitor + (ClearOnFailure :: next :: FailureWall :: repeat :: s) + .put( + ContinuousEventMonitor, + EventMonitor( + WatchState.empty(watched.watchService(), watched.watchSources(s)), + watched.pollInterval, + watched.antiEntropy, + shouldTerminate, + logger + ) + ) + case Some(eventMonitor) => + Watched.printIfDefined(watched watchingMessage eventMonitor.state) + val triggered = try eventMonitor.awaitEvent() + catch { + case NonFatal(e) => + log.error( + "Error occurred obtaining files to watch. Terminating continuous execution..." + ) + s.handleError(e) + false + } + if (triggered) { + Watched.printIfDefined(watched triggeredMessage eventMonitor.state) + ClearOnFailure :: next :: FailureWall :: repeat :: s + } else { + while (System.in.available() > 0) System.in.read() + eventMonitor.close() + s.remove(ContinuousEventMonitor) + } + } + } + +} diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 6f47f8880..5add4ba86 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -11,7 +11,7 @@ import java.io.File import java.util.concurrent.CountDownLatch import org.scalatest.{ FlatSpec, Matchers } -import sbt.Watched.{ WatchConfig, NullLogger, WatchSource } +import sbt.Watched.{ NullLogger, WatchSource } import sbt.internal.io.WatchState import sbt.io.IO @@ -26,7 +26,8 @@ class WatchedSpec extends FlatSpec with Matchers { () => latch.getCount == 0, triggeredMessage = _ => { latch.countDown(); None }, watchingMessage = _ => { new File(dir, "foo").createNewFile(); None }, - watchState = WatchState.empty(Watched.createWatchService, WatchSource(dir.toRealPath) :: Nil), + watchState = + WatchState.empty(Watched.createWatchService(), WatchSource(dir.toRealPath) :: Nil), pollInterval = 5.millis, antiEntropy = 5.millis ) From 7d3d3c71d6aaad4d102582b101efb5f97622b35c Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 25 Sep 2018 19:39:19 -0700 Subject: [PATCH 08/17] Refactor Watched This commit reworks Watched to be more testable and extensible. It also adds some small features. The previous implementation presented a number of challenges: 1) It relied on external side effects to terminate the watch, which was difficult to test 2) It exposed irrelevant implementation details to the user in the methods that exposed the WatchState as a parameter. 3) It spun up two worker threads. One was to monitor System.in for user input. The other was to poll the watch service for events and write them to a queue. The user input thread actually broke '~console' because nearly every console session will hit the key, which would eventually cause the watch to stop when the user exited the console. To address (1), I add the shouldTerminate method to WatchConfig. This takes the current watch iteration is input and if the function returns true, the watch will stop. To address (2), I replace the triggeredMessage and watchingMessage keys with watchTriggeredMessage and watchStartMessage. The latter two keys are functions that do not take the WatchState as parameters. Both functions take the current iteration count as a parameter and the watchTriggeredMessage also has a parameter for the path that triggered the build. To address (3), I stop using the sbt.internal.io.EventMonitor and instead use the sbt.io.FileEventMonitor. The latter class is similar to the former except that it's polling method accepts a duration, which may be finite or infinite) and returns all of the events that occurred since it was last polled. By adding the ability to poll for a finite amount of time, we can interleave polling for events with polling System.in for user input, all on the main thread. This eliminates the two extraneous threads and fixes the '~console' use case I described before. I also let the user configure the function that reads from System.in via the watchHandleInput method. In fact, this method need not read from System.in at all since it's just () => Watched.Action. The reason that it isn't () => Boolean is that I'd like to leave open the option for the ability to trigger a build via user input, not just terminating the watch. My initial idea was to add the ability to type 'r' to re-build in addition to to exit. This doesn't work without integrating jline though because the input is buffered. Regardless, for testing purposes, it gives us the ability to add a timeout to the watch by making handleInput return true when a deadline expires. The tests are a bit wonky because I still need to rely on side effects in the logging methods to orchestrate the sequence of file events that I'd like to test. While I could move some of this logic into a background thread, there still needs to be coordination between the state of the watch and the background thread. I think it's easier to reason about when all of the work occurs on the same thread, even if it makes these user provided functions impure. I deprecated all of the previous watch related keys that are no longer used with the new infrastructure. To avoid breaking existing builds, I make the watchConfig task use the deprecated logging methods if they are defined in the user's builds, but sbt will not longer set the default values. For the vast majority of users, it should be straightforward to migrate their builds to use the new keys. My hunch is that the of the deprecated keys, only triggeredMessage is widely used (in conjunction with the clear screen method) and it is dead simple to replace it with watchTriggeredMessage. Note: The FileTreeViewConfig class is not really necessary for this commit. It will become more important in a subsequent commit which introduces an optional global file system cache. --- .travis.yml | 2 +- .../main/scala/sbt/FileTreeViewConfig.scala | 80 ++++ main-command/src/main/scala/sbt/Watched.scala | 359 +++++++++++------- .../src/test/scala/sbt/WatchedSpec.scala | 104 ++++- main/src/main/scala/sbt/Defaults.scala | 50 ++- main/src/main/scala/sbt/Keys.scala | 14 +- main/src/main/scala/sbt/Main.scala | 10 +- .../sbt-test/watch/on-start-watch/build.sbt | 13 + .../watch/on-start-watch/project/Count.scala | 6 + .../on-start-watch/src/main/scala/A.scala | 3 + sbt/src/sbt-test/watch/on-start-watch/test | 4 + 11 files changed, 470 insertions(+), 175 deletions(-) create mode 100644 main-command/src/main/scala/sbt/FileTreeViewConfig.scala create mode 100644 sbt/src/sbt-test/watch/on-start-watch/build.sbt create mode 100644 sbt/src/sbt-test/watch/on-start-watch/project/Count.scala create mode 100644 sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala create mode 100644 sbt/src/sbt-test/watch/on-start-watch/test diff --git a/.travis.yml b/.travis.yml index d17c5e546..1db7df73b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ env: - SBT_CMD="scripted source-dependencies/*1of3" - SBT_CMD="scripted source-dependencies/*2of3" - SBT_CMD="scripted source-dependencies/*3of3" - - SBT_CMD="scripted tests/*" + - SBT_CMD="scripted tests/* watch/*" - SBT_CMD="repoOverrideTest:scripted dependency-management/*" notifications: diff --git a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala new file mode 100644 index 000000000..10414348b --- /dev/null +++ b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala @@ -0,0 +1,80 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +import java.nio.file.Path + +import sbt.Watched.WatchSource +import sbt.internal.io.{ WatchServiceBackedObservable, WatchState } +import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView, TypedPath } +import sbt.util.Logger + +import scala.concurrent.duration.FiniteDuration + +/** + * Configuration for viewing and monitoring the file system. + */ +final class FileTreeViewConfig private ( + val newDataView: () => FileTreeDataView[Path], + val newMonitor: ( + FileTreeDataView[Path], + Seq[WatchSource], + Logger + ) => FileEventMonitor[Path] +) +object FileTreeViewConfig { + + /** + * Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded + * by {{{sbt.io.FileTreeDataView[Path]}}}. The reason for this is to ensure that a + * sbt.io.FileTreeDataView that is instantiated by [[FileTreeViewConfig.newDataView]] can be + * passed into [[FileTreeViewConfig.newMonitor]] without constraining the type of view to be + * {{{sbt.io.FileTreeDataView[Path]}}}. + * @param newDataView create a new sbt.io.FileTreeDataView. This value may be cached in a global + * attribute + * @param newMonitor create a new sbt.io.FileEventMonitor using the sbt.io.FileTreeDataView + * created by newDataView + * @tparam T the subtype of sbt.io.FileTreeDataView that is returned by [[FileTreeViewConfig.newDataView]] + * @return a [[FileTreeViewConfig]] instance. + */ + def apply[T <: FileTreeDataView[Path]]( + newDataView: () => T, + newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[Path] + ): FileTreeViewConfig = + new FileTreeViewConfig( + newDataView, + (view: FileTreeDataView[Path], sources: Seq[WatchSource], logger: Logger) => + newMonitor(view.asInstanceOf[T], sources, logger) + ) + + /** + * Provides a default [[FileTreeViewConfig]]. This view does not cache entries. + * @param pollingInterval the maximum duration that the sbt.internal.io.EventMonitor will poll + * the underlying sbt.io.WatchService when monitoring for file events + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @return a [[FileTreeViewConfig]] instance. + */ + def default(pollingInterval: FiniteDuration, antiEntropy: FiniteDuration): FileTreeViewConfig = + FileTreeViewConfig( + () => FileTreeView.DEFAULT.asDataView(_.getPath), + (_: FileTreeDataView[Path], sources, logger) => { + val ioLogger: sbt.io.WatchLogger = msg => logger.debug(msg.toString) + FileEventMonitor.antiEntropy( + new WatchServiceBackedObservable( + WatchState.empty(Watched.createWatchService(), sources), + pollingInterval, + (_: TypedPath).getPath, + closeService = true, + ioLogger + ), + antiEntropy, + ioLogger + ) + } + ) +} diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index 3d80f64ca..bd31f11e5 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -8,21 +8,16 @@ package sbt import java.io.File -import java.nio.file.FileSystems +import java.nio.file.{ FileSystems, Path } -import sbt.BasicCommandStrings.{ - ContinuousExecutePrefix, - FailureWall, - continuousBriefHelp, - continuousDetail -} +import sbt.BasicCommandStrings.{ ContinuousExecutePrefix, continuousBriefHelp, continuousDetail } import sbt.BasicCommands.otherCommandParser -import sbt.CommandUtil.withAttribute import sbt.internal.LegacyWatched import sbt.internal.io.{ EventMonitor, Source, WatchState } import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const import sbt.internal.util.complete.DefaultParsers +import sbt.io.FileEventMonitor.Event import sbt.io._ import sbt.util.{ Level, Logger } @@ -30,6 +25,7 @@ import scala.annotation.tailrec import scala.concurrent.duration._ import scala.util.Properties +@deprecated("Watched is no longer used to implement continuous execution", "1.3.0") trait Watched { /** The files watched when an action is run with a proceeding ~ */ @@ -59,18 +55,64 @@ trait Watched { } object Watched { - val defaultWatchingMessage: WatchState => String = ws => - s"${ws.count}. Waiting for source changes... (press enter to interrupt)" + /** + * This trait is used to communicate what the watch should do next at various points in time. It + * is heavily linked to a number of callbacks in [[WatchConfig]]. For example, when the + * sbt.io.FileEventMonitor created by [[FileTreeViewConfig.newMonitor]] detects a changed source + * file, then we expect [[WatchConfig.onWatchEvent]] to return [[Trigger]]. + */ + sealed trait Action + + /** + * Action that indicates that the watch should stop. + */ + case object CancelWatch extends Action + + /** + * Action that indicates that the watch should continue as though nothing happened. This may be + * because, for example, no user input was yet available in [[WatchConfig.handleInput]]. + */ + case object Ignore extends Action + + /** + * Action that indicates that the watch process should re-run the command. + */ + case object Trigger extends Action + + type WatchSource = Source + def terminateWatch(key: Int): Boolean = Watched.isEnter(key) + /* + * Without jline, checking for enter is nearly pointless because System.in.available will not + * return a non-zero value until the user presses enter. + */ + @tailrec + final def shouldTerminate: Boolean = + (System.in.available > 0) && (terminateWatch(System.in.read()) || shouldTerminate) + final val handleInput: () => Action = () => if (shouldTerminate) CancelWatch else Ignore + val defaultStartWatch: Int => Option[String] = count => + Some(s"$count. Waiting for source changes... (press enter to interrupt)") + @deprecated("Use defaultStartWatch in conjunction with the watchStartMessage key", "1.3.0") + val defaultWatchingMessage: WatchState => String = ws => defaultStartWatch(ws.count).get def projectWatchingMessage(projectId: String): WatchState => String = - ws => - s"${ws.count}. Waiting for source changes in project $projectId... (press enter to interrupt)" + ws => projectOnWatchMessage(projectId)(ws.count).get + def projectOnWatchMessage(project: String): Int => Option[String] = + count => + Some( + s"$count. Waiting for source changes in project $project... (press enter to interrupt)" + ) + val defaultOnTriggerMessage: Int => Option[String] = _ => None + @deprecated( + "Use defaultOnTriggerMessage in conjunction with the watchTriggeredMessage key", + "1.3.0" + ) val defaultTriggeredMessage: WatchState => String = const("") + val clearOnTrigger: Int => Option[String] = _ => Some(clearScreen) + @deprecated("Use clearOnTrigger in conjunction with the watchTriggeredMessage key", "1.3.0") val clearWhenTriggered: WatchState => String = const(clearScreen) def clearScreen: String = "\u001b[2J\u001b[0;0H" - type WatchSource = Source object WatchSource { /** @@ -94,6 +136,7 @@ object Watched { } + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") private[this] class AWatched extends Watched @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") @@ -107,6 +150,7 @@ object Watched { override def watchingMessage(s: WatchState): String = base.watchingMessage(s) override def triggeredMessage(s: WatchState): String = base.triggeredMessage(s) } + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") def empty: Watched = new AWatched val PollDelay: FiniteDuration = 500.milliseconds @@ -114,24 +158,24 @@ object Watched { def isEnter(key: Int): Boolean = key == 10 || key == 13 def printIfDefined(msg: String): Unit = if (!msg.isEmpty) System.out.println(msg) - type Task = () => State - type Setup = (State, Watched, String) => (State, Logger, Task => State) + private type RunCommand = () => State + private type WatchSetup = (State, String) => (State, WatchConfig, RunCommand => State) /** * Provides the '~' continuous execution command. * @param setup a function that provides a logger and a function from (() => State) => State. * @return the '~' command. */ - def continuous(setup: Setup): Command = + def continuous(setup: WatchSetup): Command = Command(ContinuousExecutePrefix, continuousBriefHelp, continuousDetail)(otherCommandParser) { (state, command) => - Watched.command(state, command, setup) + Watched.executeContinuously(state, command, setup) } /** * Implements continuous execution. It works by first parsing the command and generating a task to * run with each build. It can run multiple commands that are separated by ";" in the command - * input. If any of these commands are invalid, the watch will immmediately exit. + * input. If any of these commands are invalid, the watch will immediately exit. * @param state the initial state * @param command the command(s) to repeatedly apply * @param setup function to generate a logger and a transformation of the resultant state. The @@ -141,93 +185,111 @@ object Watched { * @return the initial state if all of the input commands are valid. Otherwise, returns the * initial state with the failure transformation. */ - private[sbt] def command( + private[sbt] def executeContinuously( state: State, command: String, - setup: Setup, - ): State = - withAttribute(state, Watched.Configuration, "Continuous execution not configured.") { w => - val (s0, logger, process) = setup(state, w, command) - val s = FailureWall :: s0 - val parser = Command.combine(s.definedCommands)(s) - val commands = command.split(";") match { - case Array("", rest @ _*) => rest - case Array(cmd) => Seq(cmd) - } - val tasks = commands.foldLeft(Nil: Seq[Either[String, () => Either[Exception, Boolean]]]) { - case (t, cmd) => - t :+ (DefaultParsers.parse(cmd, parser) match { - case Right(task) => - Right { () => - try { - process(task) - Right(true) - } catch { case e: Exception => Left(e) } - } - case Left(_) => Left(cmd) - }) - } - val (valid, invalid) = tasks.partition(_.isRight) - if (invalid.isEmpty) { - val task = () => - valid.foldLeft(Right(true): Either[Exception, Boolean]) { - case (status, Right(t)) => if (status.getOrElse(true)) t() else status - case _ => throw new IllegalStateException("Should be unreachable") - } - @tailrec def shouldTerminate: Boolean = - (System.in.available > 0) && (w.terminateWatch(System.in.read()) || shouldTerminate) - val watchState = WatchState.empty(w.watchService(), w.watchSources(s)) - val config = WatchConfig.default( - logger, - () => shouldTerminate, - count => Some(w.triggeredMessage(watchState.withCount(count))).filter(_.nonEmpty), - count => Some(w.watchingMessage(watchState.withCount(count))).filter(_.nonEmpty), - watchState, - w.pollInterval, - w.antiEntropy - ) - watch(task, config) - state - } else { - logger.error( - s"Terminating watch due to invalid command(s): ${invalid.mkString("'", "', '", "'")}" - ) - state.fail - } + setup: WatchSetup, + ): State = { + val (s, config, newState) = setup(state, command) + val commands = command.split(";") match { + case Array("", rest @ _*) => rest + case Array(cmd) => Seq(cmd) } + val parser = Command.combine(s.definedCommands)(s) + val tasks = commands.foldLeft(Nil: Seq[Either[String, () => Either[Exception, Boolean]]]) { + (t, cmd) => + t :+ (DefaultParsers.parse(cmd, parser) match { + case Right(task) => + Right { () => + try { + newState(task) + Right(true) + } catch { case e: Exception => Left(e) } + } + case Left(_) => Left(cmd) + }) + } + val (valid, invalid) = tasks.partition(_.isRight) + if (invalid.isEmpty) { + val task = () => + valid.foldLeft(Right(true): Either[Exception, Boolean]) { + case (status, Right(t)) => if (status.getOrElse(true)) t() else status + case _ => throw new IllegalStateException("Should be unreachable") + } + watch(task, config) + state + } else { + config.logger.error( + s"Terminating watch due to invalid command(s): ${invalid.mkString("'", "', '", "'")}" + ) + state.fail + } + } private[sbt] def watch( task: () => Either[Exception, _], config: WatchConfig, ): Unit = { - val eventLogger = new EventMonitor.Logger { - override def debug(msg: => Any): Unit = config.logger.debug(msg.toString) - } - def debug(msg: String): Unit = if (msg.nonEmpty) config.logger.debug(msg) - val monitor = EventMonitor( - config.watchState, - config.pollInterval, - config.antiEntropy, - config.shouldTerminate(), - eventLogger - ) + val logger = config.logger + def info(msg: String): Unit = if (msg.nonEmpty) logger.info(msg) @tailrec def impl(count: Int): Unit = { + @tailrec + def nextAction(): Action = { + config.handleInput() match { + case CancelWatch => CancelWatch + case Trigger => Trigger + case _ => + val events = config.fileEventMonitor.poll(10.millis) + val next = events match { + case Seq() => (Ignore, None) + case Seq(head, tail @ _*) => + /* + * We traverse all of the events and find the one for which we give the highest + * weight. + * CancelWatch > Trigger > Ignore + */ + tail.foldLeft((config.onWatchEvent(head), Some(head))) { + case (r @ (CancelWatch, _), _) => r + // If we've found a trigger, only change the accumulator if we find a CancelWatch. + case ((action, event), e) => + config.onWatchEvent(e) match { + case Trigger if action == Ignore => (Trigger, Some(e)) + case _ => (action, event) + } + } + } + next match { + case (CancelWatch, Some(event)) => + logger.debug(s"Stopping watch due to event from ${event.entry.typedPath.getPath}.") + CancelWatch + case (Trigger, Some(event)) => + logger.debug(s"Triggered by ${event.entry.typedPath.getPath}") + config.triggeredMessage(event.entry.typedPath, count).foreach(info) + Trigger + case _ => + nextAction() + } + } + } task() match { - case _: Right[Exception, _] => - config.watchingMessage(count).foreach(debug) - if (monitor.awaitEvent()) { - config.triggeredMessage(count).foreach(debug) - impl(count + 1) + case Right(status) if !config.shouldTerminate(count) => + config.watchingMessage(count).foreach(info) + nextAction() match { + case CancelWatch => () + case _ => impl(count + 1) } - case Left(e) => config.logger.error(s"Terminating watch due to Unexpected error: $e") + case Left(e) => + logger.error(s"Terminating watch due to Unexpected error: $e") + case _ => + logger.debug("Terminating watch due to WatchConfig.shouldTerminate") } } try { impl(count = 1) } finally { - monitor.close() + config.fileEventMonitor.close() while (System.in.available() > 0) System.in.read() } } @@ -288,44 +350,48 @@ trait WatchConfig { def logger: Logger /** - * Returns true if the continuous execution should stop. - * @return true if the contiuous execution should stop. + * The sbt.io.FileEventMonitor that is used to monitor the file system. + * + * @return an sbt.io.FileEventMonitor instance. */ - def shouldTerminate(): Boolean + def fileEventMonitor: FileEventMonitor[Path] /** - * The message to print when a build is triggered. - * @param count the current continous iteration count - * @return an optional string to log + * A function that is periodically invoked to determine whether the watch should stop or + * trigger. Usually this will read from System.in to react to user input. + * @return an [[Watched.Action Action]] that will determine the next step in the watch. */ - def triggeredMessage(count: Int): Option[String] + def handleInput(): Watched.Action /** - * The message to print at the beginning of each watch iteration. - * @param count the current watch iteration - * @return an optional string to log before each watch iteration. + * This is run before each watch iteration and if it returns true, the watch is terminated. + * @param count The current number of watch iterstaions. + * @return true if the watch should stop. + */ + def shouldTerminate(count: Int): Boolean + + /** + * Callback that is invoked whenever a file system vent is detected. The next step of the watch + * is determined by the [[Watched.Action Action]] returned by the callback. + * @param event the detected sbt.io.FileEventMonitor.Event. + * @return the next [[Watched.Action Action]] to run. + */ + def onWatchEvent(event: Event[Path]): Watched.Action + + /** + * The optional message to log when a build is triggered. + * @param typedPath the path that triggered the build + * @param count the current iteration + * @return an optional log message. + */ + def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] + + /** + * The optional message to log before each watch iteration. + * @param count the current iteration + * @return an optional log message. */ def watchingMessage(count: Int): Option[String] - - /** - * The WatchState that provides the WatchService that will be used to monitor events. - * @return the WatchState. - */ - def watchState: WatchState - - /** - * The maximum duration that the EventMonitor background thread will poll the underlying - * [[sbt.io.WatchService]] for events. - * @return - */ - def pollInterval: FiniteDuration - - /** - * The period for which files that trigger a build are quarantined from triggering a new build - * if they are modified. - * @return the anti-entropy period. - */ - def antiEntropy: FiniteDuration } /** @@ -334,45 +400,46 @@ trait WatchConfig { object WatchConfig { /** - * Generate an instance of [[WatchConfig]]. - * - * @param logger an [[sbt.util.Logger]] instance - * @param shouldStop returns true if the watch should stop - * @param triggeredMessage function to generate an optional message to print when a build is - - * @param watchingMessage function to generate an optional message to print before each watch - * iteration - * @param watchState the [[WatchState]] which provides an [[sbt.io.WatchService]] to monitor - * file system vents - * @param pollInterval the maximum polling time of the [[sbt.io.WatchService]] - * @param antiEntropy the period for which a file that triggered a build is quarantined so that - * any events detected during this period do not trigger a build. - * @return an instance of [[WatchConfig]]. + * Create an instance of [[WatchConfig]]. + * @param logger logger for watch events + * @param fileEventMonitor the monitor for file system events. + * @param handleInput callback that is periodically invoked to check whether to continue or + * terminate the watch based on user input. It is also possible to, for + * example time out the watch using this callback. + * @param onWatchEvent callback that is invoked when + * @param triggeredMessage optional message that will be logged when a new build is triggered. + * The input parameters are the sbt.io.TypedPath that triggered the new + * build and the current iteration count. + * @param watchingMessage optional message that is printed before each watch iteration begins. + * The input parameter is the current iteration count. + * @return a [[WatchConfig]] instance. */ def default( logger: Logger, - shouldStop: () => Boolean, - triggeredMessage: Int => Option[String], - watchingMessage: Int => Option[String], - watchState: WatchState, - pollInterval: FiniteDuration, - antiEntropy: FiniteDuration, + fileEventMonitor: FileEventMonitor[Path], + handleInput: () => Watched.Action, + shouldTerminate: Int => Boolean, + onWatchEvent: Event[Path] => Watched.Action, + triggeredMessage: (TypedPath, Int) => Option[String], + watchingMessage: Int => Option[String] ): WatchConfig = { val l = logger - val ss = shouldStop + val fem = fileEventMonitor + val hi = handleInput + val st = shouldTerminate + val owe = onWatchEvent val tm = triggeredMessage val wm = watchingMessage - val ws = watchState - val pi = pollInterval - val ae = antiEntropy new WatchConfig { override def logger: Logger = l - override def shouldTerminate(): Boolean = ss() - override def triggeredMessage(count: Int): Option[String] = tm(count) + override def fileEventMonitor: FileEventMonitor[Path] = fem + override def handleInput(): Watched.Action = hi() + override def shouldTerminate(count: Int): Boolean = + st(count) + override def onWatchEvent(event: Event[Path]): Watched.Action = owe(event) + override def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] = + tm(typedPath, count) override def watchingMessage(count: Int): Option[String] = wm(count) - override def watchState: WatchState = ws - override def pollInterval: FiniteDuration = pi - override def antiEntropy: FiniteDuration = ae } } } diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 5add4ba86..1ea7f36f7 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -8,31 +8,99 @@ package sbt import java.io.File -import java.util.concurrent.CountDownLatch +import java.nio.file.{ Files, Path } +import java.util.concurrent.atomic.AtomicBoolean import org.scalatest.{ FlatSpec, Matchers } -import sbt.Watched.{ NullLogger, WatchSource } -import sbt.internal.io.WatchState -import sbt.io.IO +import sbt.Watched._ +import sbt.WatchedSpec._ +import sbt.io.FileEventMonitor.Event +import sbt.io.{ FileEventMonitor, IO, TypedPath } +import sbt.util.Logger +import scala.collection.mutable import scala.concurrent.duration._ -import WatchedSpec._ class WatchedSpec extends FlatSpec with Matchers { - "Watched" should "stop" in IO.withTemporaryDirectory { dir => - val latch = new CountDownLatch(1) - val config = WatchConfig.default( - NullLogger, - () => latch.getCount == 0, - triggeredMessage = _ => { latch.countDown(); None }, - watchingMessage = _ => { new File(dir, "foo").createNewFile(); None }, - watchState = - WatchState.empty(Watched.createWatchService(), WatchSource(dir.toRealPath) :: Nil), - pollInterval = 5.millis, - antiEntropy = 5.millis + object Defaults { + private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis, 50.millis) + def config( + sources: Seq[WatchSource], + fileEventMonitor: Option[FileEventMonitor[Path]] = None, + logger: Logger = NullLogger, + handleInput: () => Action = () => Ignore, + shouldTerminate: Int => Boolean = _ => true, + onWatchEvent: Event[Path] => Action = _ => Ignore, + triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None, + watchingMessage: Int => Option[String] = _ => None + ): WatchConfig = { + val monitor = fileEventMonitor.getOrElse( + fileTreeViewConfig.newMonitor(fileTreeViewConfig.newDataView(), sources, logger) + ) + WatchConfig.default( + logger = logger, + monitor, + handleInput, + shouldTerminate, + onWatchEvent, + triggeredMessage, + watchingMessage + ) + } + } + "Watched.watch" should "stop" in IO.withTemporaryDirectory { dir => + val config = Defaults.config(sources = Seq(WatchSource(dir.toRealPath))) + Watched.watch(() => Right(true), config) should be(()) + } + it should "trigger" in IO.withTemporaryDirectory { dir => + val triggered = new AtomicBoolean(false) + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + shouldTerminate = count => count == 2, + onWatchEvent = _ => { triggered.set(true); Trigger }, + watchingMessage = _ => { + new File(dir, "file").createNewFile; None + } ) - Watched.watch(() => Right(true), config) - assert(latch.getCount == 0) + Watched.watch(() => Right(true), config) should be(()) + assert(triggered.get()) + } + it should "filter events" in IO.withTemporaryDirectory { dir => + val realDir = dir.toRealPath + val queue = new mutable.Queue[TypedPath] + val foo = realDir.toPath.resolve("foo") + val bar = realDir.toPath.resolve("bar") + val config = Defaults.config( + sources = Seq(WatchSource(realDir)), + shouldTerminate = count => count == 2, + onWatchEvent = e => if (e.entry.typedPath.getPath == foo) Trigger else Ignore, + triggeredMessage = (tp, _) => { queue += tp; None }, + watchingMessage = _ => { Files.createFile(bar); Thread.sleep(5); Files.createFile(foo); None } + ) + Watched.watch(() => Right(true), config) should be(()) + queue.toIndexedSeq.map(_.getPath) shouldBe Seq(foo) + } + it should "enforce anti-entropy" in IO.withTemporaryDirectory { dir => + val realDir = dir.toRealPath + val queue = new mutable.Queue[TypedPath] + val foo = realDir.toPath.resolve("foo") + val bar = realDir.toPath.resolve("bar") + val config = Defaults.config( + sources = Seq(WatchSource(realDir)), + shouldTerminate = count => count == 3, + onWatchEvent = _ => Trigger, + triggeredMessage = (tp, _) => { queue += tp; None }, + watchingMessage = count => { + if (count == 1) Files.createFile(bar) + else if (count == 2) { + bar.toFile.setLastModified(5000) + Files.createFile(foo) + } + None + } + ) + Watched.watch(() => Right(true), config) should be(()) + queue.toIndexedSeq.map(_.getPath) shouldBe Seq(bar, foo) } } diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 469886edd..8e41ada1d 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -40,16 +40,17 @@ import sbt.internal.util.Types._ import sbt.io.syntax._ import sbt.io.{ AllPassFilter, + DirectoryFilter, FileFilter, GlobFilter, + Hash, HiddenFileFilter, IO, NameFilter, NothingFilter, Path, PathFinder, - DirectoryFilter, - Hash + TypedPath }, Path._ import sbt.librarymanagement.Artifact.{ DocClassifier, SourceClassifier } import sbt.librarymanagement.Configurations.{ @@ -264,6 +265,11 @@ object Defaults extends BuildCommon { concurrentRestrictions := defaultRestrictions.value, parallelExecution :== true, pollInterval :== new FiniteDuration(500, TimeUnit.MILLISECONDS), + watchTriggeredMessage := { (_, _) => + None + }, + watchStartMessage := Watched.defaultStartWatch, + fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value), watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), watchLogger := streams.value.log, watchService :== { () => @@ -600,8 +606,43 @@ object Defaults extends BuildCommon { clean := (Def.task { IO.delete(cleanFiles.value) } tag (Tags.Clean)).value, consoleProject := consoleProjectTask.value, watchTransitiveSources := watchTransitiveSourcesTask.value, - watchingMessage := Watched.projectWatchingMessage(thisProjectRef.value.project), - watch := watchSetting.value + watchOnEvent := { + val sources = watchTransitiveSources.value + e => + if (sources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Trigger else Watched.Ignore + }, + watchHandleInput := Watched.handleInput, + watchShouldTerminate := { _ => + false + }, + watchConfig := { + val sources = watchTransitiveSources.value + val extracted = Project.extract(state.value) + val wm = extracted + .getOpt(watchingMessage) + .map(w => (count: Int) => Some(w(WatchState.empty(sources).withCount(count)))) + .getOrElse(watchStartMessage.value) + val tm = extracted + .getOpt(triggeredMessage) + .map( + tm => (_: TypedPath, count: Int) => Some(tm(WatchState.empty(sources).withCount(count))) + ) + .getOrElse(watchTriggeredMessage.value) + val logger = watchLogger.value + val viewConfig = fileTreeViewConfig.value + WatchConfig.default( + logger, + viewConfig.newMonitor(viewConfig.newDataView(), sources, logger), + watchHandleInput.value, + watchShouldTerminate.value, + watchOnEvent.value, + tm, + wm + ) + }, + watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), + watch := watchSetting.value, + fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value), ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = @@ -622,6 +663,7 @@ object Defaults extends BuildCommon { Def.task { allUpdates.value.flatten ++ globalPluginUpdate.?.value } } + @deprecated("This is no longer used to implement continuous execution", "1.3.0") def watchSetting: Initialize[Watched] = Def.setting { val getService = watchService.value diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index e47d19bca..66ffb8808 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -8,6 +8,7 @@ package sbt import java.io.File +import java.nio.file.{ Path => JPath } import java.net.URL import scala.concurrent.duration.{ FiniteDuration, Duration } import Def.ScopedKey @@ -40,7 +41,8 @@ import sbt.internal.{ SessionSettings, LogManager } -import sbt.io.{ FileFilter, WatchService } +import sbt.io.{ FileFilter, TypedPath, WatchService } +import sbt.io.FileEventMonitor.Event import sbt.internal.io.WatchState import sbt.internal.server.ServerHandler import sbt.internal.util.{ AttributeKey, SourcePosition } @@ -141,16 +143,26 @@ object Keys { val serverHandlers = settingKey[Seq[ServerHandler]]("User-defined server handlers.") val analysis = AttributeKey[CompileAnalysis]("analysis", "Analysis of compilation, including dependencies and generated outputs.", DSetting) + @deprecated("This is no longer used for continuous execution", "1.3.0") val watch = SettingKey(BasicKeys.watch) val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) + val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting) val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) + val watchHandleInput = settingKey[() => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) + val watchOnEvent = taskKey[Event[JPath] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) + val watchShouldTerminate = settingKey[Int => Boolean]("Function that may terminate a continuous build based on the number of iterations.").withRank(BMinusSetting) val watchSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in this project for continuous execution to watch for changes.").withRank(BMinusSetting) + val watchStartMessage = settingKey[Int => Option[String]]("The message to show when triggered execution waits for sources to change. The parameter is the current watch iteration count.").withRank(DSetting) val watchTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for continuous execution to watch.").withRank(CSetting) + val watchTriggeredMessage = settingKey[(TypedPath, Int) => Option[String]]("The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count.").withRank(DSetting) + @deprecated("Use watchStartMessage instead", "1.3.0") val watchingMessage = settingKey[WatchState => String]("The message to show when triggered execution waits for sources to change.").withRank(DSetting) + @deprecated("Use watchTriggeredMessage instead", "1.3.0") val triggeredMessage = settingKey[WatchState => String]("The message to show before triggered execution executes an action after sources change.").withRank(DSetting) + val fileTreeViewConfig = taskKey[FileTreeViewConfig]("Configures how sbt will traverse and monitor the file system.").withRank(BMinusSetting) // Path Keys val baseDirectory = settingKey[File]("The base directory. Depending on the scope, this is the base directory for the build, project, configuration, or task.").withRank(AMinusSetting) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 7c3002d5e..045e36ac7 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -447,12 +447,12 @@ object BuiltinCommands { s } - def continuous: Command = Watched.continuous { (state: State, _: Watched, command: String) => + def continuous: Command = Watched.continuous { (state: State, command: String) => val extracted = Project.extract(state) - val (s, logger) = extracted.runTask(Keys.watchLogger, state) - val process: (() => State) => State = - (f: () => State) => MainLoop.processCommand(Exec(command, None), s, f) - (s, logger, process) + val (s, watchConfig) = extracted.runTask(Keys.watchConfig, state) + val updateState = + (runCommand: () => State) => MainLoop.processCommand(Exec(command, None), s, runCommand) + (s, watchConfig, updateState) } private[this] def loadedEval(s: State, arg: String): Unit = { diff --git a/sbt/src/sbt-test/watch/on-start-watch/build.sbt b/sbt/src/sbt-test/watch/on-start-watch/build.sbt new file mode 100644 index 000000000..d992f3473 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/build.sbt @@ -0,0 +1,13 @@ +val checkCount = inputKey[Unit]("check that compile has run a specified number of times") + +checkCount := { + val expected = Def.spaceDelimited().parsed.head.toInt + assert(Count.get == expected) +} + +Compile / compile := { + Count.increment() + // Trigger a new build by updating the last modified time + ((Compile / scalaSource).value / "A.scala").setLastModified(5000) + (Compile / compile).value +} diff --git a/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala new file mode 100644 index 000000000..0698b75ff --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala @@ -0,0 +1,6 @@ +object Count { + private var count = 0 + def get: Int = count + def increment(): Unit = count += 1 + def reset(): Unit = count = 0 +} \ No newline at end of file diff --git a/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala b/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala new file mode 100644 index 000000000..df9e4d3d5 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala @@ -0,0 +1,3 @@ +package a + +class A \ No newline at end of file diff --git a/sbt/src/sbt-test/watch/on-start-watch/test b/sbt/src/sbt-test/watch/on-start-watch/test new file mode 100644 index 000000000..f5fa900e7 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/test @@ -0,0 +1,4 @@ +# verify that the watch terminates when we reach the specified count +> set watchShouldTerminate := { count => count == 2 } +> ~compile +> checkCount 2 From 28fd4a1e618f938f9e1a4bd9639255c90a629cba Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 25 Sep 2018 18:34:14 -0700 Subject: [PATCH 09/17] Add the ability to halt watch on failure There may be instances where the user may wish to stop the watch if an error occurs running the task. To facilitate this, I add boolean parameter, lastStatus, to watchShouldTerminate. The value is computed by modifying the state used to run the task to have a custom onFailure command. If the task fails, the returned state will have the onFailure command will be enqueued at the head of the remaining commands. The result of the task then becomes true if the custom onFailure is not present in the remaining commands and false if it is. We don't actually run this command, so it's just implemented with the identity function. I also updated Watched.watch to return an Action instead of Unit. This enables us to return a failed state if Watched.watch returns HandleError. --- main-command/src/main/scala/sbt/Watched.scala | 144 +++++++++++++----- .../src/test/scala/sbt/WatchedSpec.scala | 47 ++++-- main/src/main/scala/sbt/Defaults.scala | 33 +++- main/src/main/scala/sbt/Keys.scala | 5 +- .../sbt-test/watch/on-start-watch/build.sbt | 42 ++++- .../watch/on-start-watch/project/Count.scala | 6 +- sbt/src/sbt-test/watch/on-start-watch/test | 26 +++- 7 files changed, 242 insertions(+), 61 deletions(-) diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index bd31f11e5..b501a1f36 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -10,7 +10,12 @@ package sbt import java.io.File import java.nio.file.{ FileSystems, Path } -import sbt.BasicCommandStrings.{ ContinuousExecutePrefix, continuousBriefHelp, continuousDetail } +import sbt.BasicCommandStrings.{ + ContinuousExecutePrefix, + FailureWall, + continuousBriefHelp, + continuousDetail +} import sbt.BasicCommands.otherCommandParser import sbt.internal.LegacyWatched import sbt.internal.io.{ EventMonitor, Source, WatchState } @@ -69,12 +74,24 @@ object Watched { */ case object CancelWatch extends Action + /** + * Action that indicates that an error has occurred. The watch will be terminated when this action + * is produced. + */ + case object HandleError extends Action + /** * Action that indicates that the watch should continue as though nothing happened. This may be * because, for example, no user input was yet available in [[WatchConfig.handleInput]]. */ case object Ignore extends Action + /** + * Action that indicates that the watch should pause while the build is reloaded. This is used to + * automatically reload the project when the build files (e.g. build.sbt) are changed. + */ + case object Reload extends Action + /** * Action that indicates that the watch process should re-run the command. */ @@ -172,6 +189,22 @@ object Watched { Watched.executeContinuously(state, command, setup) } + /** + * Default handler to transform the state when the watch terminates. When the [[Watched.Action]] is + * [[Reload]], the handler will prepend the original command (prefixed by ~) to the + * [[State.remainingCommands]] and then invoke the [[StateOps.reload]] method. When the + * [[Watched.Action]] is [[HandleError]], the handler returns the result of [[StateOps.fail]]. Otherwise + * the original state is returned. + */ + private[sbt] val onTermination: (Action, String, State) => State = (action, command, state) => + action match { + case Reload => + val continuousCommand = Exec(ContinuousExecutePrefix + command, None) + state.copy(remainingCommands = continuousCommand +: state.remainingCommands).reload + case HandleError => state.fail + case _ => state + } + /** * Implements continuous execution. It works by first parsing the command and generating a task to * run with each build. It can run multiple commands that are separated by ";" in the command @@ -190,7 +223,13 @@ object Watched { command: String, setup: WatchSetup, ): State = { - val (s, config, newState) = setup(state, command) + val (s0, config, newState) = setup(state, command) + val failureCommandName = "SbtContinuousWatchOnFail" + val onFail = Command.command(failureCommandName)(identity) + val s = (FailureWall :: s0).copy( + onFailure = Some(Exec(failureCommandName, None)), + definedCommands = s0.definedCommands :+ onFail + ) val commands = command.split(";") match { case Array("", rest @ _*) => rest case Array(cmd) => Seq(cmd) @@ -202,8 +241,7 @@ object Watched { case Right(task) => Right { () => try { - newState(task) - Right(true) + Right(newState(task).remainingCommands.forall(_.commandLine != failureCommandName)) } catch { case e: Exception => Left(e) } } case Left(_) => Left(cmd) @@ -216,8 +254,8 @@ object Watched { case (status, Right(t)) => if (status.getOrElse(true)) t() else status case _ => throw new IllegalStateException("Should be unreachable") } - watch(task, config) - state + val terminationAction = watch(task, config) + config.onWatchTerminated(terminationAction, command, state) } else { config.logger.error( s"Terminating watch due to invalid command(s): ${invalid.mkString("'", "', '", "'")}" @@ -227,19 +265,19 @@ object Watched { } private[sbt] def watch( - task: () => Either[Exception, _], - config: WatchConfig, - ): Unit = { + task: () => Either[Exception, Boolean], + config: WatchConfig + ): Action = { val logger = config.logger def info(msg: String): Unit = if (msg.nonEmpty) logger.info(msg) @tailrec - def impl(count: Int): Unit = { + def impl(count: Int): Action = { @tailrec def nextAction(): Action = { config.handleInput() match { - case CancelWatch => CancelWatch - case Trigger => Trigger + case action @ (CancelWatch | HandleError | Reload) => action + case Trigger => Trigger case _ => val events = config.fileEventMonitor.poll(10.millis) val next = events match { @@ -248,42 +286,53 @@ object Watched { /* * We traverse all of the events and find the one for which we give the highest * weight. - * CancelWatch > Trigger > Ignore + * HandleError > CancelWatch > Reload > Trigger > Ignore */ tail.foldLeft((config.onWatchEvent(head), Some(head))) { - case (r @ (CancelWatch, _), _) => r - // If we've found a trigger, only change the accumulator if we find a CancelWatch. - case ((action, event), e) => - config.onWatchEvent(e) match { - case Trigger if action == Ignore => (Trigger, Some(e)) - case _ => (action, event) + case (current @ (action, _), event) => + config.onWatchEvent(event) match { + case HandleError => (HandleError, Some(event)) + case CancelWatch if action != HandleError => (CancelWatch, Some(event)) + case Reload if action != HandleError && action != CancelWatch => + (Reload, Some(event)) + case Trigger if action == Ignore => (Trigger, Some(event)) + case _ => current } } } + // Note that nextAction should never return Ignore. next match { - case (CancelWatch, Some(event)) => - logger.debug(s"Stopping watch due to event from ${event.entry.typedPath.getPath}.") - CancelWatch + case (action @ (HandleError | CancelWatch), Some(event)) => + val cause = if (action == HandleError) "error" else "cancellation" + logger.debug(s"Stopping watch due to $cause from ${event.entry.typedPath.getPath}") + action case (Trigger, Some(event)) => logger.debug(s"Triggered by ${event.entry.typedPath.getPath}") config.triggeredMessage(event.entry.typedPath, count).foreach(info) Trigger + case (Reload, Some(event)) => + logger.info(s"Reload triggered by ${event.entry.typedPath.getPath}") + Reload case _ => nextAction() } } } task() match { - case Right(status) if !config.shouldTerminate(count) => - config.watchingMessage(count).foreach(info) - nextAction() match { - case CancelWatch => () - case _ => impl(count + 1) + case Right(status) => + config.preWatch(count, status) match { + case Ignore => + config.watchingMessage(count).foreach(info) + nextAction() match { + case action @ (CancelWatch | HandleError | Reload) => action + case _ => impl(count + 1) + } + case Trigger => impl(count + 1) + case action @ (CancelWatch | HandleError | Reload) => action } case Left(e) => logger.error(s"Terminating watch due to Unexpected error: $e") - case _ => - logger.debug("Terminating watch due to WatchConfig.shouldTerminate") + HandleError } } try { @@ -365,10 +414,11 @@ trait WatchConfig { /** * This is run before each watch iteration and if it returns true, the watch is terminated. - * @param count The current number of watch iterstaions. - * @return true if the watch should stop. + * @param count The current number of watch iterations. + * @param lastStatus true if the previous task execution completed successfully + * @return the Action to apply */ - def shouldTerminate(count: Int): Boolean + def preWatch(count: Int, lastStatus: Boolean): Watched.Action /** * Callback that is invoked whenever a file system vent is detected. The next step of the watch @@ -378,6 +428,15 @@ trait WatchConfig { */ def onWatchEvent(event: Event[Path]): Watched.Action + /** + * Transforms the state after the watch terminates. + * @param action the [[Watched.Action Action]] that caused the build to terminate + * @param command the command that the watch was repeating + * @param state the initial state prior to the start of continuous execution + * @return the updated state. + */ + def onWatchTerminated(action: Watched.Action, command: String, state: State): State + /** * The optional message to log when a build is triggered. * @param typedPath the path that triggered the build @@ -400,13 +459,20 @@ trait WatchConfig { object WatchConfig { /** - * Create an instance of [[WatchConfig]]. + * Create an instance of [[WatchConfig]]. * @param logger logger for watch events * @param fileEventMonitor the monitor for file system events. * @param handleInput callback that is periodically invoked to check whether to continue or * terminate the watch based on user input. It is also possible to, for * example time out the watch using this callback. + * @param preWatch callback to invoke before waiting for updates from the sbt.io.FileEventMonitor. + * The input parameters are the current iteration count and whether or not + * the last invocation of the command was successful. Typical uses would be to + * terminate the watch after a fixed number of iterations or to terminate the + * watch if the command was unsuccessful. * @param onWatchEvent callback that is invoked when + * @param onWatchTerminated callback that is invoked to update the state after the watch + * terminates. * @param triggeredMessage optional message that will be logged when a new build is triggered. * The input parameters are the sbt.io.TypedPath that triggered the new * build and the current iteration count. @@ -418,25 +484,29 @@ object WatchConfig { logger: Logger, fileEventMonitor: FileEventMonitor[Path], handleInput: () => Watched.Action, - shouldTerminate: Int => Boolean, + preWatch: (Int, Boolean) => Watched.Action, onWatchEvent: Event[Path] => Watched.Action, + onWatchTerminated: (Watched.Action, String, State) => State, triggeredMessage: (TypedPath, Int) => Option[String], watchingMessage: Int => Option[String] ): WatchConfig = { val l = logger val fem = fileEventMonitor val hi = handleInput - val st = shouldTerminate + val pw = preWatch val owe = onWatchEvent + val owt = onWatchTerminated val tm = triggeredMessage val wm = watchingMessage new WatchConfig { override def logger: Logger = l override def fileEventMonitor: FileEventMonitor[Path] = fem override def handleInput(): Watched.Action = hi() - override def shouldTerminate(count: Int): Boolean = - st(count) + override def preWatch(count: Int, lastResult: Boolean): Watched.Action = + pw(count, lastResult) override def onWatchEvent(event: Event[Path]): Watched.Action = owe(event) + override def onWatchTerminated(action: Watched.Action, command: String, state: State): State = + owt(action, command, state) override def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] = tm(typedPath, count) override def watchingMessage(count: Int): Option[String] = wm(count) diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 1ea7f36f7..612d232b3 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -29,7 +29,7 @@ class WatchedSpec extends FlatSpec with Matchers { fileEventMonitor: Option[FileEventMonitor[Path]] = None, logger: Logger = NullLogger, handleInput: () => Action = () => Ignore, - shouldTerminate: Int => Boolean = _ => true, + preWatch: (Int, Boolean) => Action = (_, _) => CancelWatch, onWatchEvent: Event[Path] => Action = _ => Ignore, triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None, watchingMessage: Int => Option[String] = _ => None @@ -41,8 +41,9 @@ class WatchedSpec extends FlatSpec with Matchers { logger = logger, monitor, handleInput, - shouldTerminate, + preWatch, onWatchEvent, + (_, _, state) => state, triggeredMessage, watchingMessage ) @@ -50,19 +51,19 @@ class WatchedSpec extends FlatSpec with Matchers { } "Watched.watch" should "stop" in IO.withTemporaryDirectory { dir => val config = Defaults.config(sources = Seq(WatchSource(dir.toRealPath))) - Watched.watch(() => Right(true), config) should be(()) + Watched.watch(() => Right(true), config) shouldBe CancelWatch } it should "trigger" in IO.withTemporaryDirectory { dir => val triggered = new AtomicBoolean(false) val config = Defaults.config( sources = Seq(WatchSource(dir.toRealPath)), - shouldTerminate = count => count == 2, + preWatch = (count, _) => if (count == 2) CancelWatch else Ignore, onWatchEvent = _ => { triggered.set(true); Trigger }, watchingMessage = _ => { new File(dir, "file").createNewFile; None } ) - Watched.watch(() => Right(true), config) should be(()) + Watched.watch(() => Right(true), config) shouldBe CancelWatch assert(triggered.get()) } it should "filter events" in IO.withTemporaryDirectory { dir => @@ -72,12 +73,12 @@ class WatchedSpec extends FlatSpec with Matchers { val bar = realDir.toPath.resolve("bar") val config = Defaults.config( sources = Seq(WatchSource(realDir)), - shouldTerminate = count => count == 2, + preWatch = (count, _) => if (count == 2) CancelWatch else Ignore, onWatchEvent = e => if (e.entry.typedPath.getPath == foo) Trigger else Ignore, triggeredMessage = (tp, _) => { queue += tp; None }, watchingMessage = _ => { Files.createFile(bar); Thread.sleep(5); Files.createFile(foo); None } ) - Watched.watch(() => Right(true), config) should be(()) + Watched.watch(() => Right(true), config) shouldBe CancelWatch queue.toIndexedSeq.map(_.getPath) shouldBe Seq(foo) } it should "enforce anti-entropy" in IO.withTemporaryDirectory { dir => @@ -87,21 +88,41 @@ class WatchedSpec extends FlatSpec with Matchers { val bar = realDir.toPath.resolve("bar") val config = Defaults.config( sources = Seq(WatchSource(realDir)), - shouldTerminate = count => count == 3, + preWatch = (count, _) => if (count == 3) CancelWatch else Ignore, onWatchEvent = _ => Trigger, triggeredMessage = (tp, _) => { queue += tp; None }, watchingMessage = count => { - if (count == 1) Files.createFile(bar) - else if (count == 2) { - bar.toFile.setLastModified(5000) - Files.createFile(foo) + count match { + case 1 => Files.createFile(bar) + case 2 => + bar.toFile.setLastModified(5000) + Files.createFile(foo) + case _ => } None } ) - Watched.watch(() => Right(true), config) should be(()) + Watched.watch(() => Right(true), config) shouldBe CancelWatch queue.toIndexedSeq.map(_.getPath) shouldBe Seq(bar, foo) } + it should "halt on error" in IO.withTemporaryDirectory { dir => + val halted = new AtomicBoolean(false) + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + preWatch = (_, lastStatus) => if (lastStatus) Ignore else { halted.set(true); HandleError } + ) + Watched.watch(() => Right(false), config) shouldBe HandleError + assert(halted.get()) + } + it should "reload" in IO.withTemporaryDirectory { dir => + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + preWatch = (_, _) => Ignore, + onWatchEvent = _ => Reload, + watchingMessage = _ => { new File(dir, "file").createNewFile(); None } + ) + Watched.watch(() => Right(true), config) shouldBe Reload + } } object WatchedSpec { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 8e41ada1d..76fbb59a8 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -250,6 +250,7 @@ object Defaults extends BuildCommon { Nil }, watchSources :== Nil, + watchProjectSources :== Nil, skip :== false, taskTemporaryDirectory := { val dir = IO.createTemporaryDirectory; dir.deleteOnExit(); dir }, onComplete := { @@ -384,6 +385,13 @@ object Defaults extends BuildCommon { else Nil bases.map(b => new Source(b, include, exclude)) ++ baseSources }, + watchProjectSources in ConfigGlobal := (watchProjectSources in ConfigGlobal).value ++ { + val baseDir = baseDirectory.value + Seq( + new Source(baseDir, "*.sbt", HiddenFileFilter, recursive = false), + new Source(baseDir / "project", "*.sbt" || "*.scala", HiddenFileFilter, recursive = true) + ) + }, managedSourceDirectories := Seq(sourceManaged.value), managedSources := generate(sourceGenerators).value, sourceGenerators :== Nil, @@ -606,17 +614,22 @@ object Defaults extends BuildCommon { clean := (Def.task { IO.delete(cleanFiles.value) } tag (Tags.Clean)).value, consoleProject := consoleProjectTask.value, watchTransitiveSources := watchTransitiveSourcesTask.value, + watchProjectTransitiveSources := watchTransitiveSourcesTaskImpl(watchProjectSources).value, watchOnEvent := { val sources = watchTransitiveSources.value + val projectSources = watchProjectTransitiveSources.value e => - if (sources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Trigger else Watched.Ignore + if (sources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Trigger + else if (projectSources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Reload + else Watched.Ignore }, watchHandleInput := Watched.handleInput, - watchShouldTerminate := { _ => - false + watchPreWatch := { (_, _) => + Watched.Ignore }, + watchOnTermination := Watched.onTermination, watchConfig := { - val sources = watchTransitiveSources.value + val sources = watchTransitiveSources.value ++ watchProjectTransitiveSources.value val extracted = Project.extract(state.value) val wm = extracted .getOpt(watchingMessage) @@ -634,8 +647,9 @@ object Defaults extends BuildCommon { logger, viewConfig.newMonitor(viewConfig.newDataView(), sources, logger), watchHandleInput.value, - watchShouldTerminate.value, + watchPreWatch.value, watchOnEvent.value, + watchOnTermination.value, tm, wm ) @@ -648,10 +662,15 @@ object Defaults extends BuildCommon { def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = generators { _.join.map(_.flatten) } - def watchTransitiveSourcesTask: Initialize[Task[Seq[Source]]] = { + def watchTransitiveSourcesTask: Initialize[Task[Seq[Source]]] = + watchTransitiveSourcesTaskImpl(watchSources) + + private def watchTransitiveSourcesTaskImpl( + key: TaskKey[Seq[Source]] + ): Initialize[Task[Seq[Source]]] = { import ScopeFilter.Make.{ inDependencies => inDeps, _ } val selectDeps = ScopeFilter(inAggregates(ThisProject) || inDeps(ThisProject)) - val allWatched = (watchSources ?? Nil).all(selectDeps) + val allWatched = (key ?? Nil).all(selectDeps) Def.task { allWatched.value.flatten } } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 66ffb8808..d2875ca9c 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -152,8 +152,11 @@ object Keys { val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) val watchHandleInput = settingKey[() => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) val watchOnEvent = taskKey[Event[JPath] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) + val watchOnTermination = taskKey[(Watched.Action, String, State) => State]("Transforms the input state after the continuous build completes.").withRank(BMinusSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) - val watchShouldTerminate = settingKey[Int => Boolean]("Function that may terminate a continuous build based on the number of iterations.").withRank(BMinusSetting) + val watchProjectSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources for the sbt meta project to watch to trigger a reload.").withRank(CSetting) + val watchProjectTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for the sbt meta project to watch to trigger a reload.").withRank(CSetting) + val watchPreWatch = settingKey[(Int, Boolean) => Watched.Action]("Function that may terminate a continuous build based on the number of iterations and the last result").withRank(BMinusSetting) val watchSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in this project for continuous execution to watch for changes.").withRank(BMinusSetting) val watchStartMessage = settingKey[Int => Option[String]]("The message to show when triggered execution waits for sources to change. The parameter is the current watch iteration count.").withRank(DSetting) val watchTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for continuous execution to watch.").withRank(CSetting) diff --git a/sbt/src/sbt-test/watch/on-start-watch/build.sbt b/sbt/src/sbt-test/watch/on-start-watch/build.sbt index d992f3473..1c6dab6c1 100644 --- a/sbt/src/sbt-test/watch/on-start-watch/build.sbt +++ b/sbt/src/sbt-test/watch/on-start-watch/build.sbt @@ -1,10 +1,50 @@ +import scala.util.Try + val checkCount = inputKey[Unit]("check that compile has run a specified number of times") +val checkReloadCount = inputKey[Unit]("check whether the project was reloaded") +val failingTask = taskKey[Unit]("should always fail") +val maybeReload = settingKey[(Int, Boolean) => Watched.Action]("possibly reload") +val resetCount = taskKey[Unit]("reset compile count") +val reloadFile = settingKey[File]("get the current reload file") checkCount := { val expected = Def.spaceDelimited().parsed.head.toInt - assert(Count.get == expected) + if (Count.get != expected) + throw new IllegalStateException(s"Expected ${expected} compilation runs, got ${Count.get}") } +maybeReload := { (_, _) => + if (Count.reloadCount(reloadFile.value) == 0) Watched.Reload else Watched.CancelWatch +} + +reloadFile := baseDirectory.value / "reload-count" + +resetCount := { + Count.reset() +} + +failingTask := { + throw new IllegalStateException("failed") +} + +watchPreWatch := maybeReload.value + +checkReloadCount := { + val expected = Def.spaceDelimited().parsed.head.toInt + assert(Count.reloadCount(reloadFile.value) == expected) +} + +val addReloadShutdownHook = Command.command("addReloadShutdownHook") { state => + state.addExitHook { + val base = Project.extract(state).get(baseDirectory) + val file = base / "reload-count" + val currentCount = Try(Count.reloadCount(file)).getOrElse(0) + IO.write(file, s"${currentCount + 1}".getBytes) + } +} + +commands += addReloadShutdownHook + Compile / compile := { Count.increment() // Trigger a new build by updating the last modified time diff --git a/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala index 0698b75ff..67d3bf940 100644 --- a/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala +++ b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala @@ -1,6 +1,10 @@ +import sbt._ +import scala.util.Try + object Count { private var count = 0 def get: Int = count def increment(): Unit = count += 1 def reset(): Unit = count = 0 -} \ No newline at end of file + def reloadCount(file: File): Int = Try(IO.read(file).toInt).getOrElse(0) +} diff --git a/sbt/src/sbt-test/watch/on-start-watch/test b/sbt/src/sbt-test/watch/on-start-watch/test index f5fa900e7..37781fce3 100644 --- a/sbt/src/sbt-test/watch/on-start-watch/test +++ b/sbt/src/sbt-test/watch/on-start-watch/test @@ -1,4 +1,28 @@ +# verify that reloading occurs if watchPreWatch returns Watched.Reload +> addReloadShutdownHook +> checkReloadCount 0 +> ~compile +> checkReloadCount 1 + # verify that the watch terminates when we reach the specified count -> set watchShouldTerminate := { count => count == 2 } +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.CancelWatch else Watched.Ignore } > ~compile > checkCount 2 + +# verify that the watch terminates and returns an error when we reach the specified count +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.HandleError else Watched.Ignore } +# Returning Watched.HandleError causes the '~' command to fail +-> ~compile +> checkCount 2 + +# verify that a re-build is triggered when we reach the specified count +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.Trigger else if (count == 3) Watched.CancelWatch else Watched.Ignore } +> ~compile +> checkCount 3 + +# verify that the watch exits and returns an error if the task fails +> set watchPreWatch := { (_, lastStatus: Boolean) => if (lastStatus) Watched.Ignore else Watched.HandleError } +-> ~failingTask From 4347d21248eb5dc50ab44504578435714402ead8 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 25 Sep 2018 18:53:12 -0700 Subject: [PATCH 10/17] Add support for user defined Actions It may be useful for users to be able to return their own custom Action types in the Config callbacks. For a contrived example, a user could add a jar file in the .ivy2 directory to the watch sources and trigger a reboot full when that jar changes. --- main-command/src/main/scala/sbt/Watched.scala | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index b501a1f36..d11875dba 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -97,6 +97,13 @@ object Watched { */ case object Trigger extends Action + /** + * A user defined Action. It is not sealed so that the user can create custom instances. If any + * of the [[Config]] callbacks, e.g. [[Config.onWatchEvent]], return an instance of [[Custom]], + * the watch will terminate. + */ + trait Custom extends Action + type WatchSource = Source def terminateWatch(key: Int): Boolean = Watched.isEnter(key) /* @@ -276,8 +283,8 @@ object Watched { @tailrec def nextAction(): Action = { config.handleInput() match { - case action @ (CancelWatch | HandleError | Reload) => action - case Trigger => Trigger + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action + case Trigger => Trigger case _ => val events = config.fileEventMonitor.poll(10.millis) val next = events match { @@ -286,9 +293,10 @@ object Watched { /* * We traverse all of the events and find the one for which we give the highest * weight. - * HandleError > CancelWatch > Reload > Trigger > Ignore + * Custom > HandleError > CancelWatch > Reload > Trigger > Ignore */ tail.foldLeft((config.onWatchEvent(head), Some(head))) { + case (current @ (_: Custom, _), _) => current case (current @ (action, _), event) => config.onWatchEvent(event) match { case HandleError => (HandleError, Some(event)) @@ -302,8 +310,11 @@ object Watched { } // Note that nextAction should never return Ignore. next match { - case (action @ (HandleError | CancelWatch), Some(event)) => - val cause = if (action == HandleError) "error" else "cancellation" + case (action @ (HandleError | CancelWatch | _: Custom), Some(event)) => + val cause = + if (action == HandleError) "error" + else if (action.isInstanceOf[Custom]) action.toString + else "cancellation" logger.debug(s"Stopping watch due to $cause from ${event.entry.typedPath.getPath}") action case (Trigger, Some(event)) => @@ -324,11 +335,11 @@ object Watched { case Ignore => config.watchingMessage(count).foreach(info) nextAction() match { - case action @ (CancelWatch | HandleError | Reload) => action - case _ => impl(count + 1) + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action + case _ => impl(count + 1) } - case Trigger => impl(count + 1) - case action @ (CancelWatch | HandleError | Reload) => action + case Trigger => impl(count + 1) + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action } case Left(e) => logger.error(s"Terminating watch due to Unexpected error: $e") From d31fae59f7dc7239fb49b6d9853d6eb38ecbea37 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 25 Aug 2018 16:43:48 -0700 Subject: [PATCH 11/17] Add global file repository task Every time that the compile task is run, there are potentially a large number of iops that must occur in order for sbt to generate the source file list as well as for zinc to check which files have changed since the last build. This can lead to a noticeable delay between when a build is started (either manually or by triggered execution) and when compilation actually begins. To reduce this latency, I am adding a global view of the file system that will be stored in BasicKeys.globalFileTreeView. To make this work, I introduce the StampedFile trait, which augments the java.io.File class with a stamp method that returns the zinc stamp for the file. For source files, this will be a hash of the file, while for binaries, it is just the last modified time. In order to gain access to the sbt.internal.inc.Stamper class, I had to append addSbtZinc to the commandProj configurations. This view may or may not use an in-memory cache of the file system tree to return the results. Because there is always the risk of the cache getting out of sync with the actual file system, I both make it optional to use a cache and provide a mechanism for flushing the cache. Moreover, the in-memory cache implementation in sbt.io, which is backed by a swoval FileTreeRepository, has the property that touching a monitored directory invalidates the entire directory within the cache, so the flush command isn't even strictly needed in general. Because caching is optional, the global is of a FileTreeDataView, which doesn't specify a caching strategy. Subsequent commits will make use of this to potentially speed up incremental compilation by caching the Stamps of the source files so that zinc does not need to compute the hashes itself and will allow for continuous builds to use the cache to monitor events instead of creating a new, standalone FileEventMonitor. --- build.sbt | 3 +- .../main/scala/sbt/BasicCommandStrings.scala | 4 ++ .../src/main/scala/sbt/BasicKeys.scala | 6 +++ .../main/scala/sbt/FileTreeViewConfig.scala | 26 +++++------ main-command/src/main/scala/sbt/Watched.scala | 45 +++++++++++++++---- .../src/test/scala/sbt/WatchedSpec.scala | 6 +-- main/src/main/scala/sbt/Defaults.scala | 7 ++- main/src/main/scala/sbt/Keys.scala | 6 +-- main/src/main/scala/sbt/Main.scala | 28 ++++++++++-- 9 files changed, 96 insertions(+), 35 deletions(-) diff --git a/build.sbt b/build.sbt index 0dd097327..5d8694d49 100644 --- a/build.sbt +++ b/build.sbt @@ -507,7 +507,8 @@ lazy val commandProj = (project in file("main-command")) addSbtUtilLogging, addSbtCompilerInterface, addSbtCompilerClasspath, - addSbtLmCore + addSbtLmCore, + addSbtZinc ) // The core macro project defines the main logic of the DSL, abstracted diff --git a/main-command/src/main/scala/sbt/BasicCommandStrings.scala b/main-command/src/main/scala/sbt/BasicCommandStrings.scala index 6eeb06ee5..7ec8c794c 100644 --- a/main-command/src/main/scala/sbt/BasicCommandStrings.scala +++ b/main-command/src/main/scala/sbt/BasicCommandStrings.scala @@ -231,4 +231,8 @@ $AliasCommand name= val ContinuousExecutePrefix = "~" def continuousDetail = "Executes the specified command whenever source files change." def continuousBriefHelp = (ContinuousExecutePrefix + " ", continuousDetail) + def FlushFileTreeRepository = "flushFileTreeRepository" + def FlushDetailed: String = + "Resets the global file repository in the event that the repository has become inconsistent " + + "with the file system." } diff --git a/main-command/src/main/scala/sbt/BasicKeys.scala b/main-command/src/main/scala/sbt/BasicKeys.scala index ff466e9e8..66de24ce6 100644 --- a/main-command/src/main/scala/sbt/BasicKeys.scala +++ b/main-command/src/main/scala/sbt/BasicKeys.scala @@ -11,6 +11,7 @@ import java.io.File import sbt.internal.util.AttributeKey import sbt.internal.inc.classpath.ClassLoaderCache import sbt.internal.server.ServerHandler +import sbt.io.FileTreeDataView import sbt.librarymanagement.ModuleID import sbt.util.Level @@ -100,6 +101,11 @@ object BasicKeys { "List of template resolver infos.", 1000 ) + private[sbt] val globalFileTreeView = AttributeKey[FileTreeDataView[StampedFile]]( + "globalFileTreeView", + "provides a view into the file system that may or may not cache the tree in memory", + 1000 + ) } case class TemplateResolverInfo(module: ModuleID, implementationClass: String) diff --git a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala index 10414348b..9fe1771d4 100644 --- a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala +++ b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala @@ -6,11 +6,9 @@ */ package sbt -import java.nio.file.Path - import sbt.Watched.WatchSource import sbt.internal.io.{ WatchServiceBackedObservable, WatchState } -import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView, TypedPath } +import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView } import sbt.util.Logger import scala.concurrent.duration.FiniteDuration @@ -19,21 +17,21 @@ import scala.concurrent.duration.FiniteDuration * Configuration for viewing and monitoring the file system. */ final class FileTreeViewConfig private ( - val newDataView: () => FileTreeDataView[Path], + val newDataView: () => FileTreeDataView[StampedFile], val newMonitor: ( - FileTreeDataView[Path], + FileTreeDataView[StampedFile], Seq[WatchSource], Logger - ) => FileEventMonitor[Path] + ) => FileEventMonitor[StampedFile] ) object FileTreeViewConfig { /** * Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded - * by {{{sbt.io.FileTreeDataView[Path]}}}. The reason for this is to ensure that a + * by {{{sbt.io.FileTreeDataView[StampedFile]}}}. The reason for this is to ensure that a * sbt.io.FileTreeDataView that is instantiated by [[FileTreeViewConfig.newDataView]] can be * passed into [[FileTreeViewConfig.newMonitor]] without constraining the type of view to be - * {{{sbt.io.FileTreeDataView[Path]}}}. + * {{{sbt.io.FileTreeDataView[StampedFile]}}}. * @param newDataView create a new sbt.io.FileTreeDataView. This value may be cached in a global * attribute * @param newMonitor create a new sbt.io.FileEventMonitor using the sbt.io.FileTreeDataView @@ -41,13 +39,13 @@ object FileTreeViewConfig { * @tparam T the subtype of sbt.io.FileTreeDataView that is returned by [[FileTreeViewConfig.newDataView]] * @return a [[FileTreeViewConfig]] instance. */ - def apply[T <: FileTreeDataView[Path]]( + def apply[T <: FileTreeDataView[StampedFile]]( newDataView: () => T, - newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[Path] + newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[StampedFile] ): FileTreeViewConfig = new FileTreeViewConfig( newDataView, - (view: FileTreeDataView[Path], sources: Seq[WatchSource], logger: Logger) => + (view: FileTreeDataView[StampedFile], sources: Seq[WatchSource], logger: Logger) => newMonitor(view.asInstanceOf[T], sources, logger) ) @@ -61,14 +59,14 @@ object FileTreeViewConfig { */ def default(pollingInterval: FiniteDuration, antiEntropy: FiniteDuration): FileTreeViewConfig = FileTreeViewConfig( - () => FileTreeView.DEFAULT.asDataView(_.getPath), - (_: FileTreeDataView[Path], sources, logger) => { + () => FileTreeView.DEFAULT.asDataView(StampedFile.converter), + (_: FileTreeDataView[StampedFile], sources, logger) => { val ioLogger: sbt.io.WatchLogger = msg => logger.debug(msg.toString) FileEventMonitor.antiEntropy( new WatchServiceBackedObservable( WatchState.empty(Watched.createWatchService(), sources), pollingInterval, - (_: TypedPath).getPath, + StampedFile.converter, closeService = true, ioLogger ), diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index d11875dba..9f8dc25a6 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -8,7 +8,7 @@ package sbt import java.io.File -import java.nio.file.{ FileSystems, Path } +import java.nio.file.FileSystems import sbt.BasicCommandStrings.{ ContinuousExecutePrefix, @@ -18,6 +18,7 @@ import sbt.BasicCommandStrings.{ } import sbt.BasicCommands.otherCommandParser import sbt.internal.LegacyWatched +import sbt.internal.inc.Stamper import sbt.internal.io.{ EventMonitor, Source, WatchState } import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const @@ -25,6 +26,7 @@ import sbt.internal.util.complete.DefaultParsers import sbt.io.FileEventMonitor.Event import sbt.io._ import sbt.util.{ Level, Logger } +import xsbti.compile.analysis.Stamp import scala.annotation.tailrec import scala.concurrent.duration._ @@ -99,8 +101,8 @@ object Watched { /** * A user defined Action. It is not sealed so that the user can create custom instances. If any - * of the [[Config]] callbacks, e.g. [[Config.onWatchEvent]], return an instance of [[Custom]], - * the watch will terminate. + * of the [[WatchConfig]] callbacks, e.g. [[WatchConfig.onWatchEvent]], return an instance of + * [[Custom]], the watch will terminate. */ trait Custom extends Action @@ -414,7 +416,7 @@ trait WatchConfig { * * @return an sbt.io.FileEventMonitor instance. */ - def fileEventMonitor: FileEventMonitor[Path] + def fileEventMonitor: FileEventMonitor[StampedFile] /** * A function that is periodically invoked to determine whether the watch should stop or @@ -437,7 +439,7 @@ trait WatchConfig { * @param event the detected sbt.io.FileEventMonitor.Event. * @return the next [[Watched.Action Action]] to run. */ - def onWatchEvent(event: Event[Path]): Watched.Action + def onWatchEvent(event: Event[StampedFile]): Watched.Action /** * Transforms the state after the watch terminates. @@ -493,10 +495,10 @@ object WatchConfig { */ def default( logger: Logger, - fileEventMonitor: FileEventMonitor[Path], + fileEventMonitor: FileEventMonitor[StampedFile], handleInput: () => Watched.Action, preWatch: (Int, Boolean) => Watched.Action, - onWatchEvent: Event[Path] => Watched.Action, + onWatchEvent: Event[StampedFile] => Watched.Action, onWatchTerminated: (Watched.Action, String, State) => State, triggeredMessage: (TypedPath, Int) => Option[String], watchingMessage: Int => Option[String] @@ -511,11 +513,11 @@ object WatchConfig { val wm = watchingMessage new WatchConfig { override def logger: Logger = l - override def fileEventMonitor: FileEventMonitor[Path] = fem + override def fileEventMonitor: FileEventMonitor[StampedFile] = fem override def handleInput(): Watched.Action = hi() override def preWatch(count: Int, lastResult: Boolean): Watched.Action = pw(count, lastResult) - override def onWatchEvent(event: Event[Path]): Watched.Action = owe(event) + override def onWatchEvent(event: Event[StampedFile]): Watched.Action = owe(event) override def onWatchTerminated(action: Watched.Action, command: String, state: State): State = owt(action, command, state) override def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] = @@ -524,3 +526,28 @@ object WatchConfig { } } } + +trait StampedFile extends File { + def stamp: Stamp +} +object StampedFile { + val sourceConverter: TypedPath => StampedFile = + new StampedFileImpl(_: TypedPath, forceLastModified = false) + val binaryConverter: TypedPath => StampedFile = + new StampedFileImpl(_: TypedPath, forceLastModified = true) + val converter: TypedPath => StampedFile = (tp: TypedPath) => + tp.getPath.toString match { + case s if s.endsWith(".jar") => binaryConverter(tp) + case s if s.endsWith(".class") => binaryConverter(tp) + case _ => sourceConverter(tp) + } + + private class StampedFileImpl(typedPath: TypedPath, forceLastModified: Boolean) + extends java.io.File(typedPath.getPath.toString) + with StampedFile { + override val stamp: Stamp = + if (forceLastModified || typedPath.isDirectory) + Stamper.forLastModified(typedPath.getPath.toFile) + else Stamper.forHash(typedPath.getPath.toFile) + } +} diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 612d232b3..3babca877 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -8,7 +8,7 @@ package sbt import java.io.File -import java.nio.file.{ Files, Path } +import java.nio.file.Files import java.util.concurrent.atomic.AtomicBoolean import org.scalatest.{ FlatSpec, Matchers } @@ -26,11 +26,11 @@ class WatchedSpec extends FlatSpec with Matchers { private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis, 50.millis) def config( sources: Seq[WatchSource], - fileEventMonitor: Option[FileEventMonitor[Path]] = None, + fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None, logger: Logger = NullLogger, handleInput: () => Action = () => Ignore, preWatch: (Int, Boolean) => Action = (_, _) => CancelWatch, - onWatchEvent: Event[Path] => Action = _ => Ignore, + onWatchEvent: Event[StampedFile] => Action = _ => Ignore, triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None, watchingMessage: Int => Option[String] = _ => None ): WatchConfig = { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 76fbb59a8..393328a40 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -42,6 +42,7 @@ import sbt.io.{ AllPassFilter, DirectoryFilter, FileFilter, + FileTreeView, GlobFilter, Hash, HiddenFileFilter, @@ -271,6 +272,9 @@ object Defaults extends BuildCommon { }, watchStartMessage := Watched.defaultStartWatch, fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value), + fileTreeView := state.value + .get(BasicKeys.globalFileTreeView) + .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), watchLogger := streams.value.log, watchService :== { () => @@ -642,10 +646,9 @@ object Defaults extends BuildCommon { ) .getOrElse(watchTriggeredMessage.value) val logger = watchLogger.value - val viewConfig = fileTreeViewConfig.value WatchConfig.default( logger, - viewConfig.newMonitor(viewConfig.newDataView(), sources, logger), + fileTreeViewConfig.value.newMonitor(fileTreeView.value, sources, logger), watchHandleInput.value, watchPreWatch.value, watchOnEvent.value, diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index d2875ca9c..b0513181b 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -8,7 +8,6 @@ package sbt import java.io.File -import java.nio.file.{ Path => JPath } import java.net.URL import scala.concurrent.duration.{ FiniteDuration, Duration } import Def.ScopedKey @@ -41,7 +40,7 @@ import sbt.internal.{ SessionSettings, LogManager } -import sbt.io.{ FileFilter, TypedPath, WatchService } +import sbt.io.{ FileFilter, FileTreeDataView, TypedPath, WatchService } import sbt.io.FileEventMonitor.Event import sbt.internal.io.WatchState import sbt.internal.server.ServerHandler @@ -146,12 +145,13 @@ object Keys { @deprecated("This is no longer used for continuous execution", "1.3.0") val watch = SettingKey(BasicKeys.watch) val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) + val fileTreeView = taskKey[FileTreeDataView[StampedFile]]("A view of the file system") val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting) val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) val watchHandleInput = settingKey[() => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) - val watchOnEvent = taskKey[Event[JPath] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) + val watchOnEvent = taskKey[Event[StampedFile] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) val watchOnTermination = taskKey[(Watched.Action, String, State) => State]("Transforms the input state after the continuous build completes.").withRank(BMinusSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) val watchProjectSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources for the sbt meta project to watch to trigger a reload.").withRank(CSetting) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 045e36ac7..358ee2c62 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -49,7 +49,7 @@ import Project.LoadAction import xsbti.compile.CompilerCache import scala.annotation.tailrec -import sbt.io.IO +import sbt.io.{ FileTreeDataView, IO } import sbt.io.syntax._ import java.io.{ File, IOException } import java.net.URI @@ -242,7 +242,8 @@ object BuiltinCommands { boot, initialize, act, - continuous + continuous, + flushFileTreeRepository ) ++ allBasicCommands def DefaultBootCommands: Seq[String] = @@ -858,7 +859,7 @@ object BuiltinCommands { val session = Load.initialSession(structure, eval, s0) SessionSettings.checkSession(session, s) - Project.setProject(session, structure, s) + registerGlobalFileRepository(Project.setProject(session, structure, s)) } def registerCompilerCache(s: State): State = { @@ -876,6 +877,27 @@ object BuiltinCommands { } s.put(Keys.stateCompilerCache, cache) } + def registerGlobalFileRepository(s: State): State = { + val extracted = Project.extract(s) + try { + val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s) + val view: FileTreeDataView[StampedFile] = config.newDataView() + val newState = s.addExitHook { + view.close() + s.attributes.remove(BasicKeys.globalFileTreeView) + () + } + newState.get(BasicKeys.globalFileTreeView).foreach(_.close()) + newState.put(BasicKeys.globalFileTreeView, view) + } catch { + case NonFatal(_) => s + } + } + + def flushFileTreeRepository: Command = { + val help = Help.more(FlushFileTreeRepository, FlushDetailed) + Command.command(FlushFileTreeRepository, help)(registerGlobalFileRepository) + } def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 => import sbt.internal.{ ConsolePromptEvent, ConsoleUnpromptEvent } From 2b2b84f589f7aba5cce5e5076c7bf979f01ad209 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sun, 26 Aug 2018 11:16:58 -0700 Subject: [PATCH 12/17] Use FileTreeDataView to collect files Now that we have the fileTreeView task, we can generalized the process of collecting files from the view (which may or may not actually cache the underlying file tree). I moved the implementation of collectFiles and addBaseSources into the new FileManagement object because Defaults is already too large of a file. When we query the view, we also need to register the directory we're listing because if the underlying view is a cache, we must call register before any entries will be available. Because FileTreeDataView doesn't have a register method, I implement registration with a simple implicit class that pattern matches on the underlying type and only calls register if it is actually a FileRepository. A side effect of this change is that the underlying files returned by collectFiles and appendBaseSources are StampedFile instances. This is so that in a subsequent commit, I can add a Zinc external hook that will read these stamps from the files in the source input array rather than compute the stamp on the fly. This leads to a substantial reduction in Zinc startup time for projects with many source files. The file filters also may be applied more quickly because the isDirectory property (which we check for all source files) is read from a cached value rather than requiring a stat. I had to update a few of the scripted tests to use the `1.2.0` FileTreeViewConfig because those tests would copy a file and then immediately re-compile. The latency of cache invalidation is O(1-10ms), but not instantaneous so it's necessary to either use a non-caching FileTreeView or add a sleep between updates and compilation. I chose the former. --- .../main/scala/sbt/FileTreeViewConfig.scala | 48 +++++++++-- .../src/test/scala/sbt/WatchedSpec.scala | 2 +- main/src/main/scala/sbt/Defaults.scala | 45 +++++----- .../scala/sbt/internal/FileManagement.scala | 85 +++++++++++++++++++ 4 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/FileManagement.scala diff --git a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala index 9fe1771d4..4b1179bfe 100644 --- a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala +++ b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala @@ -8,10 +8,11 @@ package sbt import sbt.Watched.WatchSource import sbt.internal.io.{ WatchServiceBackedObservable, WatchState } -import sbt.io.{ FileEventMonitor, FileTreeDataView, FileTreeView } +import sbt.io._ +import FileTreeDataView.{ Observable, Observer } import sbt.util.Logger -import scala.concurrent.duration.FiniteDuration +import scala.concurrent.duration._ /** * Configuration for viewing and monitoring the file system. @@ -25,6 +26,11 @@ final class FileTreeViewConfig private ( ) => FileEventMonitor[StampedFile] ) object FileTreeViewConfig { + private implicit class RepositoryOps(val repository: FileTreeRepository[StampedFile]) { + def register(sources: Seq[WatchSource]): Unit = sources foreach { s => + repository.register(s.base.toPath, if (s.recursive) Integer.MAX_VALUE else 0) + } + } /** * Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded @@ -50,14 +56,19 @@ object FileTreeViewConfig { ) /** - * Provides a default [[FileTreeViewConfig]]. This view does not cache entries. - * @param pollingInterval the maximum duration that the sbt.internal.io.EventMonitor will poll - * the underlying sbt.io.WatchService when monitoring for file events + * Provides a [[FileTreeViewConfig]] with semantics as close as possible to sbt 1.2.0. This means + * that there is no file tree caching and the sbt.io.FileEventMonitor will use an + * sbt.io.WatchService for monitoring the file system. + * @param delay the maximum delay for which the background thread will poll the + * sbt.io.WatchService for file system events * @param antiEntropy the duration of the period after a path triggers a build for which it is * quarantined from triggering another build * @return a [[FileTreeViewConfig]] instance. */ - def default(pollingInterval: FiniteDuration, antiEntropy: FiniteDuration): FileTreeViewConfig = + def sbt1_2_compat( + delay: FiniteDuration, + antiEntropy: FiniteDuration + ): FileTreeViewConfig = FileTreeViewConfig( () => FileTreeView.DEFAULT.asDataView(StampedFile.converter), (_: FileTreeDataView[StampedFile], sources, logger) => { @@ -65,7 +76,7 @@ object FileTreeViewConfig { FileEventMonitor.antiEntropy( new WatchServiceBackedObservable( WatchState.empty(Watched.createWatchService(), sources), - pollingInterval, + delay, StampedFile.converter, closeService = true, ioLogger @@ -75,4 +86,27 @@ object FileTreeViewConfig { ) } ) + + /** + * Provides a default [[FileTreeViewConfig]]. This view caches entries and solely relies on + * file system events from the operating system to update its internal representation of the + * file tree. + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @return a [[FileTreeViewConfig]] instance. + */ + def default(antiEntropy: FiniteDuration): FileTreeViewConfig = + FileTreeViewConfig( + () => FileTreeRepository.default(StampedFile.converter), + (repository: FileTreeRepository[StampedFile], sources: Seq[WatchSource], logger: Logger) => { + repository.register(sources) + val copied = new Observable[StampedFile] { + override def addObserver(observer: Observer[StampedFile]): Int = + repository.addObserver(observer) + override def removeObserver(handle: Int): Unit = repository.removeObserver(handle) + override def close(): Unit = {} // Don't close the underlying observable + } + FileEventMonitor.antiEntropy(copied, antiEntropy, msg => logger.debug(msg.toString)) + } + ) } diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 3babca877..9d66b8f84 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -23,7 +23,7 @@ import scala.concurrent.duration._ class WatchedSpec extends FlatSpec with Matchers { object Defaults { - private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis, 50.millis) + private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis) def config( sources: Seq[WatchSource], fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None, diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 393328a40..1aa5bb35f 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -271,7 +271,7 @@ object Defaults extends BuildCommon { None }, watchStartMessage := Watched.defaultStartWatch, - fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value), + fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value), fileTreeView := state.value .get(BasicKeys.globalFileTreeView) .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), @@ -363,11 +363,13 @@ object Defaults extends BuildCommon { crossPaths.value ) }, - unmanagedSources := collectFiles( - unmanagedSourceDirectories, - includeFilter in unmanagedSources, - excludeFilter in unmanagedSources - ).value, + unmanagedSources := FileManagement + .collectFiles( + unmanagedSourceDirectories, + includeFilter in unmanagedSources, + excludeFilter in unmanagedSources + ) + .value, watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val baseDir = baseDirectory.value val bases = unmanagedSourceDirectories.value @@ -412,11 +414,13 @@ object Defaults extends BuildCommon { resourceDirectories := Classpaths .concatSettings(unmanagedResourceDirectories, managedResourceDirectories) .value, - unmanagedResources := collectFiles( - unmanagedResourceDirectories, - includeFilter in unmanagedResources, - excludeFilter in unmanagedResources - ).value, + unmanagedResources := FileManagement + .collectFiles( + unmanagedResourceDirectories, + includeFilter in unmanagedResources, + excludeFilter in unmanagedResources + ) + .value, watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val bases = unmanagedResourceDirectories.value val include = (includeFilter in unmanagedResources).value @@ -430,19 +434,11 @@ object Defaults extends BuildCommon { managedResources := generate(resourceGenerators).value, resources := Classpaths.concat(managedResources, unmanagedResources).value ) + def addBaseSources = FileManagement.appendBaseSources lazy val outputConfigPaths = Seq( classDirectory := crossTarget.value / (prefix(configuration.value.name) + "classes"), target in doc := crossTarget.value / (prefix(configuration.value.name) + "api") ) - def addBaseSources = Seq( - unmanagedSources := { - val srcs = unmanagedSources.value - val f = (includeFilter in unmanagedSources).value - val excl = (excludeFilter in unmanagedSources).value - val baseDir = baseDirectory.value - if (sourcesInBase.value) (srcs +++ baseDir * (f -- excl)).get else srcs - } - ) // This is included into JvmPlugin.projectSettings def compileBase = inTask(console)(compilersSetting :: Nil) ++ compileBaseGlobal ++ Seq( @@ -659,7 +655,7 @@ object Defaults extends BuildCommon { }, watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), watch := watchSetting.value, - fileTreeViewConfig := FileTreeViewConfig.default(pollInterval.value, watchAntiEntropy.value), + fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value) ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = @@ -1192,10 +1188,7 @@ object Defaults extends BuildCommon { dirs: ScopedTaskable[Seq[File]], filter: ScopedTaskable[FileFilter], excludes: ScopedTaskable[FileFilter] - ): Initialize[Task[Seq[File]]] = - Def.task { - dirs.toTask.value.descendantsExcept(filter.toTask.value, excludes.toTask.value).get - } + ): Initialize[Task[Seq[File]]] = FileManagement.collectFiles(dirs, filter, excludes) def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] = Def.setting { val f = artifactName.value @@ -1767,7 +1760,7 @@ object Defaults extends BuildCommon { lazy val compileSettings: Seq[Setting[_]] = configSettings ++ - (mainBgRunMainTask +: mainBgRunTask +: addBaseSources) ++ + (mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++ Classpaths.addUnmanagedLibrary lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks diff --git a/main/src/main/scala/sbt/internal/FileManagement.scala b/main/src/main/scala/sbt/internal/FileManagement.scala new file mode 100644 index 000000000..bbdbb346c --- /dev/null +++ b/main/src/main/scala/sbt/internal/FileManagement.scala @@ -0,0 +1,85 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import java.io.IOException +import java.nio.file.Path + +import sbt.Keys._ +import sbt.io.FileTreeDataView.Entry +import sbt.io.syntax.File +import sbt.io.{ FileFilter, FileTreeRepository, FileTreeDataView } +import sbt.{ Def, ScopedTaskable, StampedFile, Task } + +private[sbt] object FileManagement { + private[sbt] implicit class FileTreeDataViewOps[+T](val fileTreeDataView: FileTreeDataView[T]) { + def register(path: Path, maxDepth: Int): Either[IOException, Boolean] = { + fileTreeDataView match { + case r: FileTreeRepository[T] => r.register(path, maxDepth) + case _ => Right(false) + } + } + } + + private[sbt] def collectFiles( + dirs: ScopedTaskable[Seq[File]], + filter: ScopedTaskable[FileFilter], + excludes: ScopedTaskable[FileFilter] + ): Def.Initialize[Task[Seq[File]]] = + Def.task { + val sourceDirs = dirs.toTask.value + val view = fileTreeView.value + val include = filter.toTask.value + val ex = excludes.toTask.value + val sourceFilter: Entry[StampedFile] => Boolean = (entry: Entry[StampedFile]) => { + entry.value match { + case Right(sf) => include.accept(sf) && !ex.accept(sf) + case _ => false + } + } + sourceDirs.flatMap { dir => + view.register(dir.toPath, maxDepth = Integer.MAX_VALUE) + view + .listEntries(dir.toPath, maxDepth = Integer.MAX_VALUE, sourceFilter) + .map(e => e.value.getOrElse(e.typedPath.getPath.toFile)) + } + } + + private[sbt] def appendBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Seq( + unmanagedSources := { + val sources = unmanagedSources.value + val f = (includeFilter in unmanagedSources).value + val excl = (excludeFilter in unmanagedSources).value + val baseDir = baseDirectory.value + val view = fileTreeView.value + if (sourcesInBase.value) { + view.register(baseDir.toPath, maxDepth = 0) + sources ++ + view + .listEntries( + baseDir.toPath, + maxDepth = 0, + e => { + val tp = e.typedPath + /* + * The TypedPath has the isDirectory and isFile properties embedded. By overriding + * these methods in java.io.File, FileFilters may be applied without needing to + * stat the file (which is expensive) for isDirectory and isFile checks. + */ + val file = new java.io.File(tp.getPath.toString) { + override def isDirectory: Boolean = tp.isDirectory + override def isFile: Boolean = tp.isFile + } + f.accept(file) && !excl.accept(file) + } + ) + .flatMap(_.value.toOption) + } else sources + } + ) +} From b155ffb77bec86acfafd9fb1aae26565819b65eb Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 28 Aug 2018 09:17:58 -0700 Subject: [PATCH 13/17] Add support for polling some directories It is not always possible to monitor a directory using OS file system events. For example, inotify does not work with nfs. To work around this, I add support for a hybrid FileTreeViewConfig that caches a portion of the file system and monitors it with os file system notification, but that polls a subset of the directories. When we query the view using list or listEntries, we will actually query the file system for the polling directories while we will read from the cache for the remainder. When we are not in a continuous build (~ *), there is no polling of the pollingDirectories but the cache will continue to update the regular directories in the background. When we are in a continuous build, we use a PollingWatchService to poll the pollingDirectories and continue to use the regular repository callbacks for the other directories. I suspect that #4179 may be resolved by adding the directories for which monitoring is not working to the pollingDirectories task. --- .../main/scala/sbt/FileTreeViewConfig.scala | 62 ++++++++++++++++++- main/src/main/scala/sbt/Defaults.scala | 7 ++- main/src/main/scala/sbt/Keys.scala | 1 + 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala index 4b1179bfe..c1dd8fa6d 100644 --- a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala +++ b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala @@ -7,7 +7,7 @@ package sbt import sbt.Watched.WatchSource -import sbt.internal.io.{ WatchServiceBackedObservable, WatchState } +import sbt.internal.io.{ HybridPollingFileTreeRepository, WatchServiceBackedObservable, WatchState } import sbt.io._ import FileTreeDataView.{ Observable, Observer } import sbt.util.Logger @@ -109,4 +109,64 @@ object FileTreeViewConfig { FileEventMonitor.antiEntropy(copied, antiEntropy, msg => logger.debug(msg.toString)) } ) + + /** + * Provides a default [[FileTreeViewConfig]]. When the pollingSources argument is empty, it + * returns the same config as [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]]. + * Otherwise, it returns the same config as [[polling]]. + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @param pollingInterval the frequency with which the sbt.io.FileEventMonitor polls the file + * system for the paths included in pollingSources + * @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that + * will be periodically polled for changes during continuous builds. + * @return + */ + def default( + antiEntropy: FiniteDuration, + pollingInterval: FiniteDuration, + pollingSources: Seq[WatchSource] + ): FileTreeViewConfig = { + if (pollingSources.isEmpty) default(antiEntropy) + else polling(antiEntropy, pollingInterval, pollingSources) + } + + /** + * Provides a polling [[FileTreeViewConfig]]. Unlike the view returned by newDataView in + * [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]], + * the view returned by newDataView will not cache any portion of the file system tree that is is + * covered by the pollingSources parameter. The monitor that is generated by newMonitor, will + * poll these directories for changes rather than relying on file system events from the + * operating system. Any paths that are registered with the view that are not included in the + * pollingSources will be cached and monitored using file system events from the operating system + * in the same way that they are in the default view. + * + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @param pollingInterval the frequency with which the FileEventMonitor polls the file system + * for the paths included in pollingSources + * @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that + * will be periodically polled for changes during continuous builds. + * @return a [[FileTreeViewConfig]] instance. + */ + def polling( + antiEntropy: FiniteDuration, + pollingInterval: FiniteDuration, + pollingSources: Seq[WatchSource], + ): FileTreeViewConfig = FileTreeViewConfig( + () => FileTreeRepository.hybrid(StampedFile.converter, pollingSources: _*), + ( + repository: HybridPollingFileTreeRepository[StampedFile], + sources: Seq[WatchSource], + logger: Logger + ) => { + repository.register(sources) + FileEventMonitor + .antiEntropy( + repository.toPollingObservable(pollingInterval, sources, NullWatchLogger), + antiEntropy, + msg => logger.debug(msg.toString) + ) + } + ) } diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 1aa5bb35f..212d02442 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -250,6 +250,7 @@ object Defaults extends BuildCommon { extraLoggers :== { _ => Nil }, + pollingDirectories :== Nil, watchSources :== Nil, watchProjectSources :== Nil, skip :== false, @@ -271,7 +272,8 @@ object Defaults extends BuildCommon { None }, watchStartMessage := Watched.defaultStartWatch, - fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value), + fileTreeViewConfig := FileTreeViewConfig + .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value), fileTreeView := state.value .get(BasicKeys.globalFileTreeView) .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), @@ -655,7 +657,8 @@ object Defaults extends BuildCommon { }, watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), watch := watchSetting.value, - fileTreeViewConfig := FileTreeViewConfig.default(watchAntiEntropy.value) + fileTreeViewConfig := FileTreeViewConfig + .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value) ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b0513181b..ebf6a8cd1 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -147,6 +147,7 @@ object Keys { val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) val fileTreeView = taskKey[FileTreeDataView[StampedFile]]("A view of the file system") val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) + val pollingDirectories = settingKey[Seq[Watched.WatchSource]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting) val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting) val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) From 1f996185e15b56e287e46992e14a33da95c9bab8 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 6 Oct 2018 11:39:05 -0700 Subject: [PATCH 14/17] Only use a file repository for interactive sessions I realized that using the cache has the potential to cause issues for batch processing in CI if some tasks assume that a file created by one task will immediately be visible in the other. With the cache, there is typically on O(10ms) latency between a file being created and appearing in the cache (at least on OSX). When manually running commands, that latency doesn't matter. --- main/src/main/scala/sbt/Defaults.scala | 6 ++---- main/src/main/scala/sbt/internal/FileManagement.scala | 10 ++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 212d02442..2c4bf38e3 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -272,8 +272,7 @@ object Defaults extends BuildCommon { None }, watchStartMessage := Watched.defaultStartWatch, - fileTreeViewConfig := FileTreeViewConfig - .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value), + fileTreeViewConfig := FileManagement.defaultFileTreeView.value, fileTreeView := state.value .get(BasicKeys.globalFileTreeView) .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), @@ -657,8 +656,7 @@ object Defaults extends BuildCommon { }, watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), watch := watchSetting.value, - fileTreeViewConfig := FileTreeViewConfig - .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value) + fileTreeViewConfig := FileManagement.defaultFileTreeView.value ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = diff --git a/main/src/main/scala/sbt/internal/FileManagement.scala b/main/src/main/scala/sbt/internal/FileManagement.scala index bbdbb346c..127fd4a1d 100644 --- a/main/src/main/scala/sbt/internal/FileManagement.scala +++ b/main/src/main/scala/sbt/internal/FileManagement.scala @@ -13,10 +13,16 @@ import java.nio.file.Path import sbt.Keys._ import sbt.io.FileTreeDataView.Entry import sbt.io.syntax.File -import sbt.io.{ FileFilter, FileTreeRepository, FileTreeDataView } -import sbt.{ Def, ScopedTaskable, StampedFile, Task } +import sbt.io.{ FileFilter, FileTreeDataView, FileTreeRepository } +import sbt._ private[sbt] object FileManagement { + private[sbt] def defaultFileTreeView: Def.Initialize[Task[FileTreeViewConfig]] = Def.task { + if (state.value.remainingCommands.exists(_.commandLine == "shell")) { + FileTreeViewConfig + .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value) + } else FileTreeViewConfig.sbt1_2_compat(pollInterval.value, watchAntiEntropy.value) + } private[sbt] implicit class FileTreeDataViewOps[+T](val fileTreeDataView: FileTreeDataView[T]) { def register(path: Path, maxDepth: Int): Either[IOException, Boolean] = { fileTreeDataView match { From 25e97f99f588d1ddc9bbd114740bfb25c2a955c6 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 25 Aug 2018 16:37:22 -0700 Subject: [PATCH 15/17] Add custom external hooks For projects with a large number of files, zinc has to do a lot of work to determine which source files and binaries have changes since the last build. In a very simple project with 5000 source files, it takes roughly 750ms to do a no-op compile using the default incremental compiler options. After this change, it takes about 200ms. Of those 200ms, 50ms are due to the update task, which does a partial project resolution*. The implementation is straightforward since zinc already provides an api for overriding the built in change detection strategy. In a previous commit, I updated the sources task to return StampedFile rather than regular java.io.File instances. To compute all of the source file stamps, we simply list the sources and if the source is in fact an instance of StampedFile, we don't need to compute it, otherwise we generate a StampedFile on the fly. After building a map of stamped files for both the sources files and all of the binary dependencies, we simply diff these maps with the previous results in the changedSources, changedBinaries and removedProducts methods. The new ExternalHooks are easily disabled by setting `externalHooks := _ => None` in the project build. In the future, I could see moving ExternalHooks into the zinc project so that other tools like bloop or mill could use them. * I think this delay could be eliminated by caching the UpdateResult so long as the project doesn't depend on any snapshot libraries. For a project with a single source, the no-op compile takes O(50ms) so caching the project resolution would make compilation start nearly instantaneous. --- main/src/main/scala/sbt/Defaults.scala | 27 +++- main/src/main/scala/sbt/Keys.scala | 2 + .../scala/sbt/internal/ExternalHooks.scala | 125 ++++++++++++++++++ 3 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/ExternalHooks.scala diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 2c4bf38e3..3c54cadf2 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -276,6 +276,11 @@ object Defaults extends BuildCommon { fileTreeView := state.value .get(BasicKeys.globalFileTreeView) .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), + externalHooks := { + val view = fileTreeView.value + compileOptions => + Some(ExternalHooks(compileOptions, view)) + }, watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), watchLogger := streams.value.log, watchService :== { () => @@ -1646,12 +1651,22 @@ object Defaults extends BuildCommon { foldMappers(sourcePositionMappers.value) ) }, - compileInputs := Inputs.of( - compilers.value, - compileOptions.value, - compileIncSetup.value, - previousCompile.value - ) + compileInputs := { + val options = compileOptions.value + val setup = compileIncSetup.value + Inputs.of( + compilers.value, + options, + externalHooks + .value(options) + .map { hooks => + val newOptions = setup.incrementalCompilerOptions.withExternalHooks(hooks) + setup.withIncrementalCompilerOptions(newOptions) + } + .getOrElse(setup), + previousCompile.value + ) + } ) } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index ebf6a8cd1..9970f037b 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -20,6 +20,7 @@ import xsbti.compile.{ CompileOrder, Compilers, CompileResult, + ExternalHooks, GlobalsCache, IncOptions, Inputs, @@ -261,6 +262,7 @@ object Keys { val copyResources = taskKey[Seq[(File, File)]]("Copies resources to the output directory.").withRank(AMinusTask) val aggregate = settingKey[Boolean]("Configures task aggregation.").withRank(BMinusSetting) val sourcePositionMappers = taskKey[Seq[xsbti.Position => Option[xsbti.Position]]]("Maps positions in generated source files to the original source it was generated from").withRank(DTask) + val externalHooks = taskKey[CompileOptions => Option[ExternalHooks]]("External hooks for modifying the internal behavior of the incremental compiler.").withRank(BMinusSetting) // package keys val packageBin = taskKey[File]("Produces a main artifact, such as a binary jar.").withRank(ATask) diff --git a/main/src/main/scala/sbt/internal/ExternalHooks.scala b/main/src/main/scala/sbt/internal/ExternalHooks.scala new file mode 100644 index 000000000..08d7e0538 --- /dev/null +++ b/main/src/main/scala/sbt/internal/ExternalHooks.scala @@ -0,0 +1,125 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal +import java.nio.file.Paths +import java.util.Optional + +import sbt.StampedFile +import sbt.internal.inc.ExternalLookup +import sbt.io.syntax.File +import sbt.io.{ FileTreeRepository, FileTreeDataView, TypedPath } +import xsbti.compile._ +import xsbti.compile.analysis.Stamp + +import scala.collection.mutable + +private[sbt] object ExternalHooks { + private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_)) + def apply(options: CompileOptions, view: FileTreeDataView[StampedFile]): DefaultExternalHooks = { + import scala.collection.JavaConverters._ + val sources = options.sources() + val cachedSources = new java.util.HashMap[File, Stamp] + val converter: File => Stamp = f => StampedFile.sourceConverter(TypedPath(f.toPath)).stamp + sources.foreach { + case sf: StampedFile => cachedSources.put(sf, sf.stamp) + case f: File => cachedSources.put(f, converter(f)) + } + view match { + case r: FileTreeRepository[StampedFile] => + r.register(options.classesDirectory.toPath, Integer.MAX_VALUE) + options.classpath.foreach { f => + r.register(f.toPath, Integer.MAX_VALUE) + } + case _ => + } + val allBinaries = new java.util.HashMap[File, Stamp] + options.classpath.foreach { f => + view.listEntries(f.toPath, Integer.MAX_VALUE, _ => true) foreach { e => + e.value match { + case Right(value) => allBinaries.put(e.typedPath.getPath.toFile, value.stamp) + case _ => + } + } + // This gives us the entry for the path itself, which is necessary if the path is a jar file + // rather than a directory. + view.listEntries(f.toPath, -1, _ => true) foreach { e => + e.value match { + case Right(value) => allBinaries.put(e.typedPath.getPath.toFile, value.stamp) + case _ => + } + } + } + + val lookup = new ExternalLookup { + override def changedSources(previousAnalysis: CompileAnalysis): Option[Changes[File]] = Some { + new Changes[File] { + val getAdded: java.util.Set[File] = new java.util.HashSet[File] + val getRemoved: java.util.Set[File] = new java.util.HashSet[File] + val getChanged: java.util.Set[File] = new java.util.HashSet[File] + val getUnmodified: java.util.Set[File] = new java.util.HashSet[File] + override def isEmpty: java.lang.Boolean = + getAdded.isEmpty && getRemoved.isEmpty && getChanged.isEmpty + val prevSources: mutable.Map[File, Stamp] = + previousAnalysis.readStamps().getAllSourceStamps.asScala + prevSources.foreach { + case (file: File, s: Stamp) => + cachedSources.get(file) match { + case null => + getRemoved.add(file) + case stamp => + if ((stamp.getHash.orElse("") == s.getHash.orElse("")) && (stamp.getLastModified + .orElse(-1L) == s.getLastModified.orElse(-1L))) { + getUnmodified.add(file) + } else { + getChanged.add(file) + } + } + } + sources.foreach(file => if (!prevSources.contains(file)) getAdded.add(file)) + } + } + + override def shouldDoIncrementalCompilation( + set: Set[String], + compileAnalysis: CompileAnalysis + ): Boolean = true + + // This could use the cache as well, but it would complicate the cache implementation. + override def hashClasspath(files: Array[File]): Optional[Array[FileHash]] = + Optional.empty[Array[FileHash]] + + override def changedBinaries(previousAnalysis: CompileAnalysis): Option[Set[File]] = { + Some(previousAnalysis.readStamps.getAllBinaryStamps.asScala.flatMap { + case (file, stamp) => + allBinaries.get(file) match { + case null => + javaHome match { + case Some(h) if file.toPath.startsWith(h) => None + case _ => Some(file) + } + case cachedStamp if stamp == cachedStamp => None + case _ => Some(file) + } + }.toSet) + } + + override def removedProducts(previousAnalysis: CompileAnalysis): Option[Set[File]] = { + Some(previousAnalysis.readStamps.getAllProductStamps.asScala.flatMap { + case (file, s) => + allBinaries get file match { + case null => Some(file) + case stamp if stamp.getLastModified.orElse(0L) != s.getLastModified.orElse(0L) => + Some(file) + case _ => None + } + }.toSet) + } + } + new DefaultExternalHooks(Optional.of(lookup), Optional.empty[ClassFileManager]) + } +} From dc4f705500f37bcfa3ec817bacafcf8287df71ec Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 5 Oct 2018 10:27:50 -0700 Subject: [PATCH 16/17] Add support to rebuild a '~' task by pressing 'r' Sometimes a user may want to rerun their task even if the source files haven't changed. Presently this is a little annoying because you have to hit enter to stop the build and then up arrow or plus enter to rebuild. It's more convenient to just be able to press the 'r' key to re-run the task. To implement this, I had to make the watch task set up a jline terminal so that System.in would be character buffered instead of line buffered. Furthermore, I took advantage of the NonBlockingInputStream implementation provided by jline to wrap System.in. This was necessary because even with the jline terminal, System.in.available doesn't return > 0 until a newline character is entered. Instead, the NonBlockingInputStream does provide a peek api with a timeout that will return the next unread key off of System.in if there is one available. This can be use to proxy available in the WrappedNonBlockingInputStream. To ensure maximum user flexibility, I also update the watchHandleInput Key to take an InputStream and return an Action. This setting will now receive the wrapped System.in, which will allow the user to create their own keybindings for watch actions without needing to use jline themselves. Future work might make it more straightforward to go back to a line buffered input if that is what the user desires. --- main-command/src/main/scala/sbt/Watched.scala | 73 ++++++++++++------- .../src/test/scala/sbt/WatchedSpec.scala | 20 +++-- main/src/main/scala/sbt/Keys.scala | 4 +- 3 files changed, 59 insertions(+), 38 deletions(-) diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index 9f8dc25a6..0e9bc670e 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -7,7 +7,7 @@ package sbt -import java.io.File +import java.io.{ File, InputStream } import java.nio.file.FileSystems import sbt.BasicCommandStrings.{ @@ -20,9 +20,9 @@ import sbt.BasicCommands.otherCommandParser import sbt.internal.LegacyWatched import sbt.internal.inc.Stamper import sbt.internal.io.{ EventMonitor, Source, WatchState } -import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const import sbt.internal.util.complete.DefaultParsers +import sbt.internal.util.{ AttributeKey, JLine } import sbt.io.FileEventMonitor.Event import sbt.io._ import sbt.util.{ Level, Logger } @@ -108,25 +108,44 @@ object Watched { type WatchSource = Source def terminateWatch(key: Int): Boolean = Watched.isEnter(key) - /* - * Without jline, checking for enter is nearly pointless because System.in.available will not - * return a non-zero value until the user presses enter. - */ - @tailrec - final def shouldTerminate: Boolean = - (System.in.available > 0) && (terminateWatch(System.in.read()) || shouldTerminate) - final val handleInput: () => Action = () => if (shouldTerminate) CancelWatch else Ignore - val defaultStartWatch: Int => Option[String] = count => - Some(s"$count. Waiting for source changes... (press enter to interrupt)") + + private def withCharBufferedStdIn[R](f: InputStream => R): R = JLine.usingTerminal { terminal => + val in = terminal.wrapInIfNeeded(System.in) + try { + while (in.available > 0) in.read() + terminal.init() + f(in) + } finally { + while (in.available > 0) in.read() + terminal.reset() + } + } + + private[sbt] final val handleInput: InputStream => Action = in => { + @tailrec + def scanInput(): Action = { + if (in.available > 0) { + in.read() match { + case key if isEnter(key) => CancelWatch + case key if isR(key) => Trigger + case key if key >= 0 => scanInput() + case _ => Ignore + } + } else { + Ignore + } + } + scanInput() + } + private def waitMessage(project: String): String = + s"Waiting for source changes$project... (press enter to interrupt or 'r' to re-run the command)" + val defaultStartWatch: Int => Option[String] = count => Some(s"$count. ${waitMessage("")}") @deprecated("Use defaultStartWatch in conjunction with the watchStartMessage key", "1.3.0") val defaultWatchingMessage: WatchState => String = ws => defaultStartWatch(ws.count).get def projectWatchingMessage(projectId: String): WatchState => String = ws => projectOnWatchMessage(projectId)(ws.count).get def projectOnWatchMessage(project: String): Int => Option[String] = - count => - Some( - s"$count. Waiting for source changes in project $project... (press enter to interrupt)" - ) + count => Some(s"$count. ${waitMessage(s" in project $project")}") val defaultOnTriggerMessage: Int => Option[String] = _ => None @deprecated( @@ -182,6 +201,7 @@ object Watched { val PollDelay: FiniteDuration = 500.milliseconds val AntiEntropy: FiniteDuration = 40.milliseconds def isEnter(key: Int): Boolean = key == 10 || key == 13 + def isR(key: Int): Boolean = key == 82 || key == 114 def printIfDefined(msg: String): Unit = if (!msg.isEmpty) System.out.println(msg) private type RunCommand = () => State @@ -231,7 +251,7 @@ object Watched { state: State, command: String, setup: WatchSetup, - ): State = { + ): State = withCharBufferedStdIn { in => val (s0, config, newState) = setup(state, command) val failureCommandName = "SbtContinuousWatchOnFail" val onFail = Command.command(failureCommandName)(identity) @@ -263,7 +283,7 @@ object Watched { case (status, Right(t)) => if (status.getOrElse(true)) t() else status case _ => throw new IllegalStateException("Should be unreachable") } - val terminationAction = watch(task, config) + val terminationAction = watch(in, task, config) config.onWatchTerminated(terminationAction, command, state) } else { config.logger.error( @@ -274,6 +294,7 @@ object Watched { } private[sbt] def watch( + in: InputStream, task: () => Either[Exception, Boolean], config: WatchConfig ): Action = { @@ -284,7 +305,7 @@ object Watched { def impl(count: Int): Action = { @tailrec def nextAction(): Action = { - config.handleInput() match { + config.handleInput(in) match { case action @ (CancelWatch | HandleError | Reload | _: Custom) => action case Trigger => Trigger case _ => @@ -348,12 +369,8 @@ object Watched { HandleError } } - try { - impl(count = 1) - } finally { - config.fileEventMonitor.close() - while (System.in.available() > 0) System.in.read() - } + try impl(count = 1) + finally config.fileEventMonitor.close() } @deprecated("Replaced by Watched.command", "1.3.0") @@ -423,7 +440,7 @@ trait WatchConfig { * trigger. Usually this will read from System.in to react to user input. * @return an [[Watched.Action Action]] that will determine the next step in the watch. */ - def handleInput(): Watched.Action + def handleInput(inputStream: InputStream): Watched.Action /** * This is run before each watch iteration and if it returns true, the watch is terminated. @@ -496,7 +513,7 @@ object WatchConfig { def default( logger: Logger, fileEventMonitor: FileEventMonitor[StampedFile], - handleInput: () => Watched.Action, + handleInput: InputStream => Watched.Action, preWatch: (Int, Boolean) => Watched.Action, onWatchEvent: Event[StampedFile] => Watched.Action, onWatchTerminated: (Watched.Action, String, State) => State, @@ -514,7 +531,7 @@ object WatchConfig { new WatchConfig { override def logger: Logger = l override def fileEventMonitor: FileEventMonitor[StampedFile] = fem - override def handleInput(): Watched.Action = hi() + override def handleInput(inputStream: InputStream): Watched.Action = hi(inputStream) override def preWatch(count: Int, lastResult: Boolean): Watched.Action = pw(count, lastResult) override def onWatchEvent(event: Event[StampedFile]): Watched.Action = owe(event) diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala index 9d66b8f84..70d386cab 100644 --- a/main-command/src/test/scala/sbt/WatchedSpec.scala +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -7,7 +7,7 @@ package sbt -import java.io.File +import java.io.{ File, InputStream } import java.nio.file.Files import java.util.concurrent.atomic.AtomicBoolean @@ -28,7 +28,7 @@ class WatchedSpec extends FlatSpec with Matchers { sources: Seq[WatchSource], fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None, logger: Logger = NullLogger, - handleInput: () => Action = () => Ignore, + handleInput: InputStream => Action = _ => Ignore, preWatch: (Int, Boolean) => Action = (_, _) => CancelWatch, onWatchEvent: Event[StampedFile] => Action = _ => Ignore, triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None, @@ -49,9 +49,13 @@ class WatchedSpec extends FlatSpec with Matchers { ) } } + object NullInputStream extends InputStream { + override def available(): Int = 0 + override def read(): Int = -1 + } "Watched.watch" should "stop" in IO.withTemporaryDirectory { dir => val config = Defaults.config(sources = Seq(WatchSource(dir.toRealPath))) - Watched.watch(() => Right(true), config) shouldBe CancelWatch + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch } it should "trigger" in IO.withTemporaryDirectory { dir => val triggered = new AtomicBoolean(false) @@ -63,7 +67,7 @@ class WatchedSpec extends FlatSpec with Matchers { new File(dir, "file").createNewFile; None } ) - Watched.watch(() => Right(true), config) shouldBe CancelWatch + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch assert(triggered.get()) } it should "filter events" in IO.withTemporaryDirectory { dir => @@ -78,7 +82,7 @@ class WatchedSpec extends FlatSpec with Matchers { triggeredMessage = (tp, _) => { queue += tp; None }, watchingMessage = _ => { Files.createFile(bar); Thread.sleep(5); Files.createFile(foo); None } ) - Watched.watch(() => Right(true), config) shouldBe CancelWatch + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch queue.toIndexedSeq.map(_.getPath) shouldBe Seq(foo) } it should "enforce anti-entropy" in IO.withTemporaryDirectory { dir => @@ -102,7 +106,7 @@ class WatchedSpec extends FlatSpec with Matchers { None } ) - Watched.watch(() => Right(true), config) shouldBe CancelWatch + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch queue.toIndexedSeq.map(_.getPath) shouldBe Seq(bar, foo) } it should "halt on error" in IO.withTemporaryDirectory { dir => @@ -111,7 +115,7 @@ class WatchedSpec extends FlatSpec with Matchers { sources = Seq(WatchSource(dir.toRealPath)), preWatch = (_, lastStatus) => if (lastStatus) Ignore else { halted.set(true); HandleError } ) - Watched.watch(() => Right(false), config) shouldBe HandleError + Watched.watch(NullInputStream, () => Right(false), config) shouldBe HandleError assert(halted.get()) } it should "reload" in IO.withTemporaryDirectory { dir => @@ -121,7 +125,7 @@ class WatchedSpec extends FlatSpec with Matchers { onWatchEvent = _ => Reload, watchingMessage = _ => { new File(dir, "file").createNewFile(); None } ) - Watched.watch(() => Right(true), config) shouldBe Reload + Watched.watch(NullInputStream, () => Right(true), config) shouldBe Reload } } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 9970f037b..fcd817bc3 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -7,7 +7,7 @@ package sbt -import java.io.File +import java.io.{ File, InputStream } import java.net.URL import scala.concurrent.duration.{ FiniteDuration, Duration } import Def.ScopedKey @@ -152,7 +152,7 @@ object Keys { val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting) val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) - val watchHandleInput = settingKey[() => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) + val watchHandleInput = settingKey[InputStream => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) val watchOnEvent = taskKey[Event[StampedFile] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) val watchOnTermination = taskKey[(Watched.Action, String, State) => State]("Transforms the input state after the continuous build completes.").withRank(BMinusSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) From 0a9ae7b4b49da95bcac27db8dace663b54ff7686 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 9 Oct 2018 21:47:57 -0700 Subject: [PATCH 17/17] Use FileTreeRepository when interactive or continuous Ideally we use the FileTreeRepository for interactive sessions by default. A continuous build is effectively interactive, so I'd like that case to also use the file tree repository. To avoid breaking scripted tests, many of which implicitly expect file tree changes to be instantaneously available, we set interactive to true only if we are not in a scripted run, which can be verified by checking that the commands contains "setUpScripted". --- main/src/main/scala/sbt/internal/FileManagement.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/main/src/main/scala/sbt/internal/FileManagement.scala b/main/src/main/scala/sbt/internal/FileManagement.scala index 127fd4a1d..15a17461a 100644 --- a/main/src/main/scala/sbt/internal/FileManagement.scala +++ b/main/src/main/scala/sbt/internal/FileManagement.scala @@ -15,10 +15,17 @@ import sbt.io.FileTreeDataView.Entry import sbt.io.syntax.File import sbt.io.{ FileFilter, FileTreeDataView, FileTreeRepository } import sbt._ +import BasicCommandStrings.ContinuousExecutePrefix private[sbt] object FileManagement { private[sbt] def defaultFileTreeView: Def.Initialize[Task[FileTreeViewConfig]] = Def.task { - if (state.value.remainingCommands.exists(_.commandLine == "shell")) { + val remaining = state.value.remainingCommands.map(_.commandLine.trim) + // If the session is interactive or if the commands include a continuous build, then use + // the default configuration. Otherwise, use the sbt1_2_compat config, which does not cache + // anything, which makes it less likely to cause issues with CI. + val interactive = remaining.contains("shell") && !remaining.contains("setUpScripted") + val continuous = remaining.exists(_.startsWith(ContinuousExecutePrefix)) + if (interactive || continuous) { FileTreeViewConfig .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value) } else FileTreeViewConfig.sbt1_2_compat(pollInterval.value, watchAntiEntropy.value)