diff --git a/.travis.yml b/.travis.yml index d17c5e546..1db7df73b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ env: - SBT_CMD="scripted source-dependencies/*1of3" - SBT_CMD="scripted source-dependencies/*2of3" - SBT_CMD="scripted source-dependencies/*3of3" - - SBT_CMD="scripted tests/*" + - SBT_CMD="scripted tests/* watch/*" - SBT_CMD="repoOverrideTest:scripted dependency-management/*" notifications: diff --git a/build.sbt b/build.sbt index fcc53da8a..41de4a9ee 100644 --- a/build.sbt +++ b/build.sbt @@ -507,7 +507,8 @@ lazy val commandProj = (project in file("main-command")) addSbtUtilLogging, addSbtCompilerInterface, addSbtCompilerClasspath, - addSbtLmCore + addSbtLmCore, + addSbtZinc ) // The core macro project defines the main logic of the DSL, abstracted diff --git a/main-command/src/main/scala/sbt/BasicCommandStrings.scala b/main-command/src/main/scala/sbt/BasicCommandStrings.scala index 6eeb06ee5..7ec8c794c 100644 --- a/main-command/src/main/scala/sbt/BasicCommandStrings.scala +++ b/main-command/src/main/scala/sbt/BasicCommandStrings.scala @@ -231,4 +231,8 @@ $AliasCommand name= val ContinuousExecutePrefix = "~" def continuousDetail = "Executes the specified command whenever source files change." def continuousBriefHelp = (ContinuousExecutePrefix + " ", continuousDetail) + def FlushFileTreeRepository = "flushFileTreeRepository" + def FlushDetailed: String = + "Resets the global file repository in the event that the repository has become inconsistent " + + "with the file system." } diff --git a/main-command/src/main/scala/sbt/BasicCommands.scala b/main-command/src/main/scala/sbt/BasicCommands.scala index 2351b185b..0e8c51a41 100644 --- a/main-command/src/main/scala/sbt/BasicCommands.scala +++ b/main-command/src/main/scala/sbt/BasicCommands.scala @@ -50,7 +50,6 @@ object BasicCommands { call, early, exit, - continuous, history, oldshell, client, @@ -254,6 +253,7 @@ object BasicCommands { def exit: Command = Command.command(TerminateAction, exitBrief, exitBrief)(_ exit true) + @deprecated("Replaced by BuiltInCommands.continuous", "1.3.0") def continuous: Command = Command(ContinuousExecutePrefix, continuousBriefHelp, continuousDetail)(otherCommandParser) { (s, arg) => diff --git a/main-command/src/main/scala/sbt/BasicKeys.scala b/main-command/src/main/scala/sbt/BasicKeys.scala index ff466e9e8..66de24ce6 100644 --- a/main-command/src/main/scala/sbt/BasicKeys.scala +++ b/main-command/src/main/scala/sbt/BasicKeys.scala @@ -11,6 +11,7 @@ import java.io.File import sbt.internal.util.AttributeKey import sbt.internal.inc.classpath.ClassLoaderCache import sbt.internal.server.ServerHandler +import sbt.io.FileTreeDataView import sbt.librarymanagement.ModuleID import sbt.util.Level @@ -100,6 +101,11 @@ object BasicKeys { "List of template resolver infos.", 1000 ) + private[sbt] val globalFileTreeView = AttributeKey[FileTreeDataView[StampedFile]]( + "globalFileTreeView", + "provides a view into the file system that may or may not cache the tree in memory", + 1000 + ) } case class TemplateResolverInfo(module: ModuleID, implementationClass: String) diff --git a/main-command/src/main/scala/sbt/FileTreeViewConfig.scala b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala new file mode 100644 index 000000000..c1dd8fa6d --- /dev/null +++ b/main-command/src/main/scala/sbt/FileTreeViewConfig.scala @@ -0,0 +1,172 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +import sbt.Watched.WatchSource +import sbt.internal.io.{ HybridPollingFileTreeRepository, WatchServiceBackedObservable, WatchState } +import sbt.io._ +import FileTreeDataView.{ Observable, Observer } +import sbt.util.Logger + +import scala.concurrent.duration._ + +/** + * Configuration for viewing and monitoring the file system. + */ +final class FileTreeViewConfig private ( + val newDataView: () => FileTreeDataView[StampedFile], + val newMonitor: ( + FileTreeDataView[StampedFile], + Seq[WatchSource], + Logger + ) => FileEventMonitor[StampedFile] +) +object FileTreeViewConfig { + private implicit class RepositoryOps(val repository: FileTreeRepository[StampedFile]) { + def register(sources: Seq[WatchSource]): Unit = sources foreach { s => + repository.register(s.base.toPath, if (s.recursive) Integer.MAX_VALUE else 0) + } + } + + /** + * Create a new FileTreeViewConfig. This factory takes a generic parameter, T, that is bounded + * by {{{sbt.io.FileTreeDataView[StampedFile]}}}. The reason for this is to ensure that a + * sbt.io.FileTreeDataView that is instantiated by [[FileTreeViewConfig.newDataView]] can be + * passed into [[FileTreeViewConfig.newMonitor]] without constraining the type of view to be + * {{{sbt.io.FileTreeDataView[StampedFile]}}}. + * @param newDataView create a new sbt.io.FileTreeDataView. This value may be cached in a global + * attribute + * @param newMonitor create a new sbt.io.FileEventMonitor using the sbt.io.FileTreeDataView + * created by newDataView + * @tparam T the subtype of sbt.io.FileTreeDataView that is returned by [[FileTreeViewConfig.newDataView]] + * @return a [[FileTreeViewConfig]] instance. + */ + def apply[T <: FileTreeDataView[StampedFile]]( + newDataView: () => T, + newMonitor: (T, Seq[WatchSource], Logger) => FileEventMonitor[StampedFile] + ): FileTreeViewConfig = + new FileTreeViewConfig( + newDataView, + (view: FileTreeDataView[StampedFile], sources: Seq[WatchSource], logger: Logger) => + newMonitor(view.asInstanceOf[T], sources, logger) + ) + + /** + * Provides a [[FileTreeViewConfig]] with semantics as close as possible to sbt 1.2.0. This means + * that there is no file tree caching and the sbt.io.FileEventMonitor will use an + * sbt.io.WatchService for monitoring the file system. + * @param delay the maximum delay for which the background thread will poll the + * sbt.io.WatchService for file system events + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @return a [[FileTreeViewConfig]] instance. + */ + def sbt1_2_compat( + delay: FiniteDuration, + antiEntropy: FiniteDuration + ): FileTreeViewConfig = + FileTreeViewConfig( + () => FileTreeView.DEFAULT.asDataView(StampedFile.converter), + (_: FileTreeDataView[StampedFile], sources, logger) => { + val ioLogger: sbt.io.WatchLogger = msg => logger.debug(msg.toString) + FileEventMonitor.antiEntropy( + new WatchServiceBackedObservable( + WatchState.empty(Watched.createWatchService(), sources), + delay, + StampedFile.converter, + closeService = true, + ioLogger + ), + antiEntropy, + ioLogger + ) + } + ) + + /** + * Provides a default [[FileTreeViewConfig]]. This view caches entries and solely relies on + * file system events from the operating system to update its internal representation of the + * file tree. + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @return a [[FileTreeViewConfig]] instance. + */ + def default(antiEntropy: FiniteDuration): FileTreeViewConfig = + FileTreeViewConfig( + () => FileTreeRepository.default(StampedFile.converter), + (repository: FileTreeRepository[StampedFile], sources: Seq[WatchSource], logger: Logger) => { + repository.register(sources) + val copied = new Observable[StampedFile] { + override def addObserver(observer: Observer[StampedFile]): Int = + repository.addObserver(observer) + override def removeObserver(handle: Int): Unit = repository.removeObserver(handle) + override def close(): Unit = {} // Don't close the underlying observable + } + FileEventMonitor.antiEntropy(copied, antiEntropy, msg => logger.debug(msg.toString)) + } + ) + + /** + * Provides a default [[FileTreeViewConfig]]. When the pollingSources argument is empty, it + * returns the same config as [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]]. + * Otherwise, it returns the same config as [[polling]]. + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @param pollingInterval the frequency with which the sbt.io.FileEventMonitor polls the file + * system for the paths included in pollingSources + * @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that + * will be periodically polled for changes during continuous builds. + * @return + */ + def default( + antiEntropy: FiniteDuration, + pollingInterval: FiniteDuration, + pollingSources: Seq[WatchSource] + ): FileTreeViewConfig = { + if (pollingSources.isEmpty) default(antiEntropy) + else polling(antiEntropy, pollingInterval, pollingSources) + } + + /** + * Provides a polling [[FileTreeViewConfig]]. Unlike the view returned by newDataView in + * [[sbt.FileTreeViewConfig.default(antiEntropy:scala\.concurrent\.duration\.FiniteDuration)*]], + * the view returned by newDataView will not cache any portion of the file system tree that is is + * covered by the pollingSources parameter. The monitor that is generated by newMonitor, will + * poll these directories for changes rather than relying on file system events from the + * operating system. Any paths that are registered with the view that are not included in the + * pollingSources will be cached and monitored using file system events from the operating system + * in the same way that they are in the default view. + * + * @param antiEntropy the duration of the period after a path triggers a build for which it is + * quarantined from triggering another build + * @param pollingInterval the frequency with which the FileEventMonitor polls the file system + * for the paths included in pollingSources + * @param pollingSources the sources that will not be cached in the sbt.io.FileTreeRepository and that + * will be periodically polled for changes during continuous builds. + * @return a [[FileTreeViewConfig]] instance. + */ + def polling( + antiEntropy: FiniteDuration, + pollingInterval: FiniteDuration, + pollingSources: Seq[WatchSource], + ): FileTreeViewConfig = FileTreeViewConfig( + () => FileTreeRepository.hybrid(StampedFile.converter, pollingSources: _*), + ( + repository: HybridPollingFileTreeRepository[StampedFile], + sources: Seq[WatchSource], + logger: Logger + ) => { + repository.register(sources) + FileEventMonitor + .antiEntropy( + repository.toPollingObservable(pollingInterval, sources, NullWatchLogger), + antiEntropy, + msg => logger.debug(msg.toString) + ) + } + ) +} diff --git a/main-command/src/main/scala/sbt/Watched.scala b/main-command/src/main/scala/sbt/Watched.scala index 1331fd97d..0e9bc670e 100644 --- a/main-command/src/main/scala/sbt/Watched.scala +++ b/main-command/src/main/scala/sbt/Watched.scala @@ -7,20 +7,32 @@ package sbt -import java.io.File +import java.io.{ File, InputStream } import java.nio.file.FileSystems -import sbt.BasicCommandStrings.ClearOnFailure -import sbt.State.FailureWall +import sbt.BasicCommandStrings.{ + ContinuousExecutePrefix, + FailureWall, + continuousBriefHelp, + continuousDetail +} +import sbt.BasicCommands.otherCommandParser +import sbt.internal.LegacyWatched +import sbt.internal.inc.Stamper import sbt.internal.io.{ EventMonitor, Source, WatchState } -import sbt.internal.util.AttributeKey import sbt.internal.util.Types.const +import sbt.internal.util.complete.DefaultParsers +import sbt.internal.util.{ AttributeKey, JLine } +import sbt.io.FileEventMonitor.Event import sbt.io._ +import sbt.util.{ Level, Logger } +import xsbti.compile.analysis.Stamp import scala.annotation.tailrec import scala.concurrent.duration._ import scala.util.Properties +@deprecated("Watched is no longer used to implement continuous execution", "1.3.0") trait Watched { /** The files watched when an action is run with a proceeding ~ */ @@ -50,18 +62,102 @@ trait Watched { } object Watched { - val defaultWatchingMessage: WatchState => String = ws => - s"${ws.count}. Waiting for source changes... (press enter to interrupt)" + /** + * This trait is used to communicate what the watch should do next at various points in time. It + * is heavily linked to a number of callbacks in [[WatchConfig]]. For example, when the + * sbt.io.FileEventMonitor created by [[FileTreeViewConfig.newMonitor]] detects a changed source + * file, then we expect [[WatchConfig.onWatchEvent]] to return [[Trigger]]. + */ + sealed trait Action + + /** + * Action that indicates that the watch should stop. + */ + case object CancelWatch extends Action + + /** + * Action that indicates that an error has occurred. The watch will be terminated when this action + * is produced. + */ + case object HandleError extends Action + + /** + * Action that indicates that the watch should continue as though nothing happened. This may be + * because, for example, no user input was yet available in [[WatchConfig.handleInput]]. + */ + case object Ignore extends Action + + /** + * Action that indicates that the watch should pause while the build is reloaded. This is used to + * automatically reload the project when the build files (e.g. build.sbt) are changed. + */ + case object Reload extends Action + + /** + * Action that indicates that the watch process should re-run the command. + */ + case object Trigger extends Action + + /** + * A user defined Action. It is not sealed so that the user can create custom instances. If any + * of the [[WatchConfig]] callbacks, e.g. [[WatchConfig.onWatchEvent]], return an instance of + * [[Custom]], the watch will terminate. + */ + trait Custom extends Action + + type WatchSource = Source + def terminateWatch(key: Int): Boolean = Watched.isEnter(key) + + private def withCharBufferedStdIn[R](f: InputStream => R): R = JLine.usingTerminal { terminal => + val in = terminal.wrapInIfNeeded(System.in) + try { + while (in.available > 0) in.read() + terminal.init() + f(in) + } finally { + while (in.available > 0) in.read() + terminal.reset() + } + } + + private[sbt] final val handleInput: InputStream => Action = in => { + @tailrec + def scanInput(): Action = { + if (in.available > 0) { + in.read() match { + case key if isEnter(key) => CancelWatch + case key if isR(key) => Trigger + case key if key >= 0 => scanInput() + case _ => Ignore + } + } else { + Ignore + } + } + scanInput() + } + private def waitMessage(project: String): String = + s"Waiting for source changes$project... (press enter to interrupt or 'r' to re-run the command)" + val defaultStartWatch: Int => Option[String] = count => Some(s"$count. ${waitMessage("")}") + @deprecated("Use defaultStartWatch in conjunction with the watchStartMessage key", "1.3.0") + val defaultWatchingMessage: WatchState => String = ws => defaultStartWatch(ws.count).get def projectWatchingMessage(projectId: String): WatchState => String = - ws => - s"${ws.count}. Waiting for source changes in project $projectId... (press enter to interrupt)" + ws => projectOnWatchMessage(projectId)(ws.count).get + def projectOnWatchMessage(project: String): Int => Option[String] = + count => Some(s"$count. ${waitMessage(s" in project $project")}") + val defaultOnTriggerMessage: Int => Option[String] = _ => None + @deprecated( + "Use defaultOnTriggerMessage in conjunction with the watchTriggeredMessage key", + "1.3.0" + ) val defaultTriggeredMessage: WatchState => String = const("") + val clearOnTrigger: Int => Option[String] = _ => Some(clearScreen) + @deprecated("Use clearOnTrigger in conjunction with the watchTriggeredMessage key", "1.3.0") val clearWhenTriggered: WatchState => String = const(clearScreen) def clearScreen: String = "\u001b[2J\u001b[0;0H" - type WatchSource = Source object WatchSource { /** @@ -85,67 +181,209 @@ object Watched { } + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") private[this] class AWatched extends Watched + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") def multi(base: Watched, paths: Seq[Watched]): Watched = new AWatched { - override def watchSources(s: State) = (base.watchSources(s) /: paths)(_ ++ _.watchSources(s)) + override def watchSources(s: State): Seq[Watched.WatchSource] = + (base.watchSources(s) /: paths)(_ ++ _.watchSources(s)) override def terminateWatch(key: Int): Boolean = base.terminateWatch(key) - override val pollInterval = (base +: paths).map(_.pollInterval).min - override val antiEntropy = (base +: paths).map(_.antiEntropy).min - override def watchingMessage(s: WatchState) = base.watchingMessage(s) - override def triggeredMessage(s: WatchState) = base.triggeredMessage(s) + override val pollInterval: FiniteDuration = (base +: paths).map(_.pollInterval).min + override val antiEntropy: FiniteDuration = (base +: paths).map(_.antiEntropy).min + override def watchingMessage(s: WatchState): String = base.watchingMessage(s) + override def triggeredMessage(s: WatchState): String = base.triggeredMessage(s) } + @deprecated("This method is not used and may be removed in a future version of sbt", "1.3.0") def empty: Watched = new AWatched val PollDelay: FiniteDuration = 500.milliseconds val AntiEntropy: FiniteDuration = 40.milliseconds def isEnter(key: Int): Boolean = key == 10 || key == 13 - def printIfDefined(msg: String) = if (!msg.isEmpty) System.out.println(msg) + def isR(key: Int): Boolean = key == 82 || key == 114 + def printIfDefined(msg: String): Unit = if (!msg.isEmpty) System.out.println(msg) - def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = { - @tailrec def shouldTerminate: Boolean = - (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) - val log = s.log - val logger = new EventMonitor.Logger { - override def debug(msg: => Any): Unit = log.debug(msg.toString) + private type RunCommand = () => State + private type WatchSetup = (State, String) => (State, WatchConfig, RunCommand => State) + + /** + * Provides the '~' continuous execution command. + * @param setup a function that provides a logger and a function from (() => State) => State. + * @return the '~' command. + */ + def continuous(setup: WatchSetup): Command = + Command(ContinuousExecutePrefix, continuousBriefHelp, continuousDetail)(otherCommandParser) { + (state, command) => + Watched.executeContinuously(state, command, setup) } - s get ContinuousEventMonitor match { - case None => - // This is the first iteration, so run the task and create a new EventMonitor - (ClearOnFailure :: next :: FailureWall :: repeat :: s) - .put( - ContinuousEventMonitor, - EventMonitor( - WatchState.empty(watched.watchService(), watched.watchSources(s)), - watched.pollInterval, - watched.antiEntropy, - shouldTerminate, - logger - ) - ) - case Some(eventMonitor) => - printIfDefined(watched watchingMessage eventMonitor.state) - val triggered = try eventMonitor.awaitEvent() - catch { - case e: Exception => - log.error( - "Error occurred obtaining files to watch. Terminating continuous execution..." - ) - s.handleError(e) - false - } - if (triggered) { - printIfDefined(watched triggeredMessage eventMonitor.state) - ClearOnFailure :: next :: FailureWall :: repeat :: s - } else { - while (System.in.available() > 0) System.in.read() - eventMonitor.close() - s.remove(ContinuousEventMonitor) - } + + /** + * Default handler to transform the state when the watch terminates. When the [[Watched.Action]] is + * [[Reload]], the handler will prepend the original command (prefixed by ~) to the + * [[State.remainingCommands]] and then invoke the [[StateOps.reload]] method. When the + * [[Watched.Action]] is [[HandleError]], the handler returns the result of [[StateOps.fail]]. Otherwise + * the original state is returned. + */ + private[sbt] val onTermination: (Action, String, State) => State = (action, command, state) => + action match { + case Reload => + val continuousCommand = Exec(ContinuousExecutePrefix + command, None) + state.copy(remainingCommands = continuousCommand +: state.remainingCommands).reload + case HandleError => state.fail + case _ => state + } + + /** + * Implements continuous execution. It works by first parsing the command and generating a task to + * run with each build. It can run multiple commands that are separated by ";" in the command + * input. If any of these commands are invalid, the watch will immediately exit. + * @param state the initial state + * @param command the command(s) to repeatedly apply + * @param setup function to generate a logger and a transformation of the resultant state. The + * purpose of the transformation is to preserve the logging semantics that existed + * in the legacy version of this function in which the task would be run through + * MainLoop.processCommand, which is unavailable in the main-command project + * @return the initial state if all of the input commands are valid. Otherwise, returns the + * initial state with the failure transformation. + */ + private[sbt] def executeContinuously( + state: State, + command: String, + setup: WatchSetup, + ): State = withCharBufferedStdIn { in => + val (s0, config, newState) = setup(state, command) + val failureCommandName = "SbtContinuousWatchOnFail" + val onFail = Command.command(failureCommandName)(identity) + val s = (FailureWall :: s0).copy( + onFailure = Some(Exec(failureCommandName, None)), + definedCommands = s0.definedCommands :+ onFail + ) + val commands = command.split(";") match { + case Array("", rest @ _*) => rest + case Array(cmd) => Seq(cmd) + } + val parser = Command.combine(s.definedCommands)(s) + val tasks = commands.foldLeft(Nil: Seq[Either[String, () => Either[Exception, Boolean]]]) { + (t, cmd) => + t :+ (DefaultParsers.parse(cmd, parser) match { + case Right(task) => + Right { () => + try { + Right(newState(task).remainingCommands.forall(_.commandLine != failureCommandName)) + } catch { case e: Exception => Left(e) } + } + case Left(_) => Left(cmd) + }) + } + val (valid, invalid) = tasks.partition(_.isRight) + if (invalid.isEmpty) { + val task = () => + valid.foldLeft(Right(true): Either[Exception, Boolean]) { + case (status, Right(t)) => if (status.getOrElse(true)) t() else status + case _ => throw new IllegalStateException("Should be unreachable") + } + val terminationAction = watch(in, task, config) + config.onWatchTerminated(terminationAction, command, state) + } else { + config.logger.error( + s"Terminating watch due to invalid command(s): ${invalid.mkString("'", "', '", "'")}" + ) + state.fail } } + private[sbt] def watch( + in: InputStream, + task: () => Either[Exception, Boolean], + config: WatchConfig + ): Action = { + val logger = config.logger + def info(msg: String): Unit = if (msg.nonEmpty) logger.info(msg) + + @tailrec + def impl(count: Int): Action = { + @tailrec + def nextAction(): Action = { + config.handleInput(in) match { + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action + case Trigger => Trigger + case _ => + val events = config.fileEventMonitor.poll(10.millis) + val next = events match { + case Seq() => (Ignore, None) + case Seq(head, tail @ _*) => + /* + * We traverse all of the events and find the one for which we give the highest + * weight. + * Custom > HandleError > CancelWatch > Reload > Trigger > Ignore + */ + tail.foldLeft((config.onWatchEvent(head), Some(head))) { + case (current @ (_: Custom, _), _) => current + case (current @ (action, _), event) => + config.onWatchEvent(event) match { + case HandleError => (HandleError, Some(event)) + case CancelWatch if action != HandleError => (CancelWatch, Some(event)) + case Reload if action != HandleError && action != CancelWatch => + (Reload, Some(event)) + case Trigger if action == Ignore => (Trigger, Some(event)) + case _ => current + } + } + } + // Note that nextAction should never return Ignore. + next match { + case (action @ (HandleError | CancelWatch | _: Custom), Some(event)) => + val cause = + if (action == HandleError) "error" + else if (action.isInstanceOf[Custom]) action.toString + else "cancellation" + logger.debug(s"Stopping watch due to $cause from ${event.entry.typedPath.getPath}") + action + case (Trigger, Some(event)) => + logger.debug(s"Triggered by ${event.entry.typedPath.getPath}") + config.triggeredMessage(event.entry.typedPath, count).foreach(info) + Trigger + case (Reload, Some(event)) => + logger.info(s"Reload triggered by ${event.entry.typedPath.getPath}") + Reload + case _ => + nextAction() + } + } + } + task() match { + case Right(status) => + config.preWatch(count, status) match { + case Ignore => + config.watchingMessage(count).foreach(info) + nextAction() match { + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action + case _ => impl(count + 1) + } + case Trigger => impl(count + 1) + case action @ (CancelWatch | HandleError | Reload | _: Custom) => action + } + case Left(e) => + logger.error(s"Terminating watch due to Unexpected error: $e") + HandleError + } + } + try impl(count = 1) + finally config.fileEventMonitor.close() + } + + @deprecated("Replaced by Watched.command", "1.3.0") + def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = + LegacyWatched.executeContinuously(watched, s, next, repeat) + + private[sbt] object NullLogger extends Logger { + override def trace(t: => Throwable): Unit = {} + override def success(message: => String): Unit = {} + override def log(level: Level.Value, message: => String): Unit = {} + } + + @deprecated("ContinuousEventMonitor attribute is not used by Watched.command", "1.3.0") val ContinuousEventMonitor = AttributeKey[EventMonitor]( "watch event monitor", @@ -178,3 +416,155 @@ object Watched { } } } + +/** + * Provides a number of configuration options for continuous execution. + */ +trait WatchConfig { + + /** + * A logger. + * @return a logger + */ + def logger: Logger + + /** + * The sbt.io.FileEventMonitor that is used to monitor the file system. + * + * @return an sbt.io.FileEventMonitor instance. + */ + def fileEventMonitor: FileEventMonitor[StampedFile] + + /** + * A function that is periodically invoked to determine whether the watch should stop or + * trigger. Usually this will read from System.in to react to user input. + * @return an [[Watched.Action Action]] that will determine the next step in the watch. + */ + def handleInput(inputStream: InputStream): Watched.Action + + /** + * This is run before each watch iteration and if it returns true, the watch is terminated. + * @param count The current number of watch iterations. + * @param lastStatus true if the previous task execution completed successfully + * @return the Action to apply + */ + def preWatch(count: Int, lastStatus: Boolean): Watched.Action + + /** + * Callback that is invoked whenever a file system vent is detected. The next step of the watch + * is determined by the [[Watched.Action Action]] returned by the callback. + * @param event the detected sbt.io.FileEventMonitor.Event. + * @return the next [[Watched.Action Action]] to run. + */ + def onWatchEvent(event: Event[StampedFile]): Watched.Action + + /** + * Transforms the state after the watch terminates. + * @param action the [[Watched.Action Action]] that caused the build to terminate + * @param command the command that the watch was repeating + * @param state the initial state prior to the start of continuous execution + * @return the updated state. + */ + def onWatchTerminated(action: Watched.Action, command: String, state: State): State + + /** + * The optional message to log when a build is triggered. + * @param typedPath the path that triggered the build + * @param count the current iteration + * @return an optional log message. + */ + def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] + + /** + * The optional message to log before each watch iteration. + * @param count the current iteration + * @return an optional log message. + */ + def watchingMessage(count: Int): Option[String] +} + +/** + * Provides a default implementation of [[WatchConfig]]. + */ +object WatchConfig { + + /** + * Create an instance of [[WatchConfig]]. + * @param logger logger for watch events + * @param fileEventMonitor the monitor for file system events. + * @param handleInput callback that is periodically invoked to check whether to continue or + * terminate the watch based on user input. It is also possible to, for + * example time out the watch using this callback. + * @param preWatch callback to invoke before waiting for updates from the sbt.io.FileEventMonitor. + * The input parameters are the current iteration count and whether or not + * the last invocation of the command was successful. Typical uses would be to + * terminate the watch after a fixed number of iterations or to terminate the + * watch if the command was unsuccessful. + * @param onWatchEvent callback that is invoked when + * @param onWatchTerminated callback that is invoked to update the state after the watch + * terminates. + * @param triggeredMessage optional message that will be logged when a new build is triggered. + * The input parameters are the sbt.io.TypedPath that triggered the new + * build and the current iteration count. + * @param watchingMessage optional message that is printed before each watch iteration begins. + * The input parameter is the current iteration count. + * @return a [[WatchConfig]] instance. + */ + def default( + logger: Logger, + fileEventMonitor: FileEventMonitor[StampedFile], + handleInput: InputStream => Watched.Action, + preWatch: (Int, Boolean) => Watched.Action, + onWatchEvent: Event[StampedFile] => Watched.Action, + onWatchTerminated: (Watched.Action, String, State) => State, + triggeredMessage: (TypedPath, Int) => Option[String], + watchingMessage: Int => Option[String] + ): WatchConfig = { + val l = logger + val fem = fileEventMonitor + val hi = handleInput + val pw = preWatch + val owe = onWatchEvent + val owt = onWatchTerminated + val tm = triggeredMessage + val wm = watchingMessage + new WatchConfig { + override def logger: Logger = l + override def fileEventMonitor: FileEventMonitor[StampedFile] = fem + override def handleInput(inputStream: InputStream): Watched.Action = hi(inputStream) + override def preWatch(count: Int, lastResult: Boolean): Watched.Action = + pw(count, lastResult) + override def onWatchEvent(event: Event[StampedFile]): Watched.Action = owe(event) + override def onWatchTerminated(action: Watched.Action, command: String, state: State): State = + owt(action, command, state) + override def triggeredMessage(typedPath: TypedPath, count: Int): Option[String] = + tm(typedPath, count) + override def watchingMessage(count: Int): Option[String] = wm(count) + } + } +} + +trait StampedFile extends File { + def stamp: Stamp +} +object StampedFile { + val sourceConverter: TypedPath => StampedFile = + new StampedFileImpl(_: TypedPath, forceLastModified = false) + val binaryConverter: TypedPath => StampedFile = + new StampedFileImpl(_: TypedPath, forceLastModified = true) + val converter: TypedPath => StampedFile = (tp: TypedPath) => + tp.getPath.toString match { + case s if s.endsWith(".jar") => binaryConverter(tp) + case s if s.endsWith(".class") => binaryConverter(tp) + case _ => sourceConverter(tp) + } + + private class StampedFileImpl(typedPath: TypedPath, forceLastModified: Boolean) + extends java.io.File(typedPath.getPath.toString) + with StampedFile { + override val stamp: Stamp = + if (forceLastModified || typedPath.isDirectory) + Stamper.forLastModified(typedPath.getPath.toFile) + else Stamper.forHash(typedPath.getPath.toFile) + } +} diff --git a/main-command/src/main/scala/sbt/internal/LegacyWatched.scala b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala new file mode 100644 index 000000000..624372efc --- /dev/null +++ b/main-command/src/main/scala/sbt/internal/LegacyWatched.scala @@ -0,0 +1,64 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import sbt.BasicCommandStrings.{ ClearOnFailure, FailureWall } +import sbt.internal.io.{ EventMonitor, WatchState } +import sbt.{ State, Watched } + +import scala.annotation.tailrec +import Watched.ContinuousEventMonitor + +import scala.util.control.NonFatal + +private[sbt] object LegacyWatched { + @deprecated("Replaced by Watched.command", "1.3.0") + def executeContinuously(watched: Watched, s: State, next: String, repeat: String): State = { + @tailrec def shouldTerminate: Boolean = + (System.in.available > 0) && (watched.terminateWatch(System.in.read()) || shouldTerminate) + val log = s.log + val logger = new EventMonitor.Logger { + override def debug(msg: => Any): Unit = log.debug(msg.toString) + } + s get ContinuousEventMonitor match { + case None => + // This is the first iteration, so run the task and create a new EventMonitor + (ClearOnFailure :: next :: FailureWall :: repeat :: s) + .put( + ContinuousEventMonitor, + EventMonitor( + WatchState.empty(watched.watchService(), watched.watchSources(s)), + watched.pollInterval, + watched.antiEntropy, + shouldTerminate, + logger + ) + ) + case Some(eventMonitor) => + Watched.printIfDefined(watched watchingMessage eventMonitor.state) + val triggered = try eventMonitor.awaitEvent() + catch { + case NonFatal(e) => + log.error( + "Error occurred obtaining files to watch. Terminating continuous execution..." + ) + s.handleError(e) + false + } + if (triggered) { + Watched.printIfDefined(watched triggeredMessage eventMonitor.state) + ClearOnFailure :: next :: FailureWall :: repeat :: s + } else { + while (System.in.available() > 0) System.in.read() + eventMonitor.close() + s.remove(ContinuousEventMonitor) + } + } + } + +} diff --git a/main-command/src/test/scala/sbt/WatchedSpec.scala b/main-command/src/test/scala/sbt/WatchedSpec.scala new file mode 100644 index 000000000..70d386cab --- /dev/null +++ b/main-command/src/test/scala/sbt/WatchedSpec.scala @@ -0,0 +1,136 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt + +import java.io.{ File, InputStream } +import java.nio.file.Files +import java.util.concurrent.atomic.AtomicBoolean + +import org.scalatest.{ FlatSpec, Matchers } +import sbt.Watched._ +import sbt.WatchedSpec._ +import sbt.io.FileEventMonitor.Event +import sbt.io.{ FileEventMonitor, IO, TypedPath } +import sbt.util.Logger + +import scala.collection.mutable +import scala.concurrent.duration._ + +class WatchedSpec extends FlatSpec with Matchers { + object Defaults { + private val fileTreeViewConfig = FileTreeViewConfig.default(50.millis) + def config( + sources: Seq[WatchSource], + fileEventMonitor: Option[FileEventMonitor[StampedFile]] = None, + logger: Logger = NullLogger, + handleInput: InputStream => Action = _ => Ignore, + preWatch: (Int, Boolean) => Action = (_, _) => CancelWatch, + onWatchEvent: Event[StampedFile] => Action = _ => Ignore, + triggeredMessage: (TypedPath, Int) => Option[String] = (_, _) => None, + watchingMessage: Int => Option[String] = _ => None + ): WatchConfig = { + val monitor = fileEventMonitor.getOrElse( + fileTreeViewConfig.newMonitor(fileTreeViewConfig.newDataView(), sources, logger) + ) + WatchConfig.default( + logger = logger, + monitor, + handleInput, + preWatch, + onWatchEvent, + (_, _, state) => state, + triggeredMessage, + watchingMessage + ) + } + } + object NullInputStream extends InputStream { + override def available(): Int = 0 + override def read(): Int = -1 + } + "Watched.watch" should "stop" in IO.withTemporaryDirectory { dir => + val config = Defaults.config(sources = Seq(WatchSource(dir.toRealPath))) + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch + } + it should "trigger" in IO.withTemporaryDirectory { dir => + val triggered = new AtomicBoolean(false) + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + preWatch = (count, _) => if (count == 2) CancelWatch else Ignore, + onWatchEvent = _ => { triggered.set(true); Trigger }, + watchingMessage = _ => { + new File(dir, "file").createNewFile; None + } + ) + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch + assert(triggered.get()) + } + it should "filter events" in IO.withTemporaryDirectory { dir => + val realDir = dir.toRealPath + val queue = new mutable.Queue[TypedPath] + val foo = realDir.toPath.resolve("foo") + val bar = realDir.toPath.resolve("bar") + val config = Defaults.config( + sources = Seq(WatchSource(realDir)), + preWatch = (count, _) => if (count == 2) CancelWatch else Ignore, + onWatchEvent = e => if (e.entry.typedPath.getPath == foo) Trigger else Ignore, + triggeredMessage = (tp, _) => { queue += tp; None }, + watchingMessage = _ => { Files.createFile(bar); Thread.sleep(5); Files.createFile(foo); None } + ) + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch + queue.toIndexedSeq.map(_.getPath) shouldBe Seq(foo) + } + it should "enforce anti-entropy" in IO.withTemporaryDirectory { dir => + val realDir = dir.toRealPath + val queue = new mutable.Queue[TypedPath] + val foo = realDir.toPath.resolve("foo") + val bar = realDir.toPath.resolve("bar") + val config = Defaults.config( + sources = Seq(WatchSource(realDir)), + preWatch = (count, _) => if (count == 3) CancelWatch else Ignore, + onWatchEvent = _ => Trigger, + triggeredMessage = (tp, _) => { queue += tp; None }, + watchingMessage = count => { + count match { + case 1 => Files.createFile(bar) + case 2 => + bar.toFile.setLastModified(5000) + Files.createFile(foo) + case _ => + } + None + } + ) + Watched.watch(NullInputStream, () => Right(true), config) shouldBe CancelWatch + queue.toIndexedSeq.map(_.getPath) shouldBe Seq(bar, foo) + } + it should "halt on error" in IO.withTemporaryDirectory { dir => + val halted = new AtomicBoolean(false) + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + preWatch = (_, lastStatus) => if (lastStatus) Ignore else { halted.set(true); HandleError } + ) + Watched.watch(NullInputStream, () => Right(false), config) shouldBe HandleError + assert(halted.get()) + } + it should "reload" in IO.withTemporaryDirectory { dir => + val config = Defaults.config( + sources = Seq(WatchSource(dir.toRealPath)), + preWatch = (_, _) => Ignore, + onWatchEvent = _ => Reload, + watchingMessage = _ => { new File(dir, "file").createNewFile(); None } + ) + Watched.watch(NullInputStream, () => Right(true), config) shouldBe Reload + } +} + +object WatchedSpec { + implicit class FileOps(val f: File) { + def toRealPath: File = f.toPath.toRealPath().toFile + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 8d721792d..3c54cadf2 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -40,16 +40,18 @@ import sbt.internal.util.Types._ import sbt.io.syntax._ import sbt.io.{ AllPassFilter, + DirectoryFilter, FileFilter, + FileTreeView, GlobFilter, + Hash, HiddenFileFilter, IO, NameFilter, NothingFilter, Path, PathFinder, - DirectoryFilter, - Hash + TypedPath }, Path._ import sbt.librarymanagement.Artifact.{ DocClassifier, SourceClassifier } import sbt.librarymanagement.Configurations.{ @@ -248,7 +250,9 @@ object Defaults extends BuildCommon { extraLoggers :== { _ => Nil }, + pollingDirectories :== Nil, watchSources :== Nil, + watchProjectSources :== Nil, skip :== false, taskTemporaryDirectory := { val dir = IO.createTemporaryDirectory; dir.deleteOnExit(); dir }, onComplete := { @@ -264,7 +268,21 @@ object Defaults extends BuildCommon { concurrentRestrictions := defaultRestrictions.value, parallelExecution :== true, pollInterval :== new FiniteDuration(500, TimeUnit.MILLISECONDS), - watchAntiEntropy :== new FiniteDuration(40, TimeUnit.MILLISECONDS), + watchTriggeredMessage := { (_, _) => + None + }, + watchStartMessage := Watched.defaultStartWatch, + fileTreeViewConfig := FileManagement.defaultFileTreeView.value, + fileTreeView := state.value + .get(BasicKeys.globalFileTreeView) + .getOrElse(FileTreeView.DEFAULT.asDataView(StampedFile.converter)), + externalHooks := { + val view = fileTreeView.value + compileOptions => + Some(ExternalHooks(compileOptions, view)) + }, + watchAntiEntropy :== new FiniteDuration(500, TimeUnit.MILLISECONDS), + watchLogger := streams.value.log, watchService :== { () => Watched.createWatchService() }, @@ -351,12 +369,14 @@ object Defaults extends BuildCommon { crossPaths.value ) }, - unmanagedSources := collectFiles( - unmanagedSourceDirectories, - includeFilter in unmanagedSources, - excludeFilter in unmanagedSources - ).value, - watchSources in ConfigGlobal ++= { + unmanagedSources := FileManagement + .collectFiles( + unmanagedSourceDirectories, + includeFilter in unmanagedSources, + excludeFilter in unmanagedSources + ) + .value, + watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val baseDir = baseDirectory.value val bases = unmanagedSourceDirectories.value val include = (includeFilter in unmanagedSources).value @@ -377,6 +397,13 @@ object Defaults extends BuildCommon { else Nil bases.map(b => new Source(b, include, exclude)) ++ baseSources }, + watchProjectSources in ConfigGlobal := (watchProjectSources in ConfigGlobal).value ++ { + val baseDir = baseDirectory.value + Seq( + new Source(baseDir, "*.sbt", HiddenFileFilter, recursive = false), + new Source(baseDir / "project", "*.sbt" || "*.scala", HiddenFileFilter, recursive = true) + ) + }, managedSourceDirectories := Seq(sourceManaged.value), managedSources := generate(sourceGenerators).value, sourceGenerators :== Nil, @@ -393,12 +420,14 @@ object Defaults extends BuildCommon { resourceDirectories := Classpaths .concatSettings(unmanagedResourceDirectories, managedResourceDirectories) .value, - unmanagedResources := collectFiles( - unmanagedResourceDirectories, - includeFilter in unmanagedResources, - excludeFilter in unmanagedResources - ).value, - watchSources in ConfigGlobal ++= { + unmanagedResources := FileManagement + .collectFiles( + unmanagedResourceDirectories, + includeFilter in unmanagedResources, + excludeFilter in unmanagedResources + ) + .value, + watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val bases = unmanagedResourceDirectories.value val include = (includeFilter in unmanagedResources).value val exclude = (excludeFilter in unmanagedResources).value @@ -411,19 +440,11 @@ object Defaults extends BuildCommon { managedResources := generate(resourceGenerators).value, resources := Classpaths.concat(managedResources, unmanagedResources).value ) + def addBaseSources = FileManagement.appendBaseSources lazy val outputConfigPaths = Seq( classDirectory := crossTarget.value / (prefix(configuration.value.name) + "classes"), target in doc := crossTarget.value / (prefix(configuration.value.name) + "api") ) - def addBaseSources = Seq( - unmanagedSources := { - val srcs = unmanagedSources.value - val f = (includeFilter in unmanagedSources).value - val excl = (excludeFilter in unmanagedSources).value - val baseDir = baseDirectory.value - if (sourcesInBase.value) (srcs +++ baseDir * (f -- excl)).get else srcs - } - ) // This is included into JvmPlugin.projectSettings def compileBase = inTask(console)(compilersSetting :: Nil) ++ compileBaseGlobal ++ Seq( @@ -599,17 +620,62 @@ object Defaults extends BuildCommon { clean := (Def.task { IO.delete(cleanFiles.value) } tag (Tags.Clean)).value, consoleProject := consoleProjectTask.value, watchTransitiveSources := watchTransitiveSourcesTask.value, - watchingMessage := Watched.projectWatchingMessage(thisProjectRef.value.project), - watch := watchSetting.value + watchProjectTransitiveSources := watchTransitiveSourcesTaskImpl(watchProjectSources).value, + watchOnEvent := { + val sources = watchTransitiveSources.value + val projectSources = watchProjectTransitiveSources.value + e => + if (sources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Trigger + else if (projectSources.exists(_.accept(e.entry.typedPath.getPath))) Watched.Reload + else Watched.Ignore + }, + watchHandleInput := Watched.handleInput, + watchPreWatch := { (_, _) => + Watched.Ignore + }, + watchOnTermination := Watched.onTermination, + watchConfig := { + val sources = watchTransitiveSources.value ++ watchProjectTransitiveSources.value + val extracted = Project.extract(state.value) + val wm = extracted + .getOpt(watchingMessage) + .map(w => (count: Int) => Some(w(WatchState.empty(sources).withCount(count)))) + .getOrElse(watchStartMessage.value) + val tm = extracted + .getOpt(triggeredMessage) + .map( + tm => (_: TypedPath, count: Int) => Some(tm(WatchState.empty(sources).withCount(count))) + ) + .getOrElse(watchTriggeredMessage.value) + val logger = watchLogger.value + WatchConfig.default( + logger, + fileTreeViewConfig.value.newMonitor(fileTreeView.value, sources, logger), + watchHandleInput.value, + watchPreWatch.value, + watchOnEvent.value, + watchOnTermination.value, + tm, + wm + ) + }, + watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), + watch := watchSetting.value, + fileTreeViewConfig := FileManagement.defaultFileTreeView.value ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = generators { _.join.map(_.flatten) } - def watchTransitiveSourcesTask: Initialize[Task[Seq[Source]]] = { + def watchTransitiveSourcesTask: Initialize[Task[Seq[Source]]] = + watchTransitiveSourcesTaskImpl(watchSources) + + private def watchTransitiveSourcesTaskImpl( + key: TaskKey[Seq[Source]] + ): Initialize[Task[Seq[Source]]] = { import ScopeFilter.Make.{ inDependencies => inDeps, _ } val selectDeps = ScopeFilter(inAggregates(ThisProject) || inDeps(ThisProject)) - val allWatched = (watchSources ?? Nil).all(selectDeps) + val allWatched = (key ?? Nil).all(selectDeps) Def.task { allWatched.value.flatten } } @@ -621,6 +687,7 @@ object Defaults extends BuildCommon { Def.task { allUpdates.value.flatten ++ globalPluginUpdate.?.value } } + @deprecated("This is no longer used to implement continuous execution", "1.3.0") def watchSetting: Initialize[Watched] = Def.setting { val getService = watchService.value @@ -1127,10 +1194,7 @@ object Defaults extends BuildCommon { dirs: ScopedTaskable[Seq[File]], filter: ScopedTaskable[FileFilter], excludes: ScopedTaskable[FileFilter] - ): Initialize[Task[Seq[File]]] = - Def.task { - dirs.toTask.value.descendantsExcept(filter.toTask.value, excludes.toTask.value).get - } + ): Initialize[Task[Seq[File]]] = FileManagement.collectFiles(dirs, filter, excludes) def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] = Def.setting { val f = artifactName.value @@ -1587,12 +1651,22 @@ object Defaults extends BuildCommon { foldMappers(sourcePositionMappers.value) ) }, - compileInputs := Inputs.of( - compilers.value, - compileOptions.value, - compileIncSetup.value, - previousCompile.value - ) + compileInputs := { + val options = compileOptions.value + val setup = compileIncSetup.value + Inputs.of( + compilers.value, + options, + externalHooks + .value(options) + .map { hooks => + val newOptions = setup.incrementalCompilerOptions.withExternalHooks(hooks) + setup.withIncrementalCompilerOptions(newOptions) + } + .getOrElse(setup), + previousCompile.value + ) + } ) } @@ -1702,7 +1776,7 @@ object Defaults extends BuildCommon { lazy val compileSettings: Seq[Setting[_]] = configSettings ++ - (mainBgRunMainTask +: mainBgRunTask +: addBaseSources) ++ + (mainBgRunMainTask +: mainBgRunTask +: FileManagement.appendBaseSources) ++ Classpaths.addUnmanagedLibrary lazy val testSettings: Seq[Setting[_]] = configSettings ++ testTasks diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b8e5ff33e..fcd817bc3 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -7,7 +7,7 @@ package sbt -import java.io.File +import java.io.{ File, InputStream } import java.net.URL import scala.concurrent.duration.{ FiniteDuration, Duration } import Def.ScopedKey @@ -20,6 +20,7 @@ import xsbti.compile.{ CompileOrder, Compilers, CompileResult, + ExternalHooks, GlobalsCache, IncOptions, Inputs, @@ -40,7 +41,8 @@ import sbt.internal.{ SessionSettings, LogManager } -import sbt.io.{ FileFilter, WatchService } +import sbt.io.{ FileFilter, FileTreeDataView, TypedPath, WatchService } +import sbt.io.FileEventMonitor.Event import sbt.internal.io.WatchState import sbt.internal.server.ServerHandler import sbt.internal.util.{ AttributeKey, SourcePosition } @@ -141,15 +143,31 @@ object Keys { val serverHandlers = settingKey[Seq[ServerHandler]]("User-defined server handlers.") val analysis = AttributeKey[CompileAnalysis]("analysis", "Analysis of compilation, including dependencies and generated outputs.", DSetting) + @deprecated("This is no longer used for continuous execution", "1.3.0") val watch = SettingKey(BasicKeys.watch) val suppressSbtShellNotification = settingKey[Boolean]("""True to suppress the "Executing in batch mode.." message.""").withRank(CSetting) - val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) + val fileTreeView = taskKey[FileTreeDataView[StampedFile]]("A view of the file system") val pollInterval = settingKey[FiniteDuration]("Interval between checks for modified sources by the continuous execution command.").withRank(BMinusSetting) + val pollingDirectories = settingKey[Seq[Watched.WatchSource]]("Directories that cannot be cached and must always be rescanned. Typically these will be NFS mounted or something similar.").withRank(DSetting) + val watchAntiEntropy = settingKey[FiniteDuration]("Duration for which the watch EventMonitor will ignore events for a file after that file has triggered a build.").withRank(BMinusSetting) + val watchConfig = taskKey[WatchConfig]("The configuration for continuous execution.").withRank(BMinusSetting) + val watchLogger = taskKey[Logger]("A logger that reports watch events.").withRank(DSetting) + val watchHandleInput = settingKey[InputStream => Watched.Action]("Function that is periodically invoked to determine if the continous build should be stopped or if a build should be triggered. It will usually read from stdin to respond to user commands.").withRank(BMinusSetting) + val watchOnEvent = taskKey[Event[StampedFile] => Watched.Action]("Determines how to handle a file event").withRank(BMinusSetting) + val watchOnTermination = taskKey[(Watched.Action, String, State) => State]("Transforms the input state after the continuous build completes.").withRank(BMinusSetting) val watchService = settingKey[() => WatchService]("Service to use to monitor file system changes.").withRank(BMinusSetting) + val watchProjectSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources for the sbt meta project to watch to trigger a reload.").withRank(CSetting) + val watchProjectTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for the sbt meta project to watch to trigger a reload.").withRank(CSetting) + val watchPreWatch = settingKey[(Int, Boolean) => Watched.Action]("Function that may terminate a continuous build based on the number of iterations and the last result").withRank(BMinusSetting) val watchSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in this project for continuous execution to watch for changes.").withRank(BMinusSetting) + val watchStartMessage = settingKey[Int => Option[String]]("The message to show when triggered execution waits for sources to change. The parameter is the current watch iteration count.").withRank(DSetting) val watchTransitiveSources = taskKey[Seq[Watched.WatchSource]]("Defines the sources in all projects for continuous execution to watch.").withRank(CSetting) + val watchTriggeredMessage = settingKey[(TypedPath, Int) => Option[String]]("The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count.").withRank(DSetting) + @deprecated("Use watchStartMessage instead", "1.3.0") val watchingMessage = settingKey[WatchState => String]("The message to show when triggered execution waits for sources to change.").withRank(DSetting) + @deprecated("Use watchTriggeredMessage instead", "1.3.0") val triggeredMessage = settingKey[WatchState => String]("The message to show before triggered execution executes an action after sources change.").withRank(DSetting) + val fileTreeViewConfig = taskKey[FileTreeViewConfig]("Configures how sbt will traverse and monitor the file system.").withRank(BMinusSetting) // Path Keys val baseDirectory = settingKey[File]("The base directory. Depending on the scope, this is the base directory for the build, project, configuration, or task.").withRank(AMinusSetting) @@ -244,6 +262,7 @@ object Keys { val copyResources = taskKey[Seq[(File, File)]]("Copies resources to the output directory.").withRank(AMinusTask) val aggregate = settingKey[Boolean]("Configures task aggregation.").withRank(BMinusSetting) val sourcePositionMappers = taskKey[Seq[xsbti.Position => Option[xsbti.Position]]]("Maps positions in generated source files to the original source it was generated from").withRank(DTask) + val externalHooks = taskKey[CompileOptions => Option[ExternalHooks]]("External hooks for modifying the internal behavior of the incremental compiler.").withRank(BMinusSetting) // package keys val packageBin = taskKey[File]("Produces a main artifact, such as a binary jar.").withRank(ATask) diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index ed669cb75..358ee2c62 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -49,7 +49,7 @@ import Project.LoadAction import xsbti.compile.CompilerCache import scala.annotation.tailrec -import sbt.io.IO +import sbt.io.{ FileTreeDataView, IO } import sbt.io.syntax._ import java.io.{ File, IOException } import java.net.URI @@ -241,7 +241,9 @@ object BuiltinCommands { export, boot, initialize, - act + act, + continuous, + flushFileTreeRepository ) ++ allBasicCommands def DefaultBootCommands: Seq[String] = @@ -446,6 +448,14 @@ object BuiltinCommands { s } + def continuous: Command = Watched.continuous { (state: State, command: String) => + val extracted = Project.extract(state) + val (s, watchConfig) = extracted.runTask(Keys.watchConfig, state) + val updateState = + (runCommand: () => State) => MainLoop.processCommand(Exec(command, None), s, runCommand) + (s, watchConfig, updateState) + } + private[this] def loadedEval(s: State, arg: String): Unit = { val extracted = Project extract s import extracted._ @@ -849,7 +859,7 @@ object BuiltinCommands { val session = Load.initialSession(structure, eval, s0) SessionSettings.checkSession(session, s) - Project.setProject(session, structure, s) + registerGlobalFileRepository(Project.setProject(session, structure, s)) } def registerCompilerCache(s: State): State = { @@ -867,6 +877,27 @@ object BuiltinCommands { } s.put(Keys.stateCompilerCache, cache) } + def registerGlobalFileRepository(s: State): State = { + val extracted = Project.extract(s) + try { + val (_, config: FileTreeViewConfig) = extracted.runTask(Keys.fileTreeViewConfig, s) + val view: FileTreeDataView[StampedFile] = config.newDataView() + val newState = s.addExitHook { + view.close() + s.attributes.remove(BasicKeys.globalFileTreeView) + () + } + newState.get(BasicKeys.globalFileTreeView).foreach(_.close()) + newState.put(BasicKeys.globalFileTreeView, view) + } catch { + case NonFatal(_) => s + } + } + + def flushFileTreeRepository: Command = { + val help = Help.more(FlushFileTreeRepository, FlushDetailed) + Command.command(FlushFileTreeRepository, help)(registerGlobalFileRepository) + } def shell: Command = Command.command(Shell, Help.more(Shell, ShellDetailed)) { s0 => import sbt.internal.{ ConsolePromptEvent, ConsoleUnpromptEvent } diff --git a/main/src/main/scala/sbt/MainLoop.scala b/main/src/main/scala/sbt/MainLoop.scala index c9f4d171c..eb32d333f 100644 --- a/main/src/main/scala/sbt/MainLoop.scala +++ b/main/src/main/scala/sbt/MainLoop.scala @@ -144,13 +144,20 @@ object MainLoop { } /** This is the main function State transfer function of the sbt command processing. */ - def processCommand(exec: Exec, state: State): State = { + def processCommand(exec: Exec, state: State): State = + processCommand(exec, state, () => Command.process(exec.commandLine, state)) + + private[sbt] def processCommand( + exec: Exec, + state: State, + runCommand: () => State + ): State = { val channelName = exec.source map (_.channelName) StandardMain.exchange publishEventMessage ExecStatusEvent("Processing", channelName, exec.execId, Vector()) try { - val newState = Command.process(exec.commandLine, state) + val newState = runCommand() val doneEvent = ExecStatusEvent( "Done", channelName, diff --git a/main/src/main/scala/sbt/internal/ExternalHooks.scala b/main/src/main/scala/sbt/internal/ExternalHooks.scala new file mode 100644 index 000000000..08d7e0538 --- /dev/null +++ b/main/src/main/scala/sbt/internal/ExternalHooks.scala @@ -0,0 +1,125 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal +import java.nio.file.Paths +import java.util.Optional + +import sbt.StampedFile +import sbt.internal.inc.ExternalLookup +import sbt.io.syntax.File +import sbt.io.{ FileTreeRepository, FileTreeDataView, TypedPath } +import xsbti.compile._ +import xsbti.compile.analysis.Stamp + +import scala.collection.mutable + +private[sbt] object ExternalHooks { + private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_)) + def apply(options: CompileOptions, view: FileTreeDataView[StampedFile]): DefaultExternalHooks = { + import scala.collection.JavaConverters._ + val sources = options.sources() + val cachedSources = new java.util.HashMap[File, Stamp] + val converter: File => Stamp = f => StampedFile.sourceConverter(TypedPath(f.toPath)).stamp + sources.foreach { + case sf: StampedFile => cachedSources.put(sf, sf.stamp) + case f: File => cachedSources.put(f, converter(f)) + } + view match { + case r: FileTreeRepository[StampedFile] => + r.register(options.classesDirectory.toPath, Integer.MAX_VALUE) + options.classpath.foreach { f => + r.register(f.toPath, Integer.MAX_VALUE) + } + case _ => + } + val allBinaries = new java.util.HashMap[File, Stamp] + options.classpath.foreach { f => + view.listEntries(f.toPath, Integer.MAX_VALUE, _ => true) foreach { e => + e.value match { + case Right(value) => allBinaries.put(e.typedPath.getPath.toFile, value.stamp) + case _ => + } + } + // This gives us the entry for the path itself, which is necessary if the path is a jar file + // rather than a directory. + view.listEntries(f.toPath, -1, _ => true) foreach { e => + e.value match { + case Right(value) => allBinaries.put(e.typedPath.getPath.toFile, value.stamp) + case _ => + } + } + } + + val lookup = new ExternalLookup { + override def changedSources(previousAnalysis: CompileAnalysis): Option[Changes[File]] = Some { + new Changes[File] { + val getAdded: java.util.Set[File] = new java.util.HashSet[File] + val getRemoved: java.util.Set[File] = new java.util.HashSet[File] + val getChanged: java.util.Set[File] = new java.util.HashSet[File] + val getUnmodified: java.util.Set[File] = new java.util.HashSet[File] + override def isEmpty: java.lang.Boolean = + getAdded.isEmpty && getRemoved.isEmpty && getChanged.isEmpty + val prevSources: mutable.Map[File, Stamp] = + previousAnalysis.readStamps().getAllSourceStamps.asScala + prevSources.foreach { + case (file: File, s: Stamp) => + cachedSources.get(file) match { + case null => + getRemoved.add(file) + case stamp => + if ((stamp.getHash.orElse("") == s.getHash.orElse("")) && (stamp.getLastModified + .orElse(-1L) == s.getLastModified.orElse(-1L))) { + getUnmodified.add(file) + } else { + getChanged.add(file) + } + } + } + sources.foreach(file => if (!prevSources.contains(file)) getAdded.add(file)) + } + } + + override def shouldDoIncrementalCompilation( + set: Set[String], + compileAnalysis: CompileAnalysis + ): Boolean = true + + // This could use the cache as well, but it would complicate the cache implementation. + override def hashClasspath(files: Array[File]): Optional[Array[FileHash]] = + Optional.empty[Array[FileHash]] + + override def changedBinaries(previousAnalysis: CompileAnalysis): Option[Set[File]] = { + Some(previousAnalysis.readStamps.getAllBinaryStamps.asScala.flatMap { + case (file, stamp) => + allBinaries.get(file) match { + case null => + javaHome match { + case Some(h) if file.toPath.startsWith(h) => None + case _ => Some(file) + } + case cachedStamp if stamp == cachedStamp => None + case _ => Some(file) + } + }.toSet) + } + + override def removedProducts(previousAnalysis: CompileAnalysis): Option[Set[File]] = { + Some(previousAnalysis.readStamps.getAllProductStamps.asScala.flatMap { + case (file, s) => + allBinaries get file match { + case null => Some(file) + case stamp if stamp.getLastModified.orElse(0L) != s.getLastModified.orElse(0L) => + Some(file) + case _ => None + } + }.toSet) + } + } + new DefaultExternalHooks(Optional.of(lookup), Optional.empty[ClassFileManager]) + } +} diff --git a/main/src/main/scala/sbt/internal/FileManagement.scala b/main/src/main/scala/sbt/internal/FileManagement.scala new file mode 100644 index 000000000..15a17461a --- /dev/null +++ b/main/src/main/scala/sbt/internal/FileManagement.scala @@ -0,0 +1,98 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import java.io.IOException +import java.nio.file.Path + +import sbt.Keys._ +import sbt.io.FileTreeDataView.Entry +import sbt.io.syntax.File +import sbt.io.{ FileFilter, FileTreeDataView, FileTreeRepository } +import sbt._ +import BasicCommandStrings.ContinuousExecutePrefix + +private[sbt] object FileManagement { + private[sbt] def defaultFileTreeView: Def.Initialize[Task[FileTreeViewConfig]] = Def.task { + val remaining = state.value.remainingCommands.map(_.commandLine.trim) + // If the session is interactive or if the commands include a continuous build, then use + // the default configuration. Otherwise, use the sbt1_2_compat config, which does not cache + // anything, which makes it less likely to cause issues with CI. + val interactive = remaining.contains("shell") && !remaining.contains("setUpScripted") + val continuous = remaining.exists(_.startsWith(ContinuousExecutePrefix)) + if (interactive || continuous) { + FileTreeViewConfig + .default(watchAntiEntropy.value, pollInterval.value, pollingDirectories.value) + } else FileTreeViewConfig.sbt1_2_compat(pollInterval.value, watchAntiEntropy.value) + } + private[sbt] implicit class FileTreeDataViewOps[+T](val fileTreeDataView: FileTreeDataView[T]) { + def register(path: Path, maxDepth: Int): Either[IOException, Boolean] = { + fileTreeDataView match { + case r: FileTreeRepository[T] => r.register(path, maxDepth) + case _ => Right(false) + } + } + } + + private[sbt] def collectFiles( + dirs: ScopedTaskable[Seq[File]], + filter: ScopedTaskable[FileFilter], + excludes: ScopedTaskable[FileFilter] + ): Def.Initialize[Task[Seq[File]]] = + Def.task { + val sourceDirs = dirs.toTask.value + val view = fileTreeView.value + val include = filter.toTask.value + val ex = excludes.toTask.value + val sourceFilter: Entry[StampedFile] => Boolean = (entry: Entry[StampedFile]) => { + entry.value match { + case Right(sf) => include.accept(sf) && !ex.accept(sf) + case _ => false + } + } + sourceDirs.flatMap { dir => + view.register(dir.toPath, maxDepth = Integer.MAX_VALUE) + view + .listEntries(dir.toPath, maxDepth = Integer.MAX_VALUE, sourceFilter) + .map(e => e.value.getOrElse(e.typedPath.getPath.toFile)) + } + } + + private[sbt] def appendBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Seq( + unmanagedSources := { + val sources = unmanagedSources.value + val f = (includeFilter in unmanagedSources).value + val excl = (excludeFilter in unmanagedSources).value + val baseDir = baseDirectory.value + val view = fileTreeView.value + if (sourcesInBase.value) { + view.register(baseDir.toPath, maxDepth = 0) + sources ++ + view + .listEntries( + baseDir.toPath, + maxDepth = 0, + e => { + val tp = e.typedPath + /* + * The TypedPath has the isDirectory and isFile properties embedded. By overriding + * these methods in java.io.File, FileFilters may be applied without needing to + * stat the file (which is expensive) for isDirectory and isFile checks. + */ + val file = new java.io.File(tp.getPath.toString) { + override def isDirectory: Boolean = tp.isDirectory + override def isFile: Boolean = tp.isFile + } + f.accept(file) && !excl.accept(file) + } + ) + .flatMap(_.value.toOption) + } else sources + } + ) +} diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index afa0125f4..cf62b1fc6 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -113,6 +113,7 @@ private[sbt] object Load { val delegates = defaultDelegates val pluginMgmt = PluginManagement(loader) val inject = InjectSettings(injectGlobal(state), Nil, const(Nil)) + System.setProperty("swoval.tmpdir", System.getProperty("swoval.tmpdir", globalBase.toString)) LoadBuildConfiguration( stagingDirectory, classpath, diff --git a/sbt/src/sbt-test/watch/on-start-watch/build.sbt b/sbt/src/sbt-test/watch/on-start-watch/build.sbt new file mode 100644 index 000000000..1c6dab6c1 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/build.sbt @@ -0,0 +1,53 @@ +import scala.util.Try + +val checkCount = inputKey[Unit]("check that compile has run a specified number of times") +val checkReloadCount = inputKey[Unit]("check whether the project was reloaded") +val failingTask = taskKey[Unit]("should always fail") +val maybeReload = settingKey[(Int, Boolean) => Watched.Action]("possibly reload") +val resetCount = taskKey[Unit]("reset compile count") +val reloadFile = settingKey[File]("get the current reload file") + +checkCount := { + val expected = Def.spaceDelimited().parsed.head.toInt + if (Count.get != expected) + throw new IllegalStateException(s"Expected ${expected} compilation runs, got ${Count.get}") +} + +maybeReload := { (_, _) => + if (Count.reloadCount(reloadFile.value) == 0) Watched.Reload else Watched.CancelWatch +} + +reloadFile := baseDirectory.value / "reload-count" + +resetCount := { + Count.reset() +} + +failingTask := { + throw new IllegalStateException("failed") +} + +watchPreWatch := maybeReload.value + +checkReloadCount := { + val expected = Def.spaceDelimited().parsed.head.toInt + assert(Count.reloadCount(reloadFile.value) == expected) +} + +val addReloadShutdownHook = Command.command("addReloadShutdownHook") { state => + state.addExitHook { + val base = Project.extract(state).get(baseDirectory) + val file = base / "reload-count" + val currentCount = Try(Count.reloadCount(file)).getOrElse(0) + IO.write(file, s"${currentCount + 1}".getBytes) + } +} + +commands += addReloadShutdownHook + +Compile / compile := { + Count.increment() + // Trigger a new build by updating the last modified time + ((Compile / scalaSource).value / "A.scala").setLastModified(5000) + (Compile / compile).value +} diff --git a/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala new file mode 100644 index 000000000..67d3bf940 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/project/Count.scala @@ -0,0 +1,10 @@ +import sbt._ +import scala.util.Try + +object Count { + private var count = 0 + def get: Int = count + def increment(): Unit = count += 1 + def reset(): Unit = count = 0 + def reloadCount(file: File): Int = Try(IO.read(file).toInt).getOrElse(0) +} diff --git a/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala b/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala new file mode 100644 index 000000000..df9e4d3d5 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/src/main/scala/A.scala @@ -0,0 +1,3 @@ +package a + +class A \ No newline at end of file diff --git a/sbt/src/sbt-test/watch/on-start-watch/test b/sbt/src/sbt-test/watch/on-start-watch/test new file mode 100644 index 000000000..37781fce3 --- /dev/null +++ b/sbt/src/sbt-test/watch/on-start-watch/test @@ -0,0 +1,28 @@ +# verify that reloading occurs if watchPreWatch returns Watched.Reload +> addReloadShutdownHook +> checkReloadCount 0 +> ~compile +> checkReloadCount 1 + +# verify that the watch terminates when we reach the specified count +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.CancelWatch else Watched.Ignore } +> ~compile +> checkCount 2 + +# verify that the watch terminates and returns an error when we reach the specified count +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.HandleError else Watched.Ignore } +# Returning Watched.HandleError causes the '~' command to fail +-> ~compile +> checkCount 2 + +# verify that a re-build is triggered when we reach the specified count +> resetCount +> set watchPreWatch := { (count: Int, _) => if (count == 2) Watched.Trigger else if (count == 3) Watched.CancelWatch else Watched.Ignore } +> ~compile +> checkCount 3 + +# verify that the watch exits and returns an error if the task fails +> set watchPreWatch := { (_, lastStatus: Boolean) => if (lastStatus) Watched.Ignore else Watched.HandleError } +-> ~failingTask