Check meta build sources before task evaluation

This commit finally fixes #241 by adding support for sbt to either
print a warning or automatically reload the project if the metabuild
sources have changed. To facilitate this, I introduce a new key,
metaBuildSourceOption which has three options:
1) IgnoreSourceChanges
2) WarnOnSourceChanges
3) ReloadOnSourceChanges

When the former is set, sbt will not check if the meta build sources
have changed. Otherwise, sbt will use the buildStructure / fileInputs to
get the ChangedFiles for the metabuild. If there are any changes, it
will either warn or reload the build depending on the value of
metaBuildSourceOption.

The mechanism for diffing the files is that I add a step to EvaluateTask
where, if the project has been loaded and
metaBuildSourceOption != IgnoreSourceChanges, we evaluate the needReload
task. If we need a reload, we return an error that indicates that a
Reload is necessary. When that error is detected, the MainLoop will
prepend "reload" to the pending commands for the state. Otherwise we
just print a warning and continue.

I benchmarked the overhead of this and it wasn't too bad. I generally
saw it taking 5-20ms to perform the check. Since this is only done once
per task evaluation run, I don't think it's a big deal. When
IgnoreSourceChanges is set, there is O(10us) overhead. If performance
does become a problem, we could add a global watch service and skip the
needReload evaluation if no files have been modified.

I removed the watchTrackMetaBuild key and made it so that the continuous
builds only track the meta build when
metaBuildSourceOption == ReloadOnSourceChanges
This commit is contained in:
Ethan Atkins 2019-05-09 20:08:16 -07:00
parent 4007810adb
commit 8f54ecd536
19 changed files with 196 additions and 55 deletions

View File

@ -20,6 +20,9 @@ final case class Reboot(
) extends xsbti.Reboot {
def arguments = argsList.toArray
}
case object Reload extends Exception
final case class ApplicationID(
groupID: String,
name: String,

View File

@ -37,7 +37,7 @@ import sbt.internal.librarymanagement.mavenint.{
SbtPomExtraProperties
}
import sbt.internal.librarymanagement.{ CustomHttp => _, _ }
import sbt.internal.nio.Globs
import sbt.internal.nio.{ CheckBuildSources, Globs }
import sbt.internal.server.{
Definition,
LanguageServerProtocol,
@ -152,7 +152,8 @@ object Defaults extends BuildCommon {
fileInputs :== Nil,
inputFileStamper :== sbt.nio.FileStamper.Hash,
outputFileStamper :== sbt.nio.FileStamper.LastModified,
watchForceTriggerOnAnyChange :== true,
onChangedBuildSource :== sbt.nio.Keys.WarnOnSourceChanges,
watchForceTriggerOnAnyChange :== false,
watchPersistFileStamps :== true,
watchTriggers :== Nil,
clean := { () },
@ -255,18 +256,10 @@ object Defaults extends BuildCommon {
outputStrategy :== None, // TODO - This might belong elsewhere.
buildStructure := Project.structure(state.value),
settingsData := buildStructure.value.data,
settingsData / fileInputs := {
val baseDir = file(".").getCanonicalFile()
val sourceFilter = "*.{sbt,scala,java}"
val projectDir = baseDir / "project"
Seq(
Glob(baseDir, "*.sbt"),
Glob(projectDir, sourceFilter),
// We only want to recursively look in source because otherwise we have to search
// the project target directories which is expensive.
Glob(projectDir / "src", RecursiveGlob / sourceFilter),
)
},
aggregate in checkBuildSources :== false,
checkBuildSources / Continuous.dynamicInputs := None,
checkBuildSources / fileInputs := CheckBuildSources.buildSourceFileInputs.value,
checkBuildSources := CheckBuildSources.needReloadImpl.value,
trapExit :== true,
connectInput :== false,
cancelable :== true,
@ -362,7 +355,6 @@ object Defaults extends BuildCommon {
watchStartMessage :== Watch.defaultStartWatch,
watchTasks := Continuous.continuousTask.evaluated,
aggregate in watchTasks :== false,
watchTrackMetaBuild :== true,
watchTriggeredMessage :== Watch.defaultOnTriggerMessage,
)
)

View File

@ -16,9 +16,10 @@ import sbt.Project.richInitializeTask
import sbt.Scope.Global
import sbt.internal.Aggregation.KeyValue
import sbt.internal.TaskName._
import sbt.internal.util._
import sbt.internal._
import sbt.internal.util._
import sbt.librarymanagement.{ Resolver, UpdateReport }
import sbt.nio.Keys.IgnoreSourceChanges
import sbt.std.Transform.DummyTaskMap
import sbt.util.{ Logger, Show }
@ -274,7 +275,7 @@ object EvaluateTask {
def injectSettings: Seq[Setting[_]] = Seq(
(state in Global) ::= dummyState,
(streamsManager in Global) ::= Def.dummyStreamsManager,
(executionRoots in Global) ::= dummyRoots
(executionRoots in Global) ::= dummyRoots,
)
@deprecated("Use variant which doesn't take a logger", "1.1.1")
@ -346,7 +347,7 @@ object EvaluateTask {
ExceptionCategory(ex) match {
case AlreadyHandled => ()
case m: MessageOnly => if (msg.isEmpty) log.error(m.message)
case f: Full => log.trace(f.exception)
case f: Full => if (f.exception != Reload) log.trace(f.exception)
}
}
@ -439,7 +440,7 @@ object EvaluateTask {
case Some(t: Task[_]) => transformNode(t).isEmpty
case _ => true
}
def run() = {
def run[R](s: State, toRun: Task[R], doShutdown: Boolean) = {
val x = new Execute[Task](
Execute.config(config.checkCycles, overwriteNode),
triggers,
@ -447,12 +448,12 @@ object EvaluateTask {
)(taskToNode)
val (newState, result) =
try {
val results = x.runKeep(root)(service)
storeValuesForPrevious(results, state, streams)
applyResults(results, state, root)
} catch { case inc: Incomplete => (state, Inc(inc)) } finally shutdown()
val results = x.runKeep(toRun)(service)
storeValuesForPrevious(results, s, streams)
applyResults(results, s, toRun)
} catch { case inc: Incomplete => (s, Inc(inc)) } finally if (doShutdown) shutdown()
val replaced = transformInc(result)
logIncResult(replaced, state, streams)
logIncResult(replaced, s, streams)
(newState, replaced)
}
object runningEngine extends RunningTaskEngine {
@ -466,8 +467,24 @@ object EvaluateTask {
// Register with our cancel handler we're about to start.
val strat = config.cancelStrategy
val cancelState = strat.onTaskEngineStart(runningEngine)
try run()
finally {
try {
(state.get(stateBuildStructure), state.get(sessionSettings)) match {
case (Some(structure), Some(settings)) =>
val extracted: Extracted = Project.extract(settings, structure)
if (extracted.get(sbt.nio.Keys.onChangedBuildSource) == IgnoreSourceChanges) {
run(state, root, doShutdown = true)
} else {
run(state, extracted.get(sbt.nio.Keys.checkBuildSources), doShutdown = false) match {
case (newState, r) =>
r.toEither match {
case Left(i) => (newState, Result.fromEither(Left(i)))
case _ => run(newState, root, doShutdown = true)
}
}
}
case _ => run(state, root, doShutdown = true)
}
} finally {
strat.onTaskEngineFinish(cancelState)
currentlyRunningEngine.set(null)
lastEvaluatedState.set(SafeState(state))

View File

@ -267,7 +267,7 @@ object BuiltinCommands {
BasicCommands.multi,
act,
continuous,
clearCaches
clearCaches,
) ++ allBasicCommands
def DefaultBootCommands: Seq[String] =
@ -880,6 +880,7 @@ object BuiltinCommands {
val session = Load.initialSession(structure, eval, s0)
SessionSettings.checkSession(session, s)
registerGlobalCaches(Project.setProject(session, structure, s))
.put(sbt.nio.Keys.hasCheckedMetaBuild, new AtomicBoolean(false))
}
def registerCompilerCache(s: State): State = {

View File

@ -11,16 +11,15 @@ import java.io.PrintWriter
import java.util.Properties
import jline.TerminalFactory
import sbt.internal.langserver.ErrorCodes
import sbt.internal.util.{ ErrorHandling, GlobalLogBacking }
import sbt.io.{ IO, Using }
import sbt.protocol._
import sbt.util.Logger
import scala.annotation.tailrec
import scala.util.control.NonFatal
import sbt.io.{ IO, Using }
import sbt.internal.util.{ ErrorHandling, GlobalLogBacking }
import sbt.internal.langserver.ErrorCodes
import sbt.util.Logger
import sbt.protocol._
object MainLoop {
/** Entry point to run the remaining commands in State with managed global logging.*/
@ -140,7 +139,10 @@ object MainLoop {
case Right(s) => s
case Left(t: xsbti.FullReload) => throw t
case Left(t: RebootCurrent) => throw t
case Left(t) => state.handleError(t)
case Left(Reload) =>
val remaining = state.currentCommand.toList ::: state.remainingCommands
state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
case Left(t) => state.handleError(t)
}
/** This is the main function State transfer function of the sbt command processing. */

View File

@ -111,6 +111,9 @@ object Aggregation {
val complete = timedRun[T](s, ts, extra)
showRun(complete, show)
complete.results match {
case Inc(i) if i.directCause.contains(Reload) =>
val remaining = s.currentCommand.toList ::: s.remainingCommands
complete.state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
case Inc(i) => complete.state.handleError(i)
case Value(_) => complete.state
}

View File

@ -316,7 +316,7 @@ private[sbt] object Continuous extends DeprecatedContinuous {
callbacks.onEnter()
// Here we enter the Watched.watch state machine. We will not return until one of the
// state machine callbacks returns Watched.CancelWatch, Watched.Custom, Watched.HandleError
// or Watched.Reload. The task defined above will be run at least once. It will be run
// or Watched.ReloadException. The task defined above will be run at least once. It will be run
// additional times whenever the state transition callbacks return Watched.Trigger.
try {
val terminationAction = Watch(task, callbacks.onStart, callbacks.nextEvent)
@ -482,7 +482,7 @@ private[sbt] object Continuous extends DeprecatedContinuous {
)(implicit extracted: Extracted): (() => Option[(Watch.Event, Watch.Action)], () => Unit) = {
val trackMetaBuild = configs.forall(_.watchSettings.trackMetaBuild)
val buildGlobs =
if (trackMetaBuild) extracted.getOpt(fileInputs in settingsData).getOrElse(Nil)
if (trackMetaBuild) extracted.getOpt(fileInputs in checkBuildSources).getOrElse(Nil)
else Nil
val retentionPeriod = configs.map(_.watchSettings.antiEntropyRetentionPeriod).max
@ -558,7 +558,20 @@ private[sbt] object Continuous extends DeprecatedContinuous {
// Create a logger with a scoped key prefix so that we can tell from which
// monitor events occurred.
FileEventMonitor.antiEntropy(
getRepository(state),
new Observable[Event] {
private[this] val repo = getRepository(state)
private[this] val observers = new Observers[Event] {
override def onNext(t: Event): Unit =
if (config.inputs().exists(_.glob.matches(t.path))) super.onNext(t)
}
private[this] val handle = repo.addObserver(observers)
override def addObserver(observer: Observer[Event]): AutoCloseable =
observers.addObserver(observer)
override def close(): Unit = {
handle.close()
observers.close()
}
},
config.watchSettings.antiEntropy,
logger.withPrefix(config.key.show),
config.watchSettings.deletionQuarantinePeriod,
@ -825,7 +838,8 @@ private[sbt] object Continuous extends DeprecatedContinuous {
val onTermination: Option[(Watch.Action, String, Int, State) => State] =
key.get(watchOnTermination)
val startMessage: StartMessage = getStartMessage(key)
val trackMetaBuild: Boolean = key.get(watchTrackMetaBuild).getOrElse(true)
val trackMetaBuild: Boolean =
key.get(onChangedBuildSource).fold(false)(_ == ReloadOnSourceChanges)
val triggerMessage: TriggerMessage = getTriggerMessage(key)
// Unlike the rest of the settings, InputStream is a TaskKey which means that if it is set,

View File

@ -0,0 +1,65 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt
package internal.nio
import sbt.Keys.{ baseDirectory, state, streams }
import sbt.SlashSyntax0._
import sbt.io.syntax._
import sbt.nio.Keys._
import sbt.nio.file.{ ChangedFiles, Glob, RecursiveGlob }
private[sbt] object CheckBuildSources {
private[sbt] def needReloadImpl: Def.Initialize[Task[Unit]] = Def.task {
val logger = streams.value.log
val checkMetaBuildParam = state.value.get(hasCheckedMetaBuild)
val firstTime = checkMetaBuildParam.fold(true)(_.get == false)
(onChangedBuildSource in Scope.Global).value match {
case IgnoreSourceChanges => ()
case o =>
logger.debug("Checking for meta build source updates")
(changedInputFiles in checkBuildSources).value match {
case Some(cf: ChangedFiles) if !firstTime =>
val rawPrefix = s"Meta build source files have changed:\n" +
(if (cf.created.nonEmpty) s"creations: ${cf.created.mkString("\n ", " \n", "\n")}"
else "") +
(if (cf.deleted.nonEmpty) s"deletions: ${cf.deleted.mkString("\n ", " \n", "\n")}"
else "") +
(if (cf.updated.nonEmpty) s"updates: ${cf.updated.mkString("\n ", " \n", "\n")}"
else "")
val prefix = rawPrefix.linesIterator.filterNot(_.trim.isEmpty).mkString("\n")
if (o == ReloadOnSourceChanges) {
logger.info(s"$prefix\nReloading sbt...")
throw Reload
} else {
val tail = "Reload sbt with the 'reload' command to apply these changes. " +
"To automatically reload upon meta build source changed detection, set " +
"`Global / onChangedBuildSource := ReloadOnSourceChanges`. To disable this " +
"warning, set `Global / onChangedBuildSource := IgnoreSourceChanges`"
logger.warn(s"$prefix\n$tail")
}
case _ => ()
}
}
checkMetaBuildParam.foreach(_.set(true))
}
private[sbt] def buildSourceFileInputs: Def.Initialize[Seq[Glob]] = Def.setting {
if (onChangedBuildSource.value != IgnoreSourceChanges) {
val baseDir = (LocalRootProject / baseDirectory).value
val sourceFilter = "*.{sbt,scala,java}"
val projectDir = baseDir / "project"
Seq(
Glob(baseDir, "*.sbt"),
Glob(projectDir, sourceFilter),
// We only want to recursively look in source because otherwise we have to search
// the project target directories which is expensive.
Glob(projectDir / "src", RecursiveGlob / sourceFilter),
)
} else Nil
}
}

View File

@ -9,6 +9,7 @@ package sbt.nio
import java.io.InputStream
import java.nio.file.Path
import java.util.concurrent.atomic.AtomicBoolean
import sbt.BuildSyntax.{ settingKey, taskKey }
import sbt.KeyRanks.{ BMinusSetting, DSetting, Invisible }
@ -22,6 +23,10 @@ import sbt.{ Def, InputKey, State, StateTransform }
import scala.concurrent.duration.FiniteDuration
object Keys {
sealed trait WatchBuildSourceOption
case object IgnoreSourceChanges extends WatchBuildSourceOption
case object WarnOnSourceChanges extends WatchBuildSourceOption
case object ReloadOnSourceChanges extends WatchBuildSourceOption
val allInputFiles =
taskKey[Seq[Path]]("All of the file inputs for a task excluding directories and hidden files.")
val changedInputFiles = taskKey[Option[ChangedFiles]]("The changed files for a task")
@ -44,10 +49,17 @@ object Keys {
val fileTreeView =
taskKey[FileTreeView.Nio[FileAttributes]]("A view of the local file system tree")
val checkBuildSources =
taskKey[Unit]("Check if any meta build sources have changed").withRank(DSetting)
// watch related settings
val watchAntiEntropyRetentionPeriod = settingKey[FiniteDuration](
"Wall clock Duration for which a FileEventMonitor will store anti-entropy events. This prevents spurious triggers when a task takes a long time to run. Higher values will consume more memory but make spurious triggers less likely."
).withRank(BMinusSetting)
val onChangedBuildSource = settingKey[WatchBuildSourceOption](
"Determines what to do if the sbt meta build sources have changed"
).withRank(DSetting)
val watchDeletionQuarantinePeriod = settingKey[FiniteDuration](
"Period for which deletion events will be quarantined. This is to prevent spurious builds when a file is updated with a rename which manifests as a file deletion followed by a file creation. The higher this value is set, the longer the delay will be between a file deletion and a build trigger but the less likely it is for a spurious trigger."
).withRank(DSetting)
@ -96,9 +108,6 @@ object Keys {
"watch",
"Watch a task (or multiple tasks) and rebuild when its file inputs change or user input is received. The semantics are more or less the same as the `~` command except that it cannot transform the state on exit. This means that it cannot be used to reload the build."
).withRank(DSetting)
val watchTrackMetaBuild = settingKey[Boolean](
s"Toggles whether or not changing the build files (e.g. **/*.sbt, project/**/*.{scala,java}) should automatically trigger a project reload"
).withRank(DSetting)
val watchTriggeredMessage = settingKey[(Int, Path, Seq[String]) => Option[String]](
"The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count."
).withRank(DSetting)
@ -139,4 +148,13 @@ object Keys {
private[sbt] val pathToFileStamp = taskKey[Path => Option[FileStamp]](
"A function that computes a file stamp for a path. It may have the side effect of updating a cache."
).withRank(Invisible)
private[this] val hasCheckedMetaBuildMsg =
"Indicates whether or not we have called the checkBuildSources task. This is to avoid warning " +
"user about build source changes if the build sources were changed while sbt was shutdown. " +
" When that occurs, the previous cache reflects the state of the old build files, but by " +
" the time the checkBuildSources task has run, the build will have already been loaded with the " +
" new meta build sources so we should neither warn the user nor automatically restart the build"
private[sbt] val hasCheckedMetaBuild =
AttributeKey[AtomicBoolean]("has-checked-meta-build", hasCheckedMetaBuildMsg, Int.MaxValue)
}

View File

@ -196,7 +196,7 @@ private[sbt] object Settings {
val inputs = (fileInputs in scopedKey.scope).value
val stamper = (inputFileStamper in scopedKey.scope).value
val forceTrigger = (watchForceTriggerOnAnyChange in scopedKey.scope).value
val dynamicInputs = Continuous.dynamicInputs.value
val dynamicInputs = (Continuous.dynamicInputs in scopedKey.scope).value
// This makes watch work by ensuring that the input glob is registered with the
// repository used by the watch process.
sbt.Keys.state.value.get(globalFileTreeRepository).foreach { repo =>

View File

@ -245,7 +245,7 @@ object Watch {
* Action that indicates that the watch should pause while the build is reloaded. This is used to
* automatically reload the project when the build files (e.g. build.sbt) are changed.
*/
case object Reload extends CancelWatch
private[sbt] case object Reload extends CancelWatch
/**
* Action that indicates that we should exit and run the provided command.

View File

@ -0,0 +1,4 @@
val foo = taskKey[Unit]("working task")
foo := { println("foo") }
Global / onChangedBuildSource := ReloadOnSourceChanges

View File

@ -0,0 +1,4 @@
val foo = taskKey[Unit]("broken task")
foo := { throw new IllegalStateException("foo") }
Global / onChangedBuildSource := ReloadOnSourceChanges

View File

@ -0,0 +1,4 @@
val foo = taskKey[Unit]("working task")
foo := { println("foo") }
Global / onChangedBuildSource := ReloadOnSourceChanges

View File

@ -0,0 +1,9 @@
> foo
$ copy-file changes/broken.sbt build.sbt
-> foo
$ copy-file changes/working.sbt build.sbt
> foo

View File

@ -5,7 +5,7 @@ val resetCount = taskKey[Unit]("reset compile count")
checkCount := {
val expected = Def.spaceDelimited().parsed.head.toInt
if (Count.get != expected)
throw new IllegalStateException(s"Expected ${expected} compilation runs, got ${Count.get}")
throw new IllegalStateException(s"Expected $expected compilation runs, got ${Count.get}")
}
resetCount := {
@ -16,10 +16,4 @@ failingTask := {
throw new IllegalStateException("failed")
}
Compile / compile := {
Count.increment()
// Trigger a new build by updating the last modified time
val file = (Compile / scalaSource).value / "A.scala"
IO.write(file, IO.read(file) + ("\n" * Count.get))
(Compile / compile).value
}
onChangedBuildSource := ReloadOnSourceChanges

View File

@ -2,3 +2,11 @@ val checkReloaded = taskKey[Unit]("Asserts that the build was reloaded")
checkReloaded := { () }
watchOnIteration := { _ => sbt.nio.Watch.CancelWatch }
Compile / compile := {
Count.increment()
// Trigger a new build by updating the last modified time
val file = (Compile / scalaSource).value / "A.scala"
IO.write(file, IO.read(file) + ("\n" * Count.get))
(Compile / compile).value
}

View File

@ -1 +1,7 @@
watchOnIteration := { _ => sbt.nio.Watch.Reload }
Compile / compile := {
Count.increment()
// Trigger a new build by updating the last modified time
val extra = baseDirectory.value / "extra.sbt"
IO.copyFile(baseDirectory.value / "changes" / "extra.sbt", extra, CopyOptions().withOverwrite(true))
(Compile / compile).value
}

View File

@ -1,6 +1,3 @@
# verify that reloading occurs if watchOnStart returns Watch.Reload
$ copy-file changes/extra.sbt extra.sbt
> ~compile
> checkReloaded