Merge pull request #5530 from eatkins/build-sources

Rework build source monitoring

build.sbt

@@ -903,6 +903,9 @@ lazy val mainProj = (project in file("main"))
         "sbt.internal.ReverseLookupClassLoaderHolder$ReverseLookupClassLoader"
       ),
       exclude[MissingClassProblem]("sbt.internal.LayeredClassLoaderImpl"),
+      exclude[MissingClassProblem]("sbt.internal.FileManagement"),
+      exclude[MissingClassProblem]("sbt.internal.FileManagement$"),
+      exclude[MissingClassProblem]("sbt.internal.FileManagement$CopiedFileTreeRepository"),
       // false positives
       exclude[DirectMissingMethodProblem]("sbt.plugins.IvyPlugin.requires"),
       exclude[DirectMissingMethodProblem]("sbt.plugins.JUnitXmlReportPlugin.requires"),

main/src/main/scala/sbt/Defaults.scala

@@ -413,7 +413,9 @@ object Defaults extends BuildCommon {
     sourceManaged := crossTarget.value / "src_managed",
     resourceManaged := crossTarget.value / "resource_managed",
     // Adds subproject build.sbt files to the global list of build files to monitor
-    Scope.Global / checkBuildSources / fileInputs += baseDirectory.value.toGlob / "*.sbt"
+    Scope.Global / checkBuildSources / pollInterval :==
+      new FiniteDuration(Int.MinValue, TimeUnit.MILLISECONDS),
+    Scope.Global / checkBuildSources / fileInputs += baseDirectory.value.toGlob / "*.sbt",
   )
 
   lazy val configPaths = sourceConfigPaths ++ resourceConfigPaths ++ outputConfigPaths

main/src/main/scala/sbt/Main.scala

@@ -12,7 +12,6 @@ import java.net.URI
 import java.nio.file.{ FileAlreadyExistsException, FileSystems, Files }
 import java.util.Properties
 import java.util.concurrent.ForkJoinPool
-import java.util.concurrent.atomic.AtomicBoolean
 
 import sbt.BasicCommandStrings.{ Shell, TemplateCommand }
 import sbt.Project.LoadAction
@@ -21,6 +20,7 @@ import sbt.internal.Aggregation.AnyKeys
 import sbt.internal.CommandStrings.BootCommand
 import sbt.internal._
 import sbt.internal.inc.ScalaInstance
+import sbt.internal.nio.CheckBuildSources
 import sbt.internal.util.Types.{ const, idFun }
 import sbt.internal.util._
 import sbt.internal.util.complete.{ Parser, SizeParser }
@@ -854,12 +854,9 @@ object BuiltinCommands {
 
     val session = Load.initialSession(structure, eval, s0)
     SessionSettings.checkSession(session, s2)
-    val s3 = addCacheStoreFactoryFactory(
+    val s3 = addCacheStoreFactoryFactory(Project.setProject(session, structure, s2))
-      Project
+    val s4 = LintUnused.lintUnusedFunc(s3)
-        .setProject(session, structure, s2)
+    CheckBuildSources.init(s4)
-        .put(sbt.nio.Keys.hasCheckedMetaBuild, new AtomicBoolean(false))
-    )
-    LintUnused.lintUnusedFunc(s3)
   }
 
   private val addCacheStoreFactoryFactory: State => State = (s: State) => {

main/src/main/scala/sbt/MainLoop.scala

@@ -10,15 +10,14 @@ package sbt
 import java.io.PrintWriter
 import java.util.Properties
 
-import sbt.internal.{ Aggregation, ShutdownHooks }
+import sbt.internal.ShutdownHooks
 import sbt.internal.langserver.ErrorCodes
 import sbt.internal.protocol.JsonRpcResponseError
-import sbt.internal.util.complete.Parser
+import sbt.internal.nio.CheckBuildSources.CheckBuildSourcesKey
 import sbt.internal.util.{ ErrorHandling, GlobalLogBacking, Terminal }
 import sbt.io.{ IO, Using }
 import sbt.protocol._
 import sbt.util.Logger
-import sbt.nio.Keys._
 
 import scala.annotation.tailrec
 import scala.util.control.NonFatal
@@ -191,8 +190,6 @@ object MainLoop {
             } else state
         }
         val newState = Command.process(exec.commandLine, progressState)
-        if (exec.commandLine.contains("session"))
-          newState.get(hasCheckedMetaBuild).foreach(_.set(false))
         val doneEvent = ExecStatusEvent(
           "Done",
           channelName,
@@ -209,26 +206,12 @@ object MainLoop {
         newState.get(sbt.Keys.currentTaskProgress).foreach(_.progress.stop())
         newState.remove(sbt.Keys.currentTaskProgress)
       }
-      // The split on space is to handle 'reboot full' and 'reboot'.
+      state.get(CheckBuildSourcesKey) match {
-      state.currentCommand.flatMap(_.commandLine.trim.split(" ").headOption) match {
+        case Some(cbs) =>
-        case Some("reload") =>
+          if (!cbs.needsReload(state, exec.commandLine)) process()
-          // Reset the hasCheckedMetaBuild parameter so that the next call to checkBuildSources
+          else Exec("reload", None, None) +: exec +: state.remove(CheckBuildSourcesKey)
-          // updates the previous cache for checkBuildSources / fileInputStamps but doesn't log.
-          state.get(hasCheckedMetaBuild).foreach(_.set(false))
-          process()
-        case Some("exit") | Some("reboot") => process()
-        case _ =>
-          val emptyState = state.copy(remainingCommands = Nil).put(Aggregation.suppressShow, true)
-          Parser.parse("checkBuildSources", emptyState.combinedParser) match {
-            case Right(cmd) =>
-              cmd() match {
-                case s if s.remainingCommands.headOption.map(_.commandLine).contains("reload") =>
-                  Exec("reload", None, None) +: exec +: state
         case _ => process()
       }
-            case _ => process()
-          }
-      }
     } catch {
       case err: JsonRpcResponseError =>
         StandardMain.exchange.respondError(err, exec.execId, channelName.map(CommandSource(_)))

main/src/main/scala/sbt/internal/Continuous.scala

@@ -588,7 +588,8 @@ private[sbt] object Continuous extends DeprecatedContinuous {
 
       private[this] val observers: Observers[Event] = new Observers
       private[this] val repo = getRepository(state)
-      private[this] val handle = repo.addObserver(observers)
+      private[this] val handles = new java.util.ArrayList[AutoCloseable]
+      handles.add(repo.addObserver(observers))
       private[this] val eventMonitorObservers = new Observers[Event]
       private[this] val configHandle: AutoCloseable =
         observers.addObserver { e =>
@@ -608,7 +609,12 @@ private[sbt] object Continuous extends DeprecatedContinuous {
             eventMonitorObservers.onNext(e)
           }
         }
-      if (trackMetaBuild) buildGlobs.foreach(repo.register)
+      if (trackMetaBuild) {
+        state.get(CheckBuildSources.CheckBuildSourcesKey).flatMap(_.fileTreeRepository) match {
+          case Some(r) => buildGlobs.foreach(r.register(_).foreach(observers.addObservable))
+          case _       => buildGlobs.foreach(repo.register)
+        }
+      }
 
       private[this] val monitor = FileEventMonitor.antiEntropy(
         eventMonitorObservers,
@@ -623,7 +629,7 @@ private[sbt] object Continuous extends DeprecatedContinuous {
 
       override def close(): Unit = {
         configHandle.close()
-        handle.close()
+        handles.forEach(_.close())
       }
     }
     val watchLogger: WatchLogger = msg => logger.debug(msg.toString)

main/src/main/scala/sbt/internal/FileManagement.scala

@@ -1,30 +0,0 @@
-/*
- * sbt
- * Copyright 2011 - 2018, Lightbend, Inc.
- * Copyright 2008 - 2010, Mark Harrah
- * Licensed under Apache License 2.0 (see LICENSE)
- */
-
-package sbt
-package internal
-
-import java.io.IOException
-import java.nio.file.Path
-
-import sbt.internal.nio.{ FileEvent, FileTreeRepository, Observable, Observer }
-import sbt.nio.file.Glob
-
-private[sbt] object FileManagement {
-  private[sbt] def copy[T](fileTreeRepository: FileTreeRepository[T]): FileTreeRepository[T] =
-    new CopiedFileTreeRepository[T](fileTreeRepository)
-  private[this] class CopiedFileTreeRepository[T](underlying: FileTreeRepository[T])
-      extends FileTreeRepository[T] {
-    override def list(path: Path): Seq[(Path, T)] = underlying.list(path)
-    override def close(): Unit = {}
-    override def register(glob: Glob): Either[IOException, Observable[FileEvent[T]]] =
-      underlying.register(glob)
-    override def addObserver(observer: Observer[FileEvent[T]]): AutoCloseable =
-      underlying.addObserver(observer)
-    override def toString: String = s"CopiedFileTreeRepository($underlying)"
-  }
-}

main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala

@@ -8,31 +8,102 @@
 package sbt
 package internal.nio
 
-import sbt.Keys.{ baseDirectory, state, streams }
+import java.nio.file.Path
+import java.util.concurrent.atomic.{ AtomicBoolean, AtomicReference }
+import sbt.BasicCommandStrings.{ RebootCommand, TerminateAction }
+import sbt.Keys.{ baseDirectory, pollInterval, state }
+import sbt.Scope.Global
 import sbt.SlashSyntax0._
+import sbt.internal.CommandStrings.LoadProject
+import sbt.internal.SysProp
+import sbt.internal.util.AttributeKey
 import sbt.io.syntax._
 import sbt.nio.FileChanges
+import sbt.nio.FileStamp
 import sbt.nio.Keys._
-import sbt.nio.file.{ Glob, ** }
+import sbt.nio.file.{ FileAttributes, FileTreeView, Glob, ** }
 import sbt.nio.file.syntax._
+import sbt.nio.Settings
 
 import scala.annotation.tailrec
+import scala.concurrent.duration.{ Deadline => SDeadline, _ }
 
-private[sbt] object CheckBuildSources {
+/**
-  private[sbt] def needReloadImpl: Def.Initialize[Task[StateTransform]] = Def.task {
+ * This class is used to determine whether sbt needs to automatically reload
-    val logger = streams.value.log
+ * the build because its source files have changed. In general, it will use
-    val st: State = state.value
+ * a FileTreeRepository to monitor the build source directories and it will
-    val firstTime = st.get(hasCheckedMetaBuild).fold(true)(_.compareAndSet(false, true))
+ * only actually check whether any sources have changed if the monitor has
-    (onChangedBuildSource in Scope.Global).value match {
+ * detected any events. Because it's using asynchronous monitoring by default,
-      case IgnoreSourceChanges => StateTransform(identity)
+ * the automatic reloading should not be relied upon in batch scripting. It is
-      case o =>
+ * possible to configure this feature by setting
-        import sbt.nio.FileStamp.Formats._
+ * `Global / onChangedBuildSource / pollInterval`. When this value is set to
-        logger.debug("Checking for meta build source updates")
+ * 0.seconds, then it will poll every time. Otherwise, it will only repoll
-        val previous = (inputFileStamps in checkBuildSources).previous
+ * the build files if the poll interval has elapsed.
-        val changes = (changedInputFiles in checkBuildSources).value
+ */
-        previous.map(changes) match {
+private[sbt] class CheckBuildSources extends AutoCloseable {
-          case Some(fileChanges @ FileChanges(created, deleted, modified, _))
+  private[this] val repository = new AtomicReference[FileTreeRepository[FileAttributes]]
-              if fileChanges.hasChanges && !firstTime =>
+  private[this] val pollingPeriod = new AtomicReference[FiniteDuration]
+  private[this] val sources = new AtomicReference[Seq[Glob]](Nil)
+  private[this] val needUpdate = new AtomicBoolean(true)
+  private[this] val lastPolled = new AtomicReference[SDeadline](SDeadline.now)
+  private[this] val previousStamps = new AtomicReference[Seq[(Path, FileStamp)]]
+  private[sbt] def fileTreeRepository: Option[FileTreeRepository[FileAttributes]] =
+    Option(repository.get)
+  private def getStamps(force: Boolean) = {
+    val now = SDeadline.now
+    val lp = lastPolled.getAndSet(now)
+    if (force || lp + pollingPeriod.get <= now) {
+      FileTreeView.default.list(sources.get) flatMap {
+        case (p, a) if a.isRegularFile => FileStamp.hash(p).map(p -> _)
+        case _                         => None
+      }
+    } else previousStamps.get
+  }
+  private def reset(state: State): Unit = {
+    val extracted = Project.extract(state)
+    val interval = extracted.get(checkBuildSources / pollInterval)
+    val newSources = extracted.get(Global / checkBuildSources / fileInputs)
+    if (interval >= 0.seconds || "polling" == SysProp.watchMode) {
+      Option(repository.getAndSet(null)).foreach(_.close())
+      pollingPeriod.set(interval)
+    } else {
+      pollingPeriod.set(0.seconds)
+      repository.get match {
+        case null =>
+          val repo = FileTreeRepository.default
+          repo.addObserver(_ => needUpdate.set(true))
+          repository.set(repo)
+          newSources.foreach(g => repo.register(g))
+        case r =>
+      }
+    }
+    val previousSources = sources.getAndSet(newSources)
+    if (previousSources != newSources) {
+      fileTreeRepository.foreach(r => newSources.foreach(g => r.register(g)))
+      previousStamps.set(getStamps(force = true))
+    }
+  }
+  private def needCheck(cmd: String): Boolean = {
+    val commands = cmd.split(";").flatMap(_.trim.split(" ").headOption).filterNot(_.isEmpty)
+    val res = !commands.exists { c =>
+      c == LoadProject || c == RebootCommand || c == TerminateAction || c == "shutdown"
+    }
+    if (!res) {
+      previousStamps.set(getStamps(force = true))
+      needUpdate.set(false)
+    }
+    res
+  }
+  @inline private def forceCheck = fileTreeRepository.isEmpty
+  private[sbt] def needsReload(state: State, cmd: String) = {
+    (needCheck(cmd) && (forceCheck || needUpdate.compareAndSet(true, false))) && {
+      val extracted = Project.extract(state)
+      val onChanges = extracted.get(Global / onChangedBuildSource)
+      val logger = state.globalLogging.full
+      val current = getStamps(force = false)
+      val previous = previousStamps.getAndSet(current)
+      Settings.changedFiles(previous, current) match {
+        case fileChanges @ FileChanges(created, deleted, modified, _) if fileChanges.hasChanges =>
           val rawPrefix = s"build source files have changed\n" +
             (if (created.nonEmpty) s"new files: ${created.mkString("\n  ", "\n  ", "\n")}"
              else "") +
@@ -43,23 +114,49 @@ private[sbt] object CheckBuildSources {
                s"modified files: ${modified.mkString("\n  ", "\n  ", "\n")}"
              else "")
           val prefix = rawPrefix.linesIterator.filterNot(_.trim.isEmpty).mkString("\n")
-            if (o == ReloadOnSourceChanges) {
+          if (onChanges == ReloadOnSourceChanges) {
             logger.info(s"$prefix\nReloading sbt...")
-              val remaining =
+            true
-                Exec("reload", None, None) :: st.currentCommand.toList ::: st.remainingCommands
-              StateTransform(_.copy(currentCommand = None, remainingCommands = remaining))
           } else {
             val tail = "Apply these changes by running `reload`.\nAutomatically reload the " +
               "build when source changes are detected by setting " +
               "`Global / onChangedBuildSource := ReloadOnSourceChanges`.\nDisable this " +
               "warning by setting `Global / onChangedBuildSource := IgnoreSourceChanges`."
             logger.warn(s"$prefix\n$tail")
-              StateTransform(identity)
+            false
           }
+        case _ => false
+      }
+    }
+  }
+  override def close(): Unit = {}
+}
+
+private[sbt] object CheckBuildSources {
+  private[sbt] val CheckBuildSourcesKey =
+    AttributeKey[CheckBuildSources]("check-build-source", "", KeyRanks.Invisible)
+  /*
+   * Reuse the same instance of CheckBuildSources across reloads but reset the state. This
+   * should allow the `set` command to work with checkBuildSources / fileInputs and
+   * checkBuildSources / pollInterval. The latter makes it possible to switch between
+   * the asynchronous and polling implementations during the same sbt session.
+   */
+  private[sbt] def init(state: State): State = state.get(CheckBuildSourcesKey) match {
+    case Some(cbs) =>
+      cbs.reset(state)
+      state
+    case _ =>
+      val cbs = new CheckBuildSources
+      cbs.reset(state)
+      state.put(CheckBuildSourcesKey, cbs)
+  }
+  private[sbt] def needReloadImpl: Def.Initialize[Task[StateTransform]] = Def.task {
+    val st = state.value
+    st.get(CheckBuildSourcesKey) match {
+      case Some(cbs) if (cbs.needsReload(st, "")) => StateTransform("reload" :: (_: State))
       case _                                      => StateTransform(identity)
     }
   }
-  }
   private[sbt] def buildSourceFileInputs: Def.Initialize[Seq[Glob]] = Def.setting {
     if (onChangedBuildSource.value != IgnoreSourceChanges) {
       val baseDir = (LocalRootProject / baseDirectory).value

sbt/src/sbt-test/nio/reload/build.sbt

@@ -1,3 +1,5 @@
+import scala.concurrent.duration._
+
 val foo = inputKey[Unit]("working task")
 foo := {
   val filename = Def.spaceDelimited("").parsed.head
@@ -12,4 +14,6 @@ exists := {
 
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
+ThisBuild / checkBuildSources / pollInterval := 0.seconds
+
 val sub = project

sbt/src/sbt-test/nio/reload/changes/broken.sbt

@@ -1,3 +1,5 @@
+import scala.concurrent.duration._
+
 val foo = inputKey[Unit]("broken task")
 foo := { throw new IllegalStateException("foo") }
 
@@ -9,4 +11,6 @@ exists := {
 
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
+ThisBuild / checkBuildSources / pollInterval := 0.seconds
+
 val sub = project

sbt/src/sbt-test/nio/reload/changes/sub.sbt

@@ -1 +1,5 @@
+import scala.concurrent.duration._
+
 libraryDependencies += "org.scala-sbt" % "sbt" % "1.3.0"
+
+ThisBuild / checkBuildSources / pollInterval := 0.seconds

sbt/src/sbt-test/nio/reload/changes/working.sbt

@@ -1,3 +1,5 @@
+import scala.concurrent.duration._
+
 val foo = inputKey[Unit]("working task")
 foo := {
   val filename = Def.spaceDelimited("").parsed.head
@@ -12,4 +14,6 @@ exists := {
 
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
+ThisBuild / checkBuildSources / pollInterval := 0.seconds
+
 val sub = project

sbt/src/sbt-test/nio/reload/test

@@ -1,3 +1,5 @@
+> show Global / checkBuildSources / pollInterval
+
 > foo bar
 
 > exists bar