Force invalidate dependency changes

After adding the automatic lookup to external hooks for missing binary
jars, the scripted test dependency-management/invalidate-internal
started failing. This was because the previous analysis contained a jar
dependency that still existed on disk but was no longer a part of the
dependency classpath. Fundamentally the problem is that the zinc
compile analysis is not tightly coupled with the sbt build state.

To fix this, we can cache the dependency classpath file stamps in the
same way that we cache the input file stamps in external hooks and
manually diff them at the sbt level. We then force updates regardless of
the difference between the zinc state and the sbt state.

main/src/main/scala/sbt/Defaults.scala

@@ -610,19 +610,30 @@ object Defaults extends BuildCommon {
     },
     externalHooks := {
       import sbt.nio.FileStamp.Formats.seqPathFileStampJsonFormatter
-      val current =
+      val currentInputs =
         (unmanagedSources / inputFileStamps).value ++ (managedSources / outputFileStamps).value
-      val previous = (externalHooks / inputFileStamps).previous
-      val changes = previous
-        .map(sbt.nio.Settings.changedFiles(_, current))
-        .getOrElse(FileChanges.noPrevious(current.map(_._1)))
-      ExternalHooks.default.value(changes, fileTreeView.value)
+      val previousInputs = (externalHooks / inputFileStamps).previous
+      val inputChanges = previousInputs
+        .map(sbt.nio.Settings.changedFiles(_, currentInputs))
+        .getOrElse(FileChanges.noPrevious(currentInputs.map(_._1)))
+      val currentOutputs = (dependencyClasspathFiles / outputFileStamps).value
+      val previousOutputs = (externalHooks / outputFileStamps).previous
+      val outputChanges = previousOutputs
+        .map(sbt.nio.Settings.changedFiles(_, currentOutputs))
+        .getOrElse(FileChanges.noPrevious(currentOutputs.map(_._1)))
+      ExternalHooks.default.value(inputChanges, outputChanges, fileTreeView.value)
     },
     externalHooks / inputFileStamps := {
       compile.value // ensures the inputFileStamps previous value is only set if compile succeeds.
       (unmanagedSources / inputFileStamps).value ++ (managedSources / outputFileStamps).value
     },
     externalHooks / inputFileStamps := (externalHooks / inputFileStamps).triggeredBy(compile).value,
+    externalHooks / outputFileStamps := {
+      compile.value // ensures the inputFileStamps previous value is only set if compile succeeds.
+      (dependencyClasspathFiles / outputFileStamps).value
+    },
+    externalHooks / outputFileStamps :=
+      (externalHooks / outputFileStamps).triggeredBy(compile).value,
     incOptions := { incOptions.value.withExternalHooks(externalHooks.value) },
     compileIncSetup := compileIncSetupTask.value,
     console := consoleTask.value,
@@ -2009,8 +2020,10 @@ object Classpaths {
         includeFilter in unmanagedJars value,
         excludeFilter in unmanagedJars value
       )
-    ).map(exportClasspath) :+
-      (sbt.nio.Keys.classpathFiles := data(fullClasspath.value).map(_.toPath))
+    ).map(exportClasspath) ++ Seq(
+      sbt.nio.Keys.classpathFiles := data(fullClasspath.value).map(_.toPath),
+      sbt.nio.Keys.dependencyClasspathFiles := data(dependencyClasspath.value).map(_.toPath),
+    )
 
   private[this] def exportClasspath(s: Setting[Task[Classpath]]): Setting[Task[Classpath]] =
     s.mapInitialize(init => Def.task { exportClasspath(streams.value, init.value) })

main/src/main/scala/sbt/internal/ExternalHooks.scala

@@ -26,7 +26,8 @@ import scala.collection.JavaConverters._
 
 private[sbt] object ExternalHooks {
   private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_))
-  private type Func = (FileChanges, FileTreeView[(Path, FileAttributes)]) => ExternalHooks
+  private type Func =
+    (FileChanges, FileChanges, FileTreeView[(Path, FileAttributes)]) => ExternalHooks
   def default: Def.Initialize[sbt.Task[Func]] = Def.task {
     val unmanagedCache = unmanagedFileStampCache.value
     val managedCache = managedFileStampCache.value
@@ -37,15 +38,16 @@ private[sbt] object ExternalHooks {
     }
     val classGlob = classDirectory.value.toGlob / RecursiveGlob / "*.class"
     val options = (compileOptions in compile).value
-    (fc: FileChanges, fileTreeView: FileTreeView[(Path, FileAttributes)]) => {
+    ((inputFileChanges, outputFileChanges, fileTreeView) => {
       fileTreeView.list(classGlob).foreach {
         case (path, _) => managedCache.update(path, FileStamper.LastModified)
       }
-      apply(fc, options, unmanagedCache, managedCache)
-    }
+      apply(inputFileChanges, outputFileChanges, options, unmanagedCache, managedCache)
+    }): Func
   }
   private def apply(
-      changedFiles: FileChanges,
+      inputFileChanges: FileChanges,
+      outputFileChanges: FileChanges,
       options: CompileOptions,
       unmanagedCache: FileStamp.Cache,
       managedCache: FileStamp.Cache
@@ -62,7 +64,7 @@ private[sbt] object ExternalHooks {
           }
           private def add(f: File, set: java.util.Set[File]): Unit = { set.add(f); () }
           val allChanges = new java.util.HashSet[File]
-          changedFiles match {
+          inputFileChanges match {
             case FileChanges(c, d, m, _) =>
               c.foreach(add(_, getAdded, allChanges))
               d.foreach(add(_, getRemoved, allChanges))
@@ -99,7 +101,11 @@ private[sbt] object ExternalHooks {
         Optional.empty[Array[FileHash]]
 
       override def changedBinaries(previousAnalysis: CompileAnalysis): Option[Set[File]] = {
-        Some(previousAnalysis.readStamps.getAllBinaryStamps.asScala.flatMap {
+        val base =
+          (outputFileChanges.modified ++ outputFileChanges.created ++ outputFileChanges.deleted)
+            .map(_.toFile)
+            .toSet
+        Some(base ++ previousAnalysis.readStamps.getAllBinaryStamps.asScala.flatMap {
           case (file, stamp) =>
             managedCache.getOrElseUpdate(file.toPath, FileStamper.LastModified) match {
               case Some(cachedStamp) if equiv(cachedStamp.stamp, stamp) => None
@@ -110,7 +116,7 @@ private[sbt] object ExternalHooks {
                   case _                                    => Some(file)
                 }
             }
-        }.toSet)
+        })
       }
 
       override def removedProducts(previousAnalysis: CompileAnalysis): Option[Set[File]] = {

main/src/main/scala/sbt/nio/Keys.scala

@@ -160,6 +160,8 @@ object Keys {
   private[sbt] val managedFileStampCache = taskKey[FileStamp.Cache](
     "Map of managed file stamps that may be cleared between task evaluation runs."
   ).withRank(Invisible)
+  private[sbt] val dependencyClasspathFiles =
+    taskKey[Seq[Path]]("The dependency classpath for a task.").withRank(Invisible)
   private[sbt] val classpathFiles =
     taskKey[Seq[Path]]("The classpath for a task.").withRank(Invisible)
 

main/src/main/scala/sbt/nio/Settings.scala

@@ -242,12 +242,12 @@ private[sbt] object Settings {
     }
     prevMap.forEach((p, _) => deletedBuilder += p)
     val unmodified = unmodifiedBuilder.result()
-    if (unmodified.size == current.size) {
+    val deleted = deletedBuilder.result()
+    val created = createdBuilder.result()
+    val modified = modifiedBuilder.result()
+    if (created.isEmpty && deleted.isEmpty && modified.isEmpty) {
       FileChanges.unmodified(unmodifiedBuilder.result)
     } else {
-      val created = createdBuilder.result()
-      val deleted = deletedBuilder.result()
-      val modified = modifiedBuilder.result()
       FileChanges(created, deleted, modified, unmodified)
     }
   }