diff --git a/notes/2.0.0/dirzip-restore-on-hit.md b/notes/2.0.0/dirzip-restore-on-hit.md new file mode 100644 index 000000000..975d0f48b --- /dev/null +++ b/notes/2.0.0/dirzip-restore-on-hit.md @@ -0,0 +1,15 @@ +### Directories declared with `Def.declareOutputDirectory` are restored on cache hits + +A directory declared as a cached task's output is packaged as a `.sbtdir.zip` +sibling of the directory. Deleting the directory leaves the sibling zip behind +(an `rm -rf` of the directory or of `classes/` does exactly this), which left +the cache convinced everything was in sync: on the next cache hit the task did not re-run, but the +directory was never re-extracted either. For sbt's own `compile`, whose classes +directory is declared this way, a deleted output directory plus a warm cache +meant `run` failed with `ClassNotFoundException` and no recompile. The cache now +re-extracts a declared directory when the directory itself is missing, at the +cost of a single stat on the warm path. + +This addresses the directory-restoration half of [#9462][i9462]. + +[i9462]: https://github.com/sbt/sbt/issues/9462 diff --git a/sbt-app/src/sbt-test/cache/compile-classes-restore/Main.scala b/sbt-app/src/sbt-test/cache/compile-classes-restore/Main.scala new file mode 100644 index 000000000..368364e2b --- /dev/null +++ b/sbt-app/src/sbt-test/cache/compile-classes-restore/Main.scala @@ -0,0 +1,2 @@ +@main def main(args: String*) = + println("hello from probe") diff --git a/sbt-app/src/sbt-test/cache/compile-classes-restore/build.sbt b/sbt-app/src/sbt-test/cache/compile-classes-restore/build.sbt new file mode 100644 index 000000000..3335d1b29 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/compile-classes-restore/build.sbt @@ -0,0 +1,53 @@ +import sbt.internal.util.CacheEventSummary + +val checkCompileHit = taskKey[Unit]("asserts the previous command took cache hits") +val delClasses = taskKey[Unit]("deletes the classes directory") +val delClassesZip = taskKey[Unit]("deletes the sibling classes.sbtdir.zip") +val checkClasses = taskKey[Unit]("asserts .class files exist") +val checkNoClasses = taskKey[Unit]("asserts no .class files exist") + +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +// A distinct project id keeps this fixture's output paths from colliding with same-named +// sibling fixtures in scripted's shared batch directory. +lazy val compileClassesRestore = project + .in(file(".")) + .settings( + scalaVersion := "3.8.4" + ) + +delClasses := Def.uncached { + val dir = (Compile / classDirectory).value + IO.delete(dir) + streams.value.log.info(s"deleted $dir") +} + +delClassesZip := Def.uncached { + val dir = (Compile / classDirectory).value + val zip = new java.io.File(dir.getParentFile, dir.getName + ".sbtdir.zip") + streams.value.log.info(s"deleting $zip (exists=${zip.exists})") + IO.delete(zip) +} + +checkClasses := Def.uncached { + val dir = (Compile / classDirectory).value + val classes = (dir ** "*.class").get() + streams.value.log.info(s"classes under $dir: ${classes.mkString(", ")}") + assert(classes.nonEmpty, s"no class files under $dir") +} + +checkNoClasses := Def.uncached { + val dir = (Compile / classDirectory).value + val classes = (dir ** "*.class").get() + streams.value.log.info(s"classes under $dir: ${classes.mkString(", ")}") + assert(classes.isEmpty, s"unexpected class files under $dir: ${classes.mkString(", ")}") +} + +checkCompileHit := Def.uncached { + val config = Def.cacheConfiguration.value + val prev = config.cacheEventLog.previous match + case s: CacheEventSummary.Data => s + case _ => sys.error("empty event log") + streams.value.log.info(s"prev hitCount=${prev.hitCount} missCount=${prev.missCount}") + assert(prev.hitCount >= 1, s"expected cache hits but hitCount=${prev.hitCount}") +} diff --git a/sbt-app/src/sbt-test/cache/compile-classes-restore/test b/sbt-app/src/sbt-test/cache/compile-classes-restore/test new file mode 100644 index 000000000..1ddc3bc71 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/compile-classes-restore/test @@ -0,0 +1,18 @@ +# Regression for #9462: compile's classes dir (declared via Def.declareOutputDirectory) +# must be restored on a cache hit after the classes directory is deleted. +> compile +> checkClasses +> run +> delClasses +> checkNoClasses +> compile +> checkCompileHit +> checkClasses +> run + +# restoration also works when the sibling classes.sbtdir.zip is gone too +> delClasses +> delClassesZip +> compile +> checkClasses +> run diff --git a/sbt-app/src/sbt-test/cache/declare-output-dir-restore/build.sbt b/sbt-app/src/sbt-test/cache/declare-output-dir-restore/build.sbt new file mode 100644 index 000000000..2b3fa4f6c --- /dev/null +++ b/sbt-app/src/sbt-test/cache/declare-output-dir-restore/build.sbt @@ -0,0 +1,60 @@ +import sbt.internal.util.CacheEventSummary +import xsbti.HashedVirtualFileRef + +val declareDir = taskKey[HashedVirtualFileRef]("writes 2 files into a dir and declares the dir") +val checkFiles = taskKey[Unit]("asserts both files exist") +val checkGone = taskKey[Unit]("asserts the dir has no files") +val delDir = taskKey[Unit]("deletes the generated dir") +val delZip = taskKey[Unit]("deletes the sibling .sbtdir.zip") +val checkHit = taskKey[Unit]("asserts previous command was a pure cache hit") + +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +lazy val declareOutputDirRestore = project.in(file(".")) + +declareDir := { + val log = streams.value.log + val dir = target.value / "gen-dir" + IO.createDirectory(dir) + IO.write(dir / "a.txt", "contents A") + IO.write(dir / "b.txt", "contents B") + log.info(s"COMPUTED declareDir (cache miss)") + val vf = fileConverter.value.toVirtualFile(dir.toPath) + Def.declareOutputDirectory(vf) +} + +checkFiles := Def.uncached { + val log = streams.value.log + val dir = target.value / "gen-dir" + val listing = if (dir.exists) (dir ** "*").get().mkString(", ") else "