From 82705f1cd5f714cbb4986e1993f85e0748d88ea0 Mon Sep 17 00:00:00 2001 From: Christian Harrington Date: Thu, 20 Aug 2026 21:52:50 +0200 Subject: [PATCH] [2.x] fix: Fixes filesystem traversal order affecting cache stability (#9646) **Problem** CompileInputs2.sources and packageBin / mappings are hashed positionally but arrive in filesystem traversal order, which differs between APFS, ext4 and overlayfs. Two machines hash an identical tree into different cache keys and cannot use each other's entries. **Solution** Sort sourcesVF by virtualized id, and mappings where packageConfigurationTask reads it. Both are where the keys are built, so nothing appended later escapes the sort. Generated-by: Claude Opus 5 --- main/src/main/scala/sbt/Defaults.scala | 5 +- .../src/sbt-test/cache/input-order/build.sbt | 72 +++++++++++++++++++ .../src/main/resources/cfg/alpha.properties | 1 + .../src/main/resources/cfg/zeta.properties | 1 + .../src/main/resources/root.properties | 1 + .../scala/com/example/alpha/Calpha1.scala | 3 + .../scala/com/example/alpha/Calpha2.scala | 3 + .../scala/com/example/alpha/Calpha3.scala | 3 + .../src/main/scala/com/example/ci/Cci1.scala | 3 + .../src/main/scala/com/example/ci/Cci2.scala | 3 + .../src/main/scala/com/example/ci/Cci3.scala | 3 + .../main/scala/com/example/lib/Clib1.scala | 3 + .../main/scala/com/example/lib/Clib2.scala | 3 + .../main/scala/com/example/lib/Clib3.scala | 3 + sbt-app/src/sbt-test/cache/input-order/test | 22 ++++++ 15 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 sbt-app/src/sbt-test/cache/input-order/build.sbt create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/alpha.properties create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/zeta.properties create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/resources/root.properties create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha1.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha2.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha3.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci1.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci2.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci3.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib1.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib2.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib3.scala create mode 100644 sbt-app/src/sbt-test/cache/input-order/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index d249a7093..2a9d17ab9 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -625,8 +625,9 @@ object Defaults extends BuildCommon with DefExtra { sources := Classpaths.concatDistinct(unmanagedSources, managedSources).value, sourcesVF := Def.uncached { val conv = fileConverter.value - sources.value.toVector.map: x => + val vs = sources.value.toVector.map: x => (conv.toVirtualFile(x.toPath()): HashedVirtualFileRef) + vs.sortBy(_.id) }, ) lazy val resourceConfigPaths = Seq( @@ -2046,7 +2047,7 @@ object Defaults extends BuildCommon with DefExtra { lazy val packageConfigurationTask: Initialize[Task[Pkg.Configuration]] = Def.task { Pkg.Configuration( - mappings.value, + mappings.value.sortBy(_._2), artifactPath.value, packageOptions.value, ) diff --git a/sbt-app/src/sbt-test/cache/input-order/build.sbt b/sbt-app/src/sbt-test/cache/input-order/build.sbt new file mode 100644 index 000000000..099c5ea10 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/build.sbt @@ -0,0 +1,72 @@ +import sbt.internal.util.CacheEventSummary + +val delClasses = taskKey[Unit]("deletes the classes directory") +val delJar = taskKey[Unit]("deletes the packaged jar") +val saveJar = taskKey[Unit]("copies the packaged jar aside for later comparison") +val checkNoMiss = taskKey[Unit]("asserts the previous command was served entirely from the cache") +val checkClasses = taskKey[Unit]("asserts class files exist") +val checkJarMatches = taskKey[Unit]("asserts the jar is byte-identical to the saved one") + +Global / localCacheDirectory := baseDirectory.value / "diskcache" + +// A distinct project id keeps this fixture's output paths from colliding with same-named +// sibling fixtures in scripted's shared batch directory. +lazy val inputOrder = project + .in(file(".")) + .settings( + scalaVersion := "3.8.4", + // Reversing stands in for a second filesystem's readdir order, which is the only part of + // this that cannot be reproduced on a single machine. + Compile / sources := { + val ss = (Compile / sources).value + if ((baseDirectory.value / "reverse.marker").exists) ss.reverse else ss + }, + Compile / packageBin / mappings := { + val ms = (Compile / packageBin / mappings).value + if ((baseDirectory.value / "reverse.marker").exists) ms.reverse else ms + }, + ) + +delClasses := Def.uncached { + val dir = (inputOrder / Compile / classDirectory).value + IO.delete(dir) + streams.value.log.info(s"deleted $dir") +} + +delJar := Def.uncached { + val conv = fileConverter.value + val jar = conv.toPath((inputOrder / Compile / packageBin / artifactPath).value).toFile() + IO.delete(jar) + streams.value.log.info(s"deleted $jar") +} + +saveJar := Def.uncached { + val conv = fileConverter.value + val jar = conv.toPath((inputOrder / Compile / packageBin).value).toFile() + IO.copyFile(jar, baseDirectory.value / "expected.jar") +} + +checkJarMatches := Def.uncached { + val conv = fileConverter.value + val jar = conv.toPath((inputOrder / Compile / packageBin).value).toFile() + val expected = baseDirectory.value / "expected.jar" + assert( + java.util.Arrays.equals(IO.readBytes(jar), IO.readBytes(expected)), + s"jar bytes changed after reordering packageBin / mappings" + ) +} + +checkNoMiss := Def.uncached { + val config = Def.cacheConfiguration.value + val prev = config.cacheEventLog.previous match + case data: CacheEventSummary.Data => data + case _ => sys.error("empty event log") + streams.value.log.info(s"prev hitCount=${prev.hitCount} missCount=${prev.missCount}") + assert(prev.missCount == 0, s"expected no cache misses but missCount=${prev.missCount}") +} + +checkClasses := Def.uncached { + val dir = (inputOrder / Compile / classDirectory).value + val classes = (dir ** "*.class").get() + assert(classes.nonEmpty, s"no class files under $dir") +} diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/alpha.properties b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/alpha.properties new file mode 100644 index 000000000..73cdb8bcc --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/alpha.properties @@ -0,0 +1 @@ +a=1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/zeta.properties b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/zeta.properties new file mode 100644 index 000000000..7fc3e8577 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/cfg/zeta.properties @@ -0,0 +1 @@ +z=1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/resources/root.properties b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/root.properties new file mode 100644 index 000000000..efe74750c --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/resources/root.properties @@ -0,0 +1 @@ +r=1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha1.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha1.scala new file mode 100644 index 000000000..fec7ed1ae --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha1.scala @@ -0,0 +1,3 @@ +package com.example.alpha + +class Calpha1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha2.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha2.scala new file mode 100644 index 000000000..409600928 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha2.scala @@ -0,0 +1,3 @@ +package com.example.alpha + +class Calpha2 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha3.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha3.scala new file mode 100644 index 000000000..ee12e0954 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/alpha/Calpha3.scala @@ -0,0 +1,3 @@ +package com.example.alpha + +class Calpha3 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci1.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci1.scala new file mode 100644 index 000000000..3357791cd --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci1.scala @@ -0,0 +1,3 @@ +package com.example.ci + +class Cci1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci2.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci2.scala new file mode 100644 index 000000000..011fab70a --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci2.scala @@ -0,0 +1,3 @@ +package com.example.ci + +class Cci2 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci3.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci3.scala new file mode 100644 index 000000000..a87b32674 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/ci/Cci3.scala @@ -0,0 +1,3 @@ +package com.example.ci + +class Cci3 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib1.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib1.scala new file mode 100644 index 000000000..e60398771 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib1.scala @@ -0,0 +1,3 @@ +package com.example.lib + +class Clib1 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib2.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib2.scala new file mode 100644 index 000000000..424f619c4 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib2.scala @@ -0,0 +1,3 @@ +package com.example.lib + +class Clib2 diff --git a/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib3.scala b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib3.scala new file mode 100644 index 000000000..3a414f624 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/src/main/scala/com/example/lib/Clib3.scala @@ -0,0 +1,3 @@ +package com.example.lib + +class Clib3 diff --git a/sbt-app/src/sbt-test/cache/input-order/test b/sbt-app/src/sbt-test/cache/input-order/test new file mode 100644 index 000000000..be57e3037 --- /dev/null +++ b/sbt-app/src/sbt-test/cache/input-order/test @@ -0,0 +1,22 @@ +# Regression for #9645: neither the traversal order of Compile / sources nor of +# packageBin / mappings may take part in a cache key. When it does, a macOS developer and a +# Linux CI runner hash an identical tree into different keys and neither can use the other's +# entries. + +# Populate the cache with both input lists in filesystem traversal order. +> compile +> packageBin +> saveJar + +# Reverse both lists. No key may move. +$ touch reverse.marker + +> delClasses +> compile +> checkNoMiss +> checkClasses + +> delJar +> packageBin +> checkNoMiss +> checkJarMatches