From 46eca942a15bb2d10ecc41bc0c47401b96b4c291 Mon Sep 17 00:00:00 2001 From: Stas Shevchenko Date: Fri, 21 Aug 2026 23:33:22 +0200 Subject: [PATCH] [2.x] fix: hash action cache inputs with full-width sha256 (#9641) mkInput folded only a 32-bit murmur hash of the task input into the cache key, so distinct inputs collide at the birthday bound (~31 per 500k realistic inputs), silently resolving a task to the wrong cached output. A new DigestHasher hashes inputs into a full-width sha256 Merkle digest. Changes all cache keys, so the cache is repopulated once. Adds a regression test. --- .../src/main/scala/sbt/util/ActionCache.scala | 7 +-- .../main/scala/sbt/util/DigestHasher.scala | 55 +++++++++++++++++++ .../test/scala/sbt/util/ActionCacheTest.scala | 26 +++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 util-cache/src/main/scala/sbt/util/DigestHasher.scala diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 1e5132063..6ed8b1559 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -36,7 +36,6 @@ import scala.annotation.{ meta, tailrec, StaticAnnotation } import scala.collection.mutable import scala.util.control.NonFatal import sjsonnew.{ HashWriter, JsonFormat } -import sjsonnew.support.murmurhash.Hasher import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter } import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes } import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef } @@ -309,8 +308,8 @@ object ActionCache: ): Digest = // Hashing serializes every task input; surface a missing input file directly rather than as an // opaque serialization failure that buries it. - val inputHash = - try Hasher.hashUnsafe[I](key) + val inputDigest = + try DigestHasher.hashUnsafe[I](key) catch case NonFatal(t) => findMissingFile(t) match @@ -320,7 +319,7 @@ object ActionCache: Digest.sha256Hash( (Vector( codeContentHash, - Digest.dummy(inputHash), + inputDigest, extraHash ) ++ { if cacheVersion == 0 then Vector.empty diff --git a/util-cache/src/main/scala/sbt/util/DigestHasher.scala b/util-cache/src/main/scala/sbt/util/DigestHasher.scala new file mode 100644 index 000000000..de307327c --- /dev/null +++ b/util-cache/src/main/scala/sbt/util/DigestHasher.scala @@ -0,0 +1,55 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.util + +import java.lang.Double as JDouble +import java.nio.charset.StandardCharsets.UTF_8 + +import sjsonnew.{ BuilderFacade, SimpleBuilderFacade, SupportHasher } + +/** + * Hashes a HashWriter input into a full-width sha256 Digest. Leaves hash their bytes under a + * per-kind tag; arrays and objects combine their child digests under their own tag, with + * objects sorted by key so the result is order-independent. Unlike the 32-bit murmur hasher, + * distinct inputs do not collide within a build-sized population. + */ +private[sbt] object DigestHasher extends SupportHasher[Digest]: + implicit val facade: BuilderFacade[Digest] = FacadeImpl + + private val arrayTag: Digest = Digest.sha256Hash(Array[Byte](6)) + private val objectTag: Digest = Digest.sha256Hash(Array[Byte](7)) + + private def tagged(tag: Byte, bytes: Array[Byte]): Digest = + Digest.sha256Hash(Array(tag) ++ bytes) + + private def longToBytes(l: Long): Array[Byte] = + val b = new Array[Byte](8) + var x = l + var i = 0 + while i < 8 do + b(i) = (x & 0xff).toByte + x >>>= 8 + i += 1 + b + + private object FacadeImpl extends SimpleBuilderFacade[Digest]: + def jnull(): Digest = tagged(0, Array.emptyByteArray) + def jfalse(): Digest = tagged(1, Array.emptyByteArray) + def jtrue(): Digest = tagged(2, Array.emptyByteArray) + def jint(i: Int): Digest = jlong(i.toLong) + def jlong(l: Long): Digest = tagged(3, longToBytes(l)) + def jdouble(d: Double): Digest = tagged(4, longToBytes(JDouble.doubleToRawLongBits(d))) + def jnumstring(s: String): Digest = jstring(s) + def jintstring(s: String): Digest = jstring(s) + def jbigdecimal(d: BigDecimal): Digest = jstring(d.toString) + def jstring(s: String): Digest = tagged(5, s.getBytes(UTF_8)) + def jarray(vs: List[Digest]): Digest = Digest.sha256Hash((arrayTag +: vs)*) + def jobject(vs: Map[String, Digest]): Digest = + val sorted = vs.toSeq.sortBy(_._1).flatMap((k, v) => Seq(jstring(k), v)) + Digest.sha256Hash((objectTag +: sorted)*) diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index ebc759377..c0c6fe9e2 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -44,6 +44,32 @@ object ActionCacheTest extends BasicTestSuite: val chain = new RuntimeException("boom", new IllegalStateException("unrelated")) assert(ActionCache.findMissingFile(chain) == None) + test("Distinct inputs that collide in the 32-bit murmur hash get distinct cache keys"): + import sjsonnew.BasicJsonProtocol.given + import sjsonnew.support.murmurhash.Hasher + // Find two distinct inputs whose 32-bit murmur hash collides (the old mkInput folded only + // that 32-bit value into the key, so these used to produce an identical cache key). The + // cache key must now distinguish them. + val seen = scala.collection.mutable.HashMap.empty[Int, String] + var a: String = null + var b: String = null + var i = 0 + val cap = 5000000 + while (b == null && i < cap) do + val key = s"input-$i" + Hasher.hashUnsafe[String](key) match + case h if seen.contains(h) => a = seen(h); b = key + case h => seen.update(h, key) + i += 1 + assert(b != null, s"no 32-bit collision found within $cap inputs") + assert(a != b) + val ka = ActionCache.mkInput(a, Digest.zero, Digest.zero, 0L) + val kb = ActionCache.mkInput(b, Digest.zero, Digest.zero, 0L) + assert( + ka != kb, + s"distinct inputs '$a' and '$b' must not share a cache key, both hashed to $ka" + ) + test("Disk cache can hold a blob"): withDiskCache(testHoldBlob)