[2.x] fix: hash action cache inputs with full-width sha256 (#9641)

mkInput folded only a 32-bit murmur hash of the task input into the cache key, so distinct inputs collide at the birthday bound (~31 per 500k realistic inputs), silently resolving a task to the wrong cached output. A new DigestHasher hashes inputs into a full-width sha256 Merkle digest. Changes all cache keys, so the cache is repopulated once. Adds a regression test.

util-cache/src/main/scala/sbt/util/ActionCache.scala

@@ -36,7 +36,6 @@ import scala.annotation.{ meta, tailrec, StaticAnnotation }
 import scala.collection.mutable
 import scala.util.control.NonFatal
 import sjsonnew.{ HashWriter, JsonFormat }
-import sjsonnew.support.murmurhash.Hasher
 import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser, PrettyPrinter }
 import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
 import xsbti.{ CompileFailed, FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
@@ -309,8 +308,8 @@ object ActionCache:
   ): Digest =
     // Hashing serializes every task input; surface a missing input file directly rather than as an
     // opaque serialization failure that buries it.
-    val inputHash =
-      try Hasher.hashUnsafe[I](key)
+    val inputDigest =
+      try DigestHasher.hashUnsafe[I](key)
       catch
         case NonFatal(t) =>
           findMissingFile(t) match
@@ -320,7 +319,7 @@ object ActionCache:
     Digest.sha256Hash(
       (Vector(
         codeContentHash,
-        Digest.dummy(inputHash),
+        inputDigest,
         extraHash
       ) ++ {
         if cacheVersion == 0 then Vector.empty

util-cache/src/main/scala/sbt/util/DigestHasher.scala

@@ -0,0 +1,55 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.util
+
+import java.lang.Double as JDouble
+import java.nio.charset.StandardCharsets.UTF_8
+
+import sjsonnew.{ BuilderFacade, SimpleBuilderFacade, SupportHasher }
+
+/**
+ * Hashes a HashWriter input into a full-width sha256 Digest. Leaves hash their bytes under a
+ * per-kind tag; arrays and objects combine their child digests under their own tag, with
+ * objects sorted by key so the result is order-independent. Unlike the 32-bit murmur hasher,
+ * distinct inputs do not collide within a build-sized population.
+ */
+private[sbt] object DigestHasher extends SupportHasher[Digest]:
+  implicit val facade: BuilderFacade[Digest] = FacadeImpl
+
+  private val arrayTag: Digest = Digest.sha256Hash(Array[Byte](6))
+  private val objectTag: Digest = Digest.sha256Hash(Array[Byte](7))
+
+  private def tagged(tag: Byte, bytes: Array[Byte]): Digest =
+    Digest.sha256Hash(Array(tag) ++ bytes)
+
+  private def longToBytes(l: Long): Array[Byte] =
+    val b = new Array[Byte](8)
+    var x = l
+    var i = 0
+    while i < 8 do
+      b(i) = (x & 0xff).toByte
+      x >>>= 8
+      i += 1
+    b
+
+  private object FacadeImpl extends SimpleBuilderFacade[Digest]:
+    def jnull(): Digest = tagged(0, Array.emptyByteArray)
+    def jfalse(): Digest = tagged(1, Array.emptyByteArray)
+    def jtrue(): Digest = tagged(2, Array.emptyByteArray)
+    def jint(i: Int): Digest = jlong(i.toLong)
+    def jlong(l: Long): Digest = tagged(3, longToBytes(l))
+    def jdouble(d: Double): Digest = tagged(4, longToBytes(JDouble.doubleToRawLongBits(d)))
+    def jnumstring(s: String): Digest = jstring(s)
+    def jintstring(s: String): Digest = jstring(s)
+    def jbigdecimal(d: BigDecimal): Digest = jstring(d.toString)
+    def jstring(s: String): Digest = tagged(5, s.getBytes(UTF_8))
+    def jarray(vs: List[Digest]): Digest = Digest.sha256Hash((arrayTag +: vs)*)
+    def jobject(vs: Map[String, Digest]): Digest =
+      val sorted = vs.toSeq.sortBy(_._1).flatMap((k, v) => Seq(jstring(k), v))
+      Digest.sha256Hash((objectTag +: sorted)*)

util-cache/src/test/scala/sbt/util/ActionCacheTest.scala

@@ -44,6 +44,32 @@ object ActionCacheTest extends BasicTestSuite:
     val chain = new RuntimeException("boom", new IllegalStateException("unrelated"))
     assert(ActionCache.findMissingFile(chain) == None)
 
+  test("Distinct inputs that collide in the 32-bit murmur hash get distinct cache keys"):
+    import sjsonnew.BasicJsonProtocol.given
+    import sjsonnew.support.murmurhash.Hasher
+    // Find two distinct inputs whose 32-bit murmur hash collides (the old mkInput folded only
+    // that 32-bit value into the key, so these used to produce an identical cache key). The
+    // cache key must now distinguish them.
+    val seen = scala.collection.mutable.HashMap.empty[Int, String]
+    var a: String = null
+    var b: String = null
+    var i = 0
+    val cap = 5000000
+    while (b == null && i < cap) do
+      val key = s"input-$i"
+      Hasher.hashUnsafe[String](key) match
+        case h if seen.contains(h) => a = seen(h); b = key
+        case h                     => seen.update(h, key)
+      i += 1
+    assert(b != null, s"no 32-bit collision found within $cap inputs")
+    assert(a != b)
+    val ka = ActionCache.mkInput(a, Digest.zero, Digest.zero, 0L)
+    val kb = ActionCache.mkInput(b, Digest.zero, Digest.zero, 0L)
+    assert(
+      ka != kb,
+      s"distinct inputs '$a' and '$b' must not share a cache key, both hashed to $ka"
+    )
+
   test("Disk cache can hold a blob"):
     withDiskCache(testHoldBlob)