Migrate FarmHash usage to xxhash64

This commit is contained in:
Eugene Yokota 2026-05-26 03:31:06 -04:00
parent 47c73b701d
commit 07efae45a6
7 changed files with 25 additions and 24 deletions

View File

@ -390,6 +390,9 @@ lazy val utilCache = project
contrabandSettings,
mimaSettings,
mimaBinaryIssueFilters ++= Seq(
exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.farmHash"),
exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.farmHashStr"),
exclude[DirectMissingMethodProblem]("sbt.util.HashUtil.toFarmHashString"),
),
Test / fork := true,
)

View File

@ -28,7 +28,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import net.openhft.hashing.LongHashFunction;
import org.scalasbt.ipcsocket.UnixDomainServerSocket;
import org.scalasbt.ipcsocket.Win32NamedPipeServerSocket;
import org.scalasbt.ipcsocket.Win32SecurityLevel;
@ -303,7 +302,9 @@ public class BootServerSocket implements AutoCloseable {
public static String socketLocation(final Path base)
throws UnsupportedEncodingException, IOException {
final Path target = base.resolve("project").resolve("target");
long hash = LongHashFunction.farmNa().hashBytes(target.toString().getBytes("UTF-8"));
long hash =
((long) target.toString().hashCode() << 32)
| (target.toString().length() * 31 & 0xffffffffL);
if (isWindows) {
return "sbt-load" + hash;
} else {

View File

@ -7,7 +7,7 @@ import xsbti.{ BasicVirtualFileRef, VirtualFile }
case class StringVirtualFile1(path: String, content: String)
extends BasicVirtualFileRef(path)
with VirtualFile:
override def contentHash: Long = HashUtil.farmHash(content.getBytes("UTF-8"))
override def contentHash: Long = HashUtil.xxhash64(content.getBytes("UTF-8"))
override def sizeBytes: Long = content.getBytes("UTF-8").size
override def contentHashStr: String =
import Digest.*

View File

@ -89,8 +89,12 @@ object Digest:
def imoxx64Hash(path: Path): Digest = apply(Imoxx64, path)
def imowy64Hash(path: Path): Digest = apply(Imowy64, path)
def xx64Hash(path: Path): Digest = apply(Xx64, path)
def wy64Hash(path: Path): Digest = apply(Wy64, path)
private[sbt] def md5Hash(bytes: Array[Byte]): Digest =
apply(Md5, hashBytes(Md5, bytes), bytes.length)

View File

@ -1,24 +1,16 @@
package sbt.util
import java.nio.file.{ Files, Path }
import net.openhft.hashing.LongHashFunction
import java.nio.file.{ Path as NioPath }
import sbt.internal.util.hashing.Hashing
object HashUtil:
private[sbt] def farmHash(bytes: Array[Byte]): Long =
LongHashFunction.farmNa().hashBytes(bytes)
private[sbt] def xxhash64(bytes: Array[Byte]): Long =
Hashing.xxhash64.hash(bytes, 0, bytes.size, 0)
private[sbt] def farmHash(path: Path): Long =
import sbt.io.Hash
// allocating many byte arrays for large files may lead to OOME
// but it is more efficient for small files
val largeFileLimit = 10 * 1024 * 1024
private[sbt] def imohash64(path: NioPath): Long =
val hash64 = Hashing.samplingFileHashWyHash64(0)
hash64.hash(path)
if Files.size(path) < largeFileLimit then farmHash(Files.readAllBytes(path))
else farmHash(Hash(path.toFile))
private[sbt] def farmHashStr(path: Path): String =
"farm64-" + farmHash(path).toHexString
private[sbt] def toFarmHashString(digest: Long): String =
s"farm64-${digest.toHexString}"
private[sbt] def imohash64Str(path: NioPath): String =
"imoxx64-" + imohash64(path).toHexString
end HashUtil

View File

@ -24,11 +24,11 @@ object StringStrings:
given Conversion[HashedVirtualFileRef, StringString] =
(x: HashedVirtualFileRef) => StringString(x.id, x.contentHashStr)
given Conversion[File, StringString] =
(x: File) => StringString(x.toString(), HashUtil.farmHashStr(x.toPath()))
(x: File) => StringString(x.toString(), HashUtil.imohash64Str(x.toPath()))
given Conversion[Path, StringString] =
(x: Path) => StringString(x.toString(), HashUtil.farmHashStr(x))
(x: Path) => StringString(x.toString(), HashUtil.imohash64Str(x))
given Conversion[VirtualFile, StringString] =
(x: VirtualFile) => StringString(x.id, s"farm64-${x.contentHash.toHexString}")
(x: VirtualFile) => StringString(x.id, s"xx64-${x.contentHash.toHexString}")
given HashWriter[StringString] = new HashWriter[StringString]:
def write[J](obj: StringString, builder: Builder[J]): Unit =

View File

@ -11,6 +11,7 @@ object HasherTest extends BasicTestSuite:
final val blankContentHash = -7286425919675154353L
val blankContentHashStr = "farm64-9ae16a3b2f90404f"
final val blankATxtHash = 1166939303L
final val blankATxtXX64 = -541480681L
test("The IntJsonFormat should convert an Int to an int hash") {
import BasicJsonProtocol.given
@ -36,7 +37,7 @@ object HasherTest extends BasicTestSuite:
import PathHashWriters.given
val x = StringVirtualFile1("a.txt", "")
val actual = Hasher.hashUnsafe(x)
assert(actual == blankATxtHash)
assert(actual == blankATxtXX64)
}
test("tuple") {