diff --git a/internal/hash-benchmark/src/main/scala/sbt/internal/util/HashBenchmark.scala b/internal/hash-benchmark/src/main/scala/sbt/internal/util/HashBenchmark.scala index 5f19269f2..165862816 100644 --- a/internal/hash-benchmark/src/main/scala/sbt/internal/util/HashBenchmark.scala +++ b/internal/hash-benchmark/src/main/scala/sbt/internal/util/HashBenchmark.scala @@ -24,14 +24,14 @@ end AbstractHashBenchmark class XXHash64HashBenchmark extends AbstractHashBenchmark: override def hash(buf: Array[Byte]): String = - val h = Hashing.xxhash64 - val hash = h.hash(buf, 0, buf.size, 0) + val h = Hashing.xxhash64(0L) + val hash = h.hash(buf, 0, buf.size) java.lang.Long.toHexString(hash) class WyHash64HashBenchmark extends AbstractHashBenchmark: override def hash(buf: Array[Byte]): String = - val h = Hashing.wyhash64 - val hash = h.hash(buf, 0, buf.size, 0) + val h = Hashing.wyhash64(0L) + val hash = h.hash(buf, 0, buf.size) java.lang.Long.toHexString(hash) class FarmHashHashBenchmark extends AbstractHashBenchmark: diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/VarHandleUtils.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/Access.scala similarity index 50% rename from internal/util-control/src/main/scala/sbt/internal/util/hashing/VarHandleUtils.scala rename to internal/util-control/src/main/scala/sbt/internal/util/hashing/Access.scala index bcdbb5847..0eda97575 100644 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/VarHandleUtils.scala +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/Access.scala @@ -12,7 +12,13 @@ package sbt.internal.util.hashing import java.lang.invoke.{ MethodHandles, VarHandle } import java.nio.{ ByteBuffer, ByteOrder } -object VarHandleUtils: +sealed trait Access[A1]: + def readByte(a: A1, off: Int): Byte + def readIntLE(a: A1, off: Int): Int + def readLongLE(a: A1, off: Int): Long +end Access + +object Access: private def getArrayClass(c: Class[?]): Class[?] = java.lang.reflect.Array.newInstance(c, 0).getClass private val LONG_HANDLE: VarHandle = @@ -24,18 +30,21 @@ object VarHandleUtils: private val BB_INT_HANDLE: VarHandle = MethodHandles.byteBufferViewVarHandle(getArrayClass(classOf[Int]), ByteOrder.LITTLE_ENDIAN) - inline def readByte(buf: Array[Byte], off: Int): Byte = - buf(off) - inline def readIntLE(buf: Array[Byte], off: Int): Int = - INT_HANDLE.get(buf, off).asInstanceOf[Int] - inline def readLongLE(buf: Array[Byte], off: Int): Long = - LONG_HANDLE.get(buf, off).asInstanceOf[Long] - inline def readByte(buf: ByteBuffer, i: Int): Byte = - buf.get(i) - inline def readIntLE(buf: ByteBuffer, i: Int): Int = - assert(buf.order() == ByteOrder.LITTLE_ENDIAN) - BB_INT_HANDLE.get(buf, i).asInstanceOf[Int] - inline def readLongLE(buf: ByteBuffer, i: Int): Long = - assert(buf.order() == ByteOrder.LITTLE_ENDIAN) - BB_LONG_HANDLE.get(buf, i).asInstanceOf[Long] -end VarHandleUtils + given Access[Array[Byte]]: + inline def readByte(buf: Array[Byte], off: Int): Byte = + buf(off) + inline def readIntLE(buf: Array[Byte], off: Int): Int = + INT_HANDLE.get(buf, off).asInstanceOf[Int] + inline def readLongLE(buf: Array[Byte], off: Int): Long = + LONG_HANDLE.get(buf, off).asInstanceOf[Long] + + given Access[ByteBuffer]: + inline def readByte(buf: ByteBuffer, off: Int): Byte = + buf.get(off) + inline def readIntLE(buf: ByteBuffer, off: Int): Int = + assert(buf.order() == ByteOrder.LITTLE_ENDIAN) + BB_INT_HANDLE.get(buf, off).asInstanceOf[Int] + inline def readLongLE(buf: ByteBuffer, off: Int): Long = + assert(buf.order() == ByteOrder.LITTLE_ENDIAN) + BB_LONG_HANDLE.get(buf, off).asInstanceOf[Long] +end Access diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/HashAlgo.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/HashAlgo.scala index c3fb0c3f4..6db43425e 100644 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/HashAlgo.scala +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/HashAlgo.scala @@ -9,12 +9,15 @@ package sbt.internal.util.hashing -import java.nio.ByteBuffer +// import java.nio.ByteBuffer + +import scala.annotation.nowarn /** * Hash algorithm interface */ -trait HashAlgo: +@nowarn +trait HashAlgo[A1: Access]: /** * Computes the 64-bits hash of buf[off:off+len] using the seed. @@ -25,34 +28,7 @@ trait HashAlgo: * @param seed the seed to use * @return the hash value */ - def hash(buf: Array[Byte], off: Int, len: Int, seed: Long): Long - - /** - * Computes the hash of the given slice of the ByteBuffer. - * ByteBuffer#position() position and ByteBuffer#limit() limit - * are not modified. - * - * @param buf the input data - * @param off the start offset in buf - * @param len the number of bytes to hash - * @param seed the seed to use - * @return the hash value - */ - def hash(buf: ByteBuffer, off: Int, len: Int, seed: Long): Long - - /** - * Computes the hash of the given ByteBuffer. The - * ByteBuffer#position() position is moved in order to reflect bytes - * which have been read. - * - * @param buf the input data - * @param seed the seed to use - * @return the hash value - */ - def hash(buf: ByteBuffer, seed: Long): Long = - val r = hash(buf, buf.position(), buf.remaining(), seed) - buf.position(buf.limit()) - r + def hash(buf: A1, off: Int, len: Int): Long override def toString(): String = getClass().getSimpleName() diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/Hashing.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/Hashing.scala index 781e21948..f7670d3b5 100644 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/Hashing.scala +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/Hashing.scala @@ -9,16 +9,30 @@ package sbt.internal.util.hashing +import java.nio.ByteBuffer + object Hashing: - def xxhash64: HashAlgo = XXHash64VarHandle.INSTANCE - def wyhash64: HashAlgo = WyHash64VarHandle.INSTANCE + def xxhash64(seed: Long): HashAlgo[Array[Byte]] = + XXHash64.byteArray(seed) + + def xxhash64ByteBuffer(seed: Long): HashAlgo[ByteBuffer] = + XXHash64.byteBuffer(seed) + + def wyhash64(seed: Long): HashAlgo[Array[Byte]] = + WyHash64.byteArray(seed) + + def wyhash64ByteBuffer(seed: Long): HashAlgo[ByteBuffer] = + WyHash64.byteBuffer(seed) def newStreamingXXHash64(seed: Long): StreamingHashAlgo = new StreamingXXHash64VarHandle(seed) + def newStreamingWyHash64(seed: Long): StreamingHashAlgo = new StreamingWyHash64VarHandle(seed) + def samplingFileHashXXHash64(seed: Long): FileHash = FileSampleHash(newStreamingXXHash64(seed)) + def samplingFileHashWyHash64(seed: Long): FileHash = FileSampleHash(newStreamingWyHash64(seed)) end Hashing diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingWyHash64VarHandle.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingWyHash64VarHandle.scala index 8130c5ce7..8b256cc21 100644 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingWyHash64VarHandle.scala +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingWyHash64VarHandle.scala @@ -9,9 +9,8 @@ package sbt.internal.util.hashing -import WyHash64VarHandle.* +import WyHash64.* import WyHashConstants.* -import VarHandleUtils.* class StreamingWyHash64VarHandle(seed: Long) extends StreamingHashAlgo(seed): protected var a: Long = 0 @@ -23,6 +22,7 @@ class StreamingWyHash64VarHandle(seed: Long) extends StreamingHashAlgo(seed): protected var totalLen: Long = 0L protected val memory = new Array[Byte](48) protected var memoryLen: Int = 0 + private val access = summon[Access[Array[Byte]]] reset() override def reset(): Unit = @@ -47,10 +47,10 @@ class StreamingWyHash64VarHandle(seed: Long) extends StreamingHashAlgo(seed): if inputLen >= 4 then val end = inputLen - 4 val quarter = (inputLen >> 3) << 2 - _a = (readIntLE(input, 0).toLong << 32) - | (readIntLE(input, quarter) & 0xffffffffL) - _b = (readIntLE(input, end) << 32).toLong - | (readIntLE(input, end - quarter) & 0xffffffffL) + _a = (access.readIntLE(input, 0).toLong << 32) + | (access.readIntLE(input, quarter) & 0xffffffffL) + _b = (access.readIntLE(input, end) << 32).toLong + | (access.readIntLE(input, end - quarter) & 0xffffffffL) else if inputLen > 0 then _a = ((input(0) & 0xffL) << 16) | ((input(inputLen >> 1) & 0xffL) << 8) | (input(inputLen - 1) & 0xffL) @@ -73,11 +73,11 @@ class StreamingWyHash64VarHandle(seed: Long) extends StreamingHashAlgo(seed): var i = 0 while i + 16 < inputLen do - v0 = mix(readLongLE(input, i) ^ PRIME64_1, readLongLE(input, i + 8) ^ v0) + v0 = mix(access.readLongLE(input, i) ^ PRIME64_1, access.readLongLE(input, i + 8) ^ v0) i += 16 - _a = readLongLE(input, inputLen - 16) - _b = readLongLE(input, inputLen - 8) + _a = access.readLongLE(input, inputLen - 16) + _b = access.readLongLE(input, inputLen - 8) end if finishHash(_a, _b, v0, this.totalLen) @@ -113,8 +113,10 @@ class StreamingWyHash64VarHandle(seed: Long) extends StreamingHashAlgo(seed): end update private def round(buf: Array[Byte], p: Int): Unit = - this.v0 = mix(readLongLE(buf, p) ^ PRIME64_1, readLongLE(buf, p + 8) ^ this.v0) - this.v1 = mix(readLongLE(buf, p + 16) ^ PRIME64_2, readLongLE(buf, p + 24) ^ this.v1) - this.v2 = mix(readLongLE(buf, p + 32) ^ PRIME64_3, readLongLE(buf, p + 40) ^ this.v2) + this.v0 = mix(access.readLongLE(buf, p) ^ PRIME64_1, access.readLongLE(buf, p + 8) ^ this.v0) + this.v1 = + mix(access.readLongLE(buf, p + 16) ^ PRIME64_2, access.readLongLE(buf, p + 24) ^ this.v1) + this.v2 = + mix(access.readLongLE(buf, p + 32) ^ PRIME64_3, access.readLongLE(buf, p + 40) ^ this.v2) end StreamingWyHash64VarHandle diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingXXHash64VarHandle.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingXXHash64VarHandle.scala index fe35fe2cb..965dc5898 100644 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingXXHash64VarHandle.scala +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/StreamingXXHash64VarHandle.scala @@ -11,7 +11,6 @@ package sbt.internal.util.hashing import java.lang.Long.rotateLeft import SafeUtils.checkRange -import VarHandleUtils.* import XXHashConstants.* /** @@ -22,6 +21,7 @@ import XXHashConstants.* * Streaming xxhash. */ class StreamingXXHash64VarHandle(seed: Long) extends AbstractStreamingXXHash64Scala(seed): + private val access = summon[Access[Array[Byte]]] override def getValue: Long = var h64: Long = 0L @@ -61,7 +61,7 @@ class StreamingXXHash64VarHandle(seed: Long) extends AbstractStreamingXXHash64Sc var off: Int = 0 while off <= memSize - 8 do - var k1: Long = readLongLE(memory, off) + var k1: Long = access.readLongLE(memory, off) k1 *= PRIME64_2 k1 = rotateLeft(k1, 31) k1 *= PRIME64_1 @@ -70,7 +70,7 @@ class StreamingXXHash64VarHandle(seed: Long) extends AbstractStreamingXXHash64Sc off += 8 if off <= memSize - 4 then - h64 ^= (readIntLE(memory, off) & 0xffffffffL) * PRIME64_1 + h64 ^= (access.readIntLE(memory, off) & 0xffffffffL) * PRIME64_1 h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3 off += 4 else () @@ -104,19 +104,19 @@ class StreamingXXHash64VarHandle(seed: Long) extends AbstractStreamingXXHash64Sc if memSize > 0 then // data left from previous update System.arraycopy(buf, off, memory, memSize, 32 - memSize) - v1 += readLongLE(memory, 0) * PRIME64_2 + v1 += access.readLongLE(memory, 0) * PRIME64_2 v1 = rotateLeft(v1, 31) v1 *= PRIME64_1 - v2 += readLongLE(memory, 8) * PRIME64_2 + v2 += access.readLongLE(memory, 8) * PRIME64_2 v2 = rotateLeft(v2, 31) v2 *= PRIME64_1 - v3 += readLongLE(memory, 16) * PRIME64_2 + v3 += access.readLongLE(memory, 16) * PRIME64_2 v3 = rotateLeft(v3, 31) v3 *= PRIME64_1 - v4 += readLongLE(memory, 24) * PRIME64_2 + v4 += access.readLongLE(memory, 24) * PRIME64_2 v4 = rotateLeft(v4, 31) v4 *= PRIME64_1 @@ -132,22 +132,22 @@ class StreamingXXHash64VarHandle(seed: Long) extends AbstractStreamingXXHash64Sc var v4: Long = this.v4 while off <= limit do - v1 += readLongLE(buf, off) * PRIME64_2 + v1 += access.readLongLE(buf, off) * PRIME64_2 v1 = rotateLeft(v1, 31) v1 *= PRIME64_1 off += 8 - v2 += readLongLE(buf, off) * PRIME64_2 + v2 += access.readLongLE(buf, off) * PRIME64_2 v2 = rotateLeft(v2, 31) v2 *= PRIME64_1 off += 8 - v3 += readLongLE(buf, off) * PRIME64_2 + v3 += access.readLongLE(buf, off) * PRIME64_2 v3 = rotateLeft(v3, 31) v3 *= PRIME64_1 off += 8 - v4 += readLongLE(buf, off) * PRIME64_2 + v4 += access.readLongLE(buf, off) * PRIME64_2 v4 = rotateLeft(v4, 31) v4 *= PRIME64_1 off += 8 diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64.scala new file mode 100644 index 000000000..7d4ddaae4 --- /dev/null +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64.scala @@ -0,0 +1,168 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + * + */ + +package sbt.internal.util.hashing + +import java.nio.ByteBuffer +import WyHashConstants.* + +object WyHash64: + private lazy val arrayInstance: WyHash64[Array[Byte]] = + new WyHash64(0) + private lazy val byteBufferInstance: WyHash64[ByteBuffer] = + new WyHash64(0) + + def byteArray(seed: Long): WyHash64[Array[Byte]] = + if seed == 0L then arrayInstance + else new WyHash64(seed) + + def byteBuffer(seed: Long): WyHash64[ByteBuffer] = + if seed == 0L then byteBufferInstance + else new WyHash64(seed) + + private[hashing] inline def initSeed(seed: Long): Long = + seed ^ mix(seed ^ PRIME64_0, PRIME64_1) + + private[hashing] def mix(a: Long, b: Long): Long = + val low = a * b + val high = unsignedMultiplyHigh(a, b) + low ^ high + + private[hashing] inline def unsignedMultiplyHigh(a: Long, b: Long): Long = + Math.multiplyHigh(a, b) + ((a >> 63) & b) + ((b >> 63) & a) + + private[hashing] inline def wyr3[A1: Access](buf: A1, off: Int, k: Int): Long = + val access = summon[Access[A1]] + ((access.readByte(buf, off) & 0xffL) << 16) + | ((access.readByte(buf, off + (k >> 1)) & 0xffL) << 8) + | (access.readByte(buf, off + k - 1) & 0xffL) + + private[hashing] inline def finishHash(a: Long, b: Long, seed: Long, len: Long): Long = + val _a = a ^ PRIME64_1 + val _b = b ^ seed + val low = _a * _b + val high = unsignedMultiplyHigh(_a, _b) + mix(low ^ PRIME64_0 ^ len, high ^ PRIME64_1) + +end WyHash64 + +/** + * Wyhash matching Zig 0.15 std.hash.Wyhash. + */ +class WyHash64[A1: Access](seed: Long) extends HashAlgo[A1]: + import WyHash64.* + + private val access: Access[A1] = summon[Access[A1]] + + override def hash(buf: A1, offset: Int, len: Int): Long = + var off = offset + var s: Long = initSeed(seed) + val secret1 = PRIME64_1 + val secret2 = PRIME64_2 + val secret3 = PRIME64_3 + var a: Long = 0L + var b: Long = 0L + + if len <= 16 then + if len >= 4 then + a = (access.readIntLE(buf, off).toLong << 32) + | (access.readIntLE(buf, off + ((len >> 3) << 2)) & 0xffffffffL) + b = (access.readIntLE(buf, off + len - 4).toLong << 32) + | (access.readIntLE(buf, off + len - 4 - ((len >> 3) << 2)) & 0xffffffffL) + else if len > 0 then + a = wyr3(buf, off, len) + b = 0 + else + a = 0 + b = 0 + else + var i = len + var p = off + var see0 = s + var see1 = s + var see2 = s + + while i > 48 do + see0 = mix(access.readLongLE(buf, p) ^ secret1, access.readLongLE(buf, p + 8) ^ see0) + see1 = mix(access.readLongLE(buf, p + 16) ^ secret2, access.readLongLE(buf, p + 24) ^ see1) + see2 = mix(access.readLongLE(buf, p + 32) ^ secret3, access.readLongLE(buf, p + 40) ^ see2) + p += 48 + i -= 48 + end while + + see0 ^= see1 ^ see2 + while i > 16 do + see0 = mix(access.readLongLE(buf, p) ^ secret1, access.readLongLE(buf, p + 8) ^ see0) + i -= 16 + p += 16 + end while + + a = access.readLongLE(buf, off + len - 16) + b = access.readLongLE(buf, off + len - 8) + s = see0 + end if + finishHash(a, b, s, len) + end hash + + // override def hash(buffer: ByteBuffer, offset: Int, len: Int, seed: Long): Long = + // if buffer.hasArray() then hash(buffer.array(), offset + buffer.arrayOffset(), len, seed) + // else + // var off = offset + // ByteBufferUtils.checkRange(buffer, off, len) + // val buf = ByteBufferUtils.inLittleEndianOrder(buffer) + // var s: Long = initSeed(seed) + // val secret1 = PRIME64_1 + // val secret2 = PRIME64_2 + // val secret3 = PRIME64_3 + // var a: Long = 0L + // var b: Long = 0L + + // if len <= 16 then + // if len >= 4 then + // a = (readIntLE(buf, off).toLong << 32) + // | (readIntLE(buf, off + ((len >> 3) << 2)) & 0xffffffffL) + // b = (readIntLE(buf, off + len - 4).toLong << 32) + // | (readIntLE(buf, off + len - 4 - ((len >> 3) << 2)) & 0xffffffffL) + // else if len > 0 then + // a = wyr3(buf, off, len) + // b = 0 + // else + // a = 0 + // b = 0 + // else + // var i = len + // var p = off + // var see0 = s + // var see1 = s + // var see2 = s + + // while i > 48 do + // see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) + // see1 = mix(readLongLE(buf, p + 16) ^ secret2, readLongLE(buf, p + 24) ^ see1) + // see2 = mix(readLongLE(buf, p + 32) ^ secret3, readLongLE(buf, p + 40) ^ see2) + // p += 48 + // i -= 48 + // end while + + // see0 ^= see1 ^ see2 + // while i > 16 do + // see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) + // i -= 16 + // p += 16 + // end while + + // a = readLongLE(buf, off + len - 16) + // b = readLongLE(buf, off + len - 8) + // s = see0 + // end if + // finishHash(a, b, s, len) + // end if + // end hash + +end WyHash64 diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64VarHandle.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64VarHandle.scala deleted file mode 100644 index 86413f3a8..000000000 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/WyHash64VarHandle.scala +++ /dev/null @@ -1,162 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - * - */ - -package sbt.internal.util.hashing - -import java.nio.ByteBuffer -import WyHashConstants.* -import VarHandleUtils.* - -object WyHash64VarHandle: - private[hashing] val INSTANCE = new WyHash64VarHandle() - - private[hashing] inline def initSeed(seed: Long): Long = - seed ^ mix(seed ^ PRIME64_0, PRIME64_1) - - private[hashing] def mix(a: Long, b: Long): Long = - val low = a * b - val high = unsignedMultiplyHigh(a, b) - low ^ high - - private[hashing] inline def unsignedMultiplyHigh(a: Long, b: Long): Long = - Math.multiplyHigh(a, b) + ((a >> 63) & b) + ((b >> 63) & a) - - private[hashing] inline def wyr3(buf: Array[Byte], off: Int, k: Int): Long = - ((buf(off) & 0xffL) << 16) - | ((buf(off + (k >> 1)) & 0xffL) << 8) - | (buf(off + k - 1) & 0xffL) - - private[hashing] inline def wyr3(buf: ByteBuffer, off: Int, k: Int): Long = - ((buf.get(off) & 0xffL) << 16) - | ((buf.get(off + (k >> 1)) & 0xffL) << 8) - | (buf.get(off + k - 1) & 0xffL) - - private[hashing] inline def finishHash(a: Long, b: Long, seed: Long, len: Long): Long = - val _a = a ^ PRIME64_1 - val _b = b ^ seed - val low = _a * _b - val high = unsignedMultiplyHigh(_a, _b) - mix(low ^ PRIME64_0 ^ len, high ^ PRIME64_1) - -end WyHash64VarHandle - -/** - * Wyhash matching Zig 0.15 std.hash.Wyhash. - */ -class WyHash64VarHandle extends HashAlgo: - import WyHash64VarHandle.* - - override def hash(buf: Array[Byte], offset: Int, len: Int, seed: Long): Long = - SafeUtils.checkRange(buf, offset, len) - - var off = offset - var s: Long = initSeed(seed) - val secret1 = PRIME64_1 - val secret2 = PRIME64_2 - val secret3 = PRIME64_3 - var a: Long = 0L - var b: Long = 0L - - if len <= 16 then - if len >= 4 then - a = (readIntLE(buf, off).toLong << 32) - | (readIntLE(buf, off + ((len >> 3) << 2)) & 0xffffffffL) - b = (readIntLE(buf, off + len - 4).toLong << 32) - | (readIntLE(buf, off + len - 4 - ((len >> 3) << 2)) & 0xffffffffL) - else if len > 0 then - a = wyr3(buf, off, len) - b = 0 - else - a = 0 - b = 0 - else - var i = len - var p = off - var see0 = s - var see1 = s - var see2 = s - - while i > 48 do - see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) - see1 = mix(readLongLE(buf, p + 16) ^ secret2, readLongLE(buf, p + 24) ^ see1) - see2 = mix(readLongLE(buf, p + 32) ^ secret3, readLongLE(buf, p + 40) ^ see2) - p += 48 - i -= 48 - end while - - see0 ^= see1 ^ see2 - while i > 16 do - see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) - i -= 16 - p += 16 - end while - - a = readLongLE(buf, off + len - 16) - b = readLongLE(buf, off + len - 8) - s = see0 - end if - finishHash(a, b, s, len) - end hash - - override def hash(buffer: ByteBuffer, offset: Int, len: Int, seed: Long): Long = - if buffer.hasArray() then hash(buffer.array(), offset + buffer.arrayOffset(), len, seed) - else - var off = offset - ByteBufferUtils.checkRange(buffer, off, len) - val buf = ByteBufferUtils.inLittleEndianOrder(buffer) - var s: Long = initSeed(seed) - val secret1 = PRIME64_1 - val secret2 = PRIME64_2 - val secret3 = PRIME64_3 - var a: Long = 0L - var b: Long = 0L - - if len <= 16 then - if len >= 4 then - a = (readIntLE(buf, off).toLong << 32) - | (readIntLE(buf, off + ((len >> 3) << 2)) & 0xffffffffL) - b = (readIntLE(buf, off + len - 4).toLong << 32) - | (readIntLE(buf, off + len - 4 - ((len >> 3) << 2)) & 0xffffffffL) - else if len > 0 then - a = wyr3(buf, off, len) - b = 0 - else - a = 0 - b = 0 - else - var i = len - var p = off - var see0 = s - var see1 = s - var see2 = s - - while i > 48 do - see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) - see1 = mix(readLongLE(buf, p + 16) ^ secret2, readLongLE(buf, p + 24) ^ see1) - see2 = mix(readLongLE(buf, p + 32) ^ secret3, readLongLE(buf, p + 40) ^ see2) - p += 48 - i -= 48 - end while - - see0 ^= see1 ^ see2 - while i > 16 do - see0 = mix(readLongLE(buf, p) ^ secret1, readLongLE(buf, p + 8) ^ see0) - i -= 16 - p += 16 - end while - - a = readLongLE(buf, off + len - 16) - b = readLongLE(buf, off + len - 8) - s = see0 - end if - finishHash(a, b, s, len) - end if - end hash - -end WyHash64VarHandle diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64.scala new file mode 100644 index 000000000..02905dcc3 --- /dev/null +++ b/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64.scala @@ -0,0 +1,132 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + * + */ + +package sbt.internal.util.hashing + +import java.lang.Long.rotateLeft +import java.nio.ByteBuffer +import XXHashConstants.* + +object XXHash64: + private lazy val arrayInstance: XXHash64[Array[Byte]] = + new XXHash64(0) + private lazy val byteBufferInstance: XXHash64[ByteBuffer] = + new XXHash64(0) + + def byteArray(seed: Long): XXHash64[Array[Byte]] = + if seed == 0L then arrayInstance + else new XXHash64(seed) + + def byteBuffer(seed: Long): XXHash64[ByteBuffer] = + if seed == 0L then byteBufferInstance + else new XXHash64(seed) +end XXHash64 + +/** + * The implementation is based on lz4-java. + * Copyright 2020 Linnaea Von Lavia and the lz4-java contributors. + * Licensed under the Apache License. + * + * Instances of this class are **not** thread-safe. + */ +class XXHash64[A1: Access](seed: Long) extends HashAlgo[A1]: + private val access: Access[A1] = summon[Access[A1]] + + override def hash(buf: A1, offset: Int, len: Int): Long = + var off = offset + val end: Int = off + len + var h64: Long = 0L + + if len >= 32 then + val limit = end - 32 + var v1: Long = seed + PRIME64_1 + PRIME64_2 + var v2: Long = seed + PRIME64_2 + var v3: Long = seed + 0 + var v4: Long = seed - PRIME64_1 + while + v1 += access.readLongLE(buf, off) * PRIME64_2 + v1 = rotateLeft(v1, 31) + v1 *= PRIME64_1 + off += 8 + + v2 += access.readLongLE(buf, off) * PRIME64_2 + v2 = rotateLeft(v2, 31) + v2 *= PRIME64_1 + off += 8 + + v3 += access.readLongLE(buf, off) * PRIME64_2 + v3 = rotateLeft(v3, 31) + v3 *= PRIME64_1 + off += 8 + + v4 += access.readLongLE(buf, off) * PRIME64_2 + v4 = rotateLeft(v4, 31) + v4 = v4 * PRIME64_1 + off += 8 + off <= limit + do () + + h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18) + + v1 *= PRIME64_2 + v1 = rotateLeft(v1, 31) + v1 *= PRIME64_1 + h64 ^= v1 + h64 = h64 * PRIME64_1 + PRIME64_4 + + v2 *= PRIME64_2 + v2 = rotateLeft(v2, 31) + v2 *= PRIME64_1 + h64 ^= v2 + h64 = h64 * PRIME64_1 + PRIME64_4 + + v3 *= PRIME64_2 + v3 = rotateLeft(v3, 31) + v3 *= PRIME64_1 + h64 ^= v3 + h64 = h64 * PRIME64_1 + PRIME64_4 + + v4 *= PRIME64_2 + v4 = rotateLeft(v4, 31) + v4 *= PRIME64_1 + h64 ^= v4 + h64 = h64 * PRIME64_1 + PRIME64_4 + else h64 = seed + PRIME64_5 + + h64 += len + + while off <= end - 8 do + var k1: Long = access.readLongLE(buf, off) + k1 *= PRIME64_2 + k1 = rotateLeft(k1, 31) + k1 *= PRIME64_1 + h64 ^= k1 + h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4 + off += 8 + + if off <= end - 4 then + h64 ^= (access.readIntLE(buf, off) & 0xffffffffL) * PRIME64_1 + h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3 + off += 4 + else () + + while off < end do + h64 ^= (access.readByte(buf, off) & 0xff) * PRIME64_5 + h64 = rotateLeft(h64, 11) * PRIME64_1 + off += 1 + + h64 ^= (h64 >>> 33) + h64 *= PRIME64_2 + h64 ^= (h64 >>> 29) + h64 *= PRIME64_3 + h64 ^= (h64 >>> 32) + + h64 + end hash +end XXHash64 diff --git a/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64VarHandle.scala b/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64VarHandle.scala deleted file mode 100644 index 8418a8c4b..000000000 --- a/internal/util-control/src/main/scala/sbt/internal/util/hashing/XXHash64VarHandle.scala +++ /dev/null @@ -1,222 +0,0 @@ -/* - * sbt - * Copyright 2023, Scala center - * Copyright 2011 - 2022, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - * - */ - -package sbt.internal.util.hashing - -import java.lang.Long.rotateLeft -import java.nio.ByteBuffer -import VarHandleUtils.* -import XXHashConstants.* - -object XXHash64VarHandle: - private[sbt] val INSTANCE = new XXHash64VarHandle() -end XXHash64VarHandle - -/** - * The implementation is based on lz4-java. - * Copyright 2020 Linnaea Von Lavia and the lz4-java contributors. - * Licensed under the Apache License. - * - * Instances of this class are **not** thread-safe. - */ -class XXHash64VarHandle extends HashAlgo: - override def hash(buf: Array[Byte], offset: Int, len: Int, seed: Long): Long = - SafeUtils.checkRange(buf, offset, len) - - var off = offset - val end: Int = off + len - var h64: Long = 0L - - if len >= 32 then - val limit = end - 32 - var v1: Long = seed + PRIME64_1 + PRIME64_2 - var v2: Long = seed + PRIME64_2 - var v3: Long = seed + 0 - var v4: Long = seed - PRIME64_1 - while - v1 += readLongLE(buf, off) * PRIME64_2 - v1 = rotateLeft(v1, 31) - v1 *= PRIME64_1 - off += 8 - - v2 += readLongLE(buf, off) * PRIME64_2 - v2 = rotateLeft(v2, 31) - v2 *= PRIME64_1 - off += 8 - - v3 += readLongLE(buf, off) * PRIME64_2 - v3 = rotateLeft(v3, 31) - v3 *= PRIME64_1 - off += 8 - - v4 += readLongLE(buf, off) * PRIME64_2 - v4 = rotateLeft(v4, 31) - v4 = v4 * PRIME64_1 - off += 8 - off <= limit - do () - - h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18) - - v1 *= PRIME64_2 - v1 = rotateLeft(v1, 31) - v1 *= PRIME64_1 - h64 ^= v1 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v2 *= PRIME64_2 - v2 = rotateLeft(v2, 31) - v2 *= PRIME64_1 - h64 ^= v2 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v3 *= PRIME64_2 - v3 = rotateLeft(v3, 31) - v3 *= PRIME64_1 - h64 ^= v3 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v4 *= PRIME64_2 - v4 = rotateLeft(v4, 31) - v4 *= PRIME64_1 - h64 ^= v4 - h64 = h64 * PRIME64_1 + PRIME64_4 - else h64 = seed + PRIME64_5 - - h64 += len - - while off <= end - 8 do - var k1: Long = readLongLE(buf, off) - k1 *= PRIME64_2 - k1 = rotateLeft(k1, 31) - k1 *= PRIME64_1 - h64 ^= k1 - h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4 - off += 8 - - if off <= end - 4 then - h64 ^= (readIntLE(buf, off) & 0xffffffffL) * PRIME64_1 - h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3 - off += 4 - else () - - while off < end do - h64 ^= (readByte(buf, off) & 0xff) * PRIME64_5 - h64 = rotateLeft(h64, 11) * PRIME64_1 - off += 1 - - h64 ^= (h64 >>> 33) - h64 *= PRIME64_2 - h64 ^= (h64 >>> 29) - h64 *= PRIME64_3 - h64 ^= (h64 >>> 32) - - h64 - end hash - - override def hash(buffer: ByteBuffer, offset: Int, len: Int, seed: Long): Long = - if buffer.hasArray() then hash(buffer.array(), offset + buffer.arrayOffset(), len, seed) - else - var off = offset - ByteBufferUtils.checkRange(buffer, off, len) - val buf = ByteBufferUtils.inLittleEndianOrder(buffer) - - val end: Int = off + len - var h64: Long = 0L - - if len >= 32 then - val limit: Int = end - 32 - var v1: Long = seed + PRIME64_1 + PRIME64_2 - var v2: Long = seed + PRIME64_2 - var v3: Long = seed + 0 - var v4: Long = seed - PRIME64_1 - while - v1 = v1 + readLongLE(buf, off) * PRIME64_2 - v1 = rotateLeft(v1, 31) - v1 = v1 * PRIME64_1 - off = off + 8 - - v2 += readLongLE(buf, off) * PRIME64_2 - v2 = rotateLeft(v2, 31) - v2 *= PRIME64_1 - off = off + 8 - - v3 += readLongLE(buf, off) * PRIME64_2 - v3 = rotateLeft(v3, 31) - v3 *= PRIME64_1 - off = off + 8 - - v4 += readLongLE(buf, off) * PRIME64_2 - v4 = rotateLeft(v4, 31) - v4 *= PRIME64_1 - off = off + 8 - - off <= limit - do () - - h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18) - - v1 *= PRIME64_2 - v1 = rotateLeft(v1, 31) - v1 *= PRIME64_1 - h64 ^= v1 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v2 *= PRIME64_2 - v2 = rotateLeft(v2, 31) - v2 *= PRIME64_1 - h64 ^= v2 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v3 *= PRIME64_2 - v3 = rotateLeft(v3, 31) - v3 *= PRIME64_1 - h64 ^= v3 - h64 = h64 * PRIME64_1 + PRIME64_4 - - v4 *= PRIME64_2 - v4 = rotateLeft(v4, 31) - v4 *= PRIME64_1 - h64 ^= v4 - h64 = h64 * PRIME64_1 + PRIME64_4 - else h64 = seed + PRIME64_5 - - h64 += len - - while off <= end - 8 do - var k1: Long = readLongLE(buf, off) - k1 *= PRIME64_2 - k1 = rotateLeft(k1, 31) - k1 *= PRIME64_1 - h64 ^= k1 - h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4 - off = off + 8 - - if off <= end - 4 then - h64 ^= (readIntLE(buf, off) & 0xffffffffL) * PRIME64_1 - h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3 - off = off + 4 - else () - - while off < end do - h64 ^= (readByte(buf, off) & 0xff) * PRIME64_5 - h64 = rotateLeft(h64, 11) * PRIME64_1 - off += 1 - - h64 ^= h64 >>> 33 - h64 *= PRIME64_2 - h64 ^= h64 >>> 29 - h64 *= PRIME64_3 - h64 ^= h64 >>> 32 - - h64 - end if - end hash - -end XXHash64VarHandle diff --git a/internal/util-control/src/test/scala/sbt/internal/util/AbstractByteBufferHashTest.scala b/internal/util-control/src/test/scala/sbt/internal/util/AbstractByteBufferHashTest.scala new file mode 100644 index 000000000..219246195 --- /dev/null +++ b/internal/util-control/src/test/scala/sbt/internal/util/AbstractByteBufferHashTest.scala @@ -0,0 +1,31 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + * + */ + +package sbt.internal.util.hashing + +import java.nio.ByteBuffer +import verify.BasicTestSuite + +abstract class AbstractByteBufferHashTest extends BasicTestSuite: + def hash64: HashAlgo[ByteBuffer] + def emptyHash: Long + def zeroHash: Long + + test("Hash empty ByteBuffer"): + val buf: ByteBuffer = ByteBuffer.allocate(0) + val r = hash64.hash(buf, 0, 0) + assert(r == emptyHash) + + test("Hash one byte ByteBuffer"): + val buf: ByteBuffer = ByteBuffer.allocate(1) + buf.put(0: Byte) + buf.rewind() + val r = hash64.hash(buf, 0, 1) + assert(r == zeroHash) +end AbstractByteBufferHashTest diff --git a/internal/util-control/src/test/scala/sbt/internal/util/AbstractHashTest.scala b/internal/util-control/src/test/scala/sbt/internal/util/AbstractHashTest.scala index bd2e93ed3..821b395b7 100644 --- a/internal/util-control/src/test/scala/sbt/internal/util/AbstractHashTest.scala +++ b/internal/util-control/src/test/scala/sbt/internal/util/AbstractHashTest.scala @@ -9,43 +9,24 @@ package sbt.internal.util.hashing -import java.nio.ByteBuffer import verify.BasicTestSuite abstract class AbstractHashTest extends BasicTestSuite: - def hash64: HashAlgo + def hash64: HashAlgo[Array[Byte]] def newStreaming(seed: Int): StreamingHashAlgo def emptyHash: Long def zeroHash: Long test("Hash empty array"): val buf: Array[Byte] = Array[Byte](0) - val r = hash64.hash(buf, 0, 0, 0) - assert(r == emptyHash) - - test("Hash empty ByteBuffer"): - val buf: ByteBuffer = ByteBuffer.allocate(0) - val r = hash64.hash(buf, 0, 0, 0) + val r = hash64.hash(buf, 0, 0) assert(r == emptyHash) test("Hash one byte array"): val buf: Array[Byte] = Array[Byte](0) - val r = hash64.hash(buf, 0, 1, 0) + val r = hash64.hash(buf, 0, 1) assert(r == zeroHash) - test("Hash one byte ByteBuffer"): - val buf: ByteBuffer = ByteBuffer.allocate(1) - buf.put(0: Byte) - buf.rewind() - val r = hash64.hash(buf, 0, 1, 0) - assert(r == zeroHash) - - test("Streaming hash empty ByteBuffer"): - val hash = newStreaming(0) - try - assert(hash.getValue == emptyHash) - finally hash.close() - test("Streaming one byte array"): val hash = newStreaming(0) try diff --git a/internal/util-control/src/test/scala/sbt/internal/util/WyHashTest.scala b/internal/util-control/src/test/scala/sbt/internal/util/WyHashTest.scala index b744c7784..222dcbbea 100644 --- a/internal/util-control/src/test/scala/sbt/internal/util/WyHashTest.scala +++ b/internal/util-control/src/test/scala/sbt/internal/util/WyHashTest.scala @@ -9,10 +9,19 @@ package sbt.internal.util.hashing -object WyHashTest extends AbstractHashTest: - override val hash64: HashAlgo = Hashing.wyhash64 +import java.nio.ByteBuffer + +object WyHashByteArrayTest extends AbstractHashTest: + override val hash64: HashAlgo[Array[Byte]] = Hashing.wyhash64(0L) override def newStreaming(seed: Int): StreamingHashAlgo = Hashing.newStreamingWyHash64(seed) override val emptyHash = 290873116282709081L override val zeroHash = -295637713410278011L -end WyHashTest +end WyHashByteArrayTest + +object WyHasByteBufferHashTest extends AbstractByteBufferHashTest: + override val hash64: HashAlgo[ByteBuffer] = + Hashing.wyhash64ByteBuffer(0L) + override val emptyHash = 290873116282709081L + override val zeroHash = -295637713410278011L +end WyHasByteBufferHashTest diff --git a/internal/util-control/src/test/scala/sbt/internal/util/XXHashTest.scala b/internal/util-control/src/test/scala/sbt/internal/util/XXHashTest.scala index e56d7994b..18fb4308c 100644 --- a/internal/util-control/src/test/scala/sbt/internal/util/XXHashTest.scala +++ b/internal/util-control/src/test/scala/sbt/internal/util/XXHashTest.scala @@ -9,10 +9,19 @@ package sbt.internal.util.hashing -object XXHashTest extends AbstractHashTest: - override val hash64: HashAlgo = Hashing.xxhash64 +import java.nio.ByteBuffer + +object XXHashByteArrayTest extends AbstractHashTest: + override val hash64: HashAlgo[Array[Byte]] = Hashing.xxhash64(0L) override def newStreaming(seed: Int): StreamingHashAlgo = Hashing.newStreamingXXHash64(seed) override val emptyHash = -1205034819632174695L override val zeroHash = -1642502924627794072L -end XXHashTest +end XXHashByteArrayTest + +object XXHashByteBufferHashTest extends AbstractByteBufferHashTest: + override val hash64: HashAlgo[ByteBuffer] = + Hashing.xxhash64ByteBuffer(0L) + override val emptyHash = -1205034819632174695L + override val zeroHash = -1642502924627794072L +end XXHashByteBufferHashTest diff --git a/util-cache/src/main/scala/sbt/util/HashUtil.scala b/util-cache/src/main/scala/sbt/util/HashUtil.scala index cc1ec9216..3ce184bd8 100644 --- a/util-cache/src/main/scala/sbt/util/HashUtil.scala +++ b/util-cache/src/main/scala/sbt/util/HashUtil.scala @@ -5,7 +5,7 @@ import sbt.internal.util.hashing.Hashing object HashUtil: private[sbt] def xxhash64(bytes: Array[Byte]): Long = - Hashing.xxhash64.hash(bytes, 0, bytes.size, 0) + Hashing.xxhash64(0L).hash(bytes, 0, bytes.size) private[sbt] def imohash64(path: NioPath): Long = val hash64 = Hashing.samplingFileHashWyHash64(0)