Skip to content

Convert XxHash64 to VarHandle #183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dain merged 2 commits into from
Feb 1, 2026
Merged

Convert XxHash64 to VarHandle #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7bb85ab
Update to Java 25 and Airbase 347
dain Feb 1, 2026
3b3c589
Convert XxHash64 to VarHandle
dain Jan 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
java: ['17', '21']
java: ['25']
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Install java
uses: actions/setup-java@v4
with:
java-version: '21'
java-version: '25'
distribution: 'temurin'
gpg-private-key: ${{ secrets.JRELEASER_GPG_SECRET_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>io.airlift</groupId>
<artifactId>airbase</artifactId>
<version>271</version>
<version>347</version>
</parent>

<artifactId>slice</artifactId>
Expand All @@ -28,8 +28,8 @@
<properties>
<air.javadoc.lint>-missing</air.javadoc.lint>
<air.test.jvmsize>3584m</air.test.jvmsize>
<project.build.targetJdk>17</project.build.targetJdk>
<air.java.version>17</air.java.version>
<project.build.targetJdk>25</project.build.targetJdk>
<air.java.version>25</air.java.version>
<air.modernizer.java-version>8</air.modernizer.java-version>
<air.check.skip-spotbugs>true</air.check.skip-spotbugs>
<air.check.skip-pmd>true</air.check.skip-pmd>
Expand Down
145 changes: 84 additions & 61 deletions src/main/java/io/airlift/slice/XxHash64.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@

import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.VarHandle;
import java.nio.ByteOrder;

import static io.airlift.slice.JvmUtils.unsafe;
import static java.lang.Long.rotateLeft;
import static java.lang.Math.min;
import static java.util.Objects.checkFromIndexSize;
import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET;

public final class XxHash64
{
Expand All @@ -32,9 +33,11 @@ public final class XxHash64

private static final long DEFAULT_SEED = 0;

private static final VarHandle LONG_HANDLE = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN);
private static final VarHandle INT_HANDLE = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN);

private final long seed;

private static final long BUFFER_ADDRESS = ARRAY_BYTE_BASE_OFFSET;
private final byte[] buffer = new byte[32];
private int bufferSize;

Expand Down Expand Up @@ -67,7 +70,7 @@ public XxHash64 update(byte[] data)
public XxHash64 update(byte[] data, int offset, int length)
{
checkFromIndexSize(offset, length, data.length);
updateHash(data, ARRAY_BYTE_BASE_OFFSET + offset, length);
updateHash(data, offset, length);
return this;
}

Expand All @@ -79,7 +82,7 @@ public XxHash64 update(Slice data)
public XxHash64 update(Slice data, int offset, int length)
{
checkFromIndexSize(offset, length, data.length());
updateHash(data.byteArray(), (long) data.byteArrayOffset() + ARRAY_BYTE_BASE_OFFSET + offset, length);
updateHash(data.byteArray(), data.byteArrayOffset() + offset, length);
return this;
}

Expand All @@ -95,7 +98,7 @@ public long hash()

hash += bodyLength + bufferSize;

return updateTail(hash, buffer, BUFFER_ADDRESS, 0, bufferSize);
return updateTail(hash, buffer, 0, bufferSize);
}

private long computeBody()
Expand All @@ -110,45 +113,45 @@ private long computeBody()
return hash;
}

private void updateHash(byte[] base, long address, int length)
private void updateHash(byte[] base, int offset, int length)
{
if (bufferSize > 0) {
int available = min(32 - bufferSize, length);

unsafe.copyMemory(base, address, buffer, BUFFER_ADDRESS + bufferSize, available);
System.arraycopy(base, offset, buffer, bufferSize, available);

bufferSize += available;
address += available;
offset += available;
length -= available;

if (bufferSize == 32) {
updateBody(buffer, BUFFER_ADDRESS, bufferSize);
updateBody(buffer, 0, bufferSize);
bufferSize = 0;
}
}

if (length >= 32) {
int index = updateBody(base, address, length);
address += index;
int index = updateBody(base, offset, length);
offset += index;
length -= index;
}

if (length > 0) {
unsafe.copyMemory(base, address, buffer, BUFFER_ADDRESS, length);
System.arraycopy(base, offset, buffer, 0, length);
bufferSize = length;
}
}

private int updateBody(byte[] base, long address, int length)
private int updateBody(byte[] base, int offset, int length)
{
int remaining = length;
while (remaining >= 32) {
v1 = mix(v1, unsafe.getLong(base, address));
v2 = mix(v2, unsafe.getLong(base, address + 8));
v3 = mix(v3, unsafe.getLong(base, address + 16));
v4 = mix(v4, unsafe.getLong(base, address + 24));
v1 = mix(v1, (long) LONG_HANDLE.get(base, offset));
v2 = mix(v2, (long) LONG_HANDLE.get(base, offset + 8));
v3 = mix(v3, (long) LONG_HANDLE.get(base, offset + 16));
v4 = mix(v4, (long) LONG_HANDLE.get(base, offset + 24));

address += 32;
offset += 32;
remaining -= 32;
}

Expand Down Expand Up @@ -192,6 +195,22 @@ public static long hash(long seed, InputStream in)
return hash.hash();
}

public static long hash(byte[] data)
{
return hash(DEFAULT_SEED, data, 0, data.length);
}

public static long hash(byte[] data, int offset, int length)
{
return hash(DEFAULT_SEED, data, offset, length);
}

public static long hash(long seed, byte[] data, int offset, int length)
{
checkFromIndexSize(offset, length, data.length);
return hashBytes(seed, data, offset, length);
}

public static long hash(Slice data)
{
return hash(data, 0, data.length());
Expand All @@ -210,75 +229,79 @@ public static long hash(Slice data, int offset, int length)
public static long hash(long seed, Slice data, int offset, int length)
{
checkFromIndexSize(offset, length, data.length());
return hashBytes(seed, data.byteArray(), data.byteArrayOffset() + offset, length);
}

byte[] base = data.byteArray();
final long address = (long) data.byteArrayOffset() + ARRAY_BYTE_BASE_OFFSET + offset;

private static long hashBytes(long seed, byte[] base, int index, int length)
{
int end = index + length;
long hash;

if (length >= 32) {
hash = updateBody(seed, base, address, length);
long v1 = seed + PRIME64_1 + PRIME64_2;
long v2 = seed + PRIME64_2;
long v3 = seed;
long v4 = seed - PRIME64_1;

while (index <= end - 32) {
v1 = mix(v1, (long) LONG_HANDLE.get(base, index));
v2 = mix(v2, (long) LONG_HANDLE.get(base, index + 8));
v3 = mix(v3, (long) LONG_HANDLE.get(base, index + 16));
v4 = mix(v4, (long) LONG_HANDLE.get(base, index + 24));
index += 32;
}

hash = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18);
hash = update(hash, v1);
hash = update(hash, v2);
hash = update(hash, v3);
hash = update(hash, v4);
}
else {
hash = seed + PRIME64_5;
}

hash += length;

// round to the closest 32 byte boundary
// this is the point up to which updateBody() processed
int index = length & 0xFFFFFFE0;

return updateTail(hash, base, address, index, length);
}

private static long updateTail(long hash, byte[] base, long address, int index, int length)
{
while (index <= length - 8) {
hash = updateTail(hash, unsafe.getLong(base, address + index));
// Process remaining bytes
while (index <= end - 8) {
hash = updateTail(hash, (long) LONG_HANDLE.get(base, index));
index += 8;
}

if (index <= length - 4) {
hash = updateTail(hash, unsafe.getInt(base, address + index));
if (index <= end - 4) {
hash = updateTail(hash, (int) INT_HANDLE.get(base, index));
index += 4;
}

while (index < length) {
hash = updateTail(hash, unsafe.getByte(base, address + index));
while (index < end) {
hash = updateTail(hash, base[index]);
index++;
}

hash = finalShuffle(hash);

return hash;
return finalShuffle(hash);
}

private static long updateBody(long seed, byte[] base, long address, int length)
private static long updateTail(long hash, byte[] base, int index, int length)
{
long v1 = seed + PRIME64_1 + PRIME64_2;
long v2 = seed + PRIME64_2;
long v3 = seed;
long v4 = seed - PRIME64_1;
int end = index + length;

int remaining = length;
while (remaining >= 32) {
v1 = mix(v1, unsafe.getLong(base, address));
v2 = mix(v2, unsafe.getLong(base, address + 8));
v3 = mix(v3, unsafe.getLong(base, address + 16));
v4 = mix(v4, unsafe.getLong(base, address + 24));

address += 32;
remaining -= 32;
while (index <= end - 8) {
hash = updateTail(hash, (long) LONG_HANDLE.get(base, index));
index += 8;
}

long hash = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18);
if (index <= end - 4) {
hash = updateTail(hash, (int) INT_HANDLE.get(base, index));
index += 4;
}

hash = update(hash, v1);
hash = update(hash, v2);
hash = update(hash, v3);
hash = update(hash, v4);
while (index < end) {
hash = updateTail(hash, base[index]);
index++;
}

return hash;
return finalShuffle(hash);
}

private static long mix(long current, long value)
Expand Down