Skip to content
Permalink
Browse files
Merge pull request #96 from DataSketches/xxHashArrays
Xx hash arrays
  • Loading branch information
leerho committed Apr 9, 2019
2 parents 1a753ac + 17abdcd commit db0ce2b4ee42363c02661d594966af9f93802bf4
Showing 4 changed files with 189 additions and 3 deletions.
@@ -228,7 +228,7 @@ static boolean isFileReadOnly(final File file) {
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.close();
return false;
} catch (final Exception e) { //could not open for write
} catch (final SecurityException | IOException f) { //could not open for write
return true;
}
}
@@ -5,6 +5,20 @@

package com.yahoo.memory;

import static com.yahoo.memory.UnsafeUtil.ARRAY_BOOLEAN_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_BYTE_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_CHAR_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_DOUBLE_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_FLOAT_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_INT_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_LONG_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.ARRAY_SHORT_BASE_OFFSET;
import static com.yahoo.memory.UnsafeUtil.CHAR_SHIFT;
import static com.yahoo.memory.UnsafeUtil.DOUBLE_SHIFT;
import static com.yahoo.memory.UnsafeUtil.FLOAT_SHIFT;
import static com.yahoo.memory.UnsafeUtil.INT_SHIFT;
import static com.yahoo.memory.UnsafeUtil.LONG_SHIFT;
import static com.yahoo.memory.UnsafeUtil.SHORT_SHIFT;
import static com.yahoo.memory.UnsafeUtil.unsafe;

/**
@@ -25,7 +39,7 @@
*
* @author Lee Rhodes
*/
class XxHash64 {
public class XxHash64 {
// Unsigned, 64-bit primes
private static final long P1 = -7046029288634856825L;
private static final long P2 = -4417276706812531889L;
@@ -150,5 +164,124 @@ private static long finalize(long hash) {
hash ^= hash >>> 32;
return hash;
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetBooleans starting at this offset
* @param lengthBooleans continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashBooleans(final boolean[] arr, final long cumOffsetBooleans,
final long lengthBooleans, final long seed) {
return hash(arr, ARRAY_BOOLEAN_BASE_OFFSET + cumOffsetBooleans, lengthBooleans, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetBytes starting at this offset
* @param lengthBytes continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashBytes(final byte[] arr, final long cumOffsetBytes,
final long lengthBytes, final long seed) {
return hash(arr, ARRAY_BYTE_BASE_OFFSET + cumOffsetBytes, lengthBytes, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetShorts starting at this offset
* @param lengthShorts continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashShorts(final short[] arr, final long cumOffsetShorts,
final long lengthShorts, final long seed) {
return hash(arr, ARRAY_SHORT_BASE_OFFSET + (cumOffsetShorts << SHORT_SHIFT),
lengthShorts << SHORT_SHIFT, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetChars starting at this offset
* @param lengthChars continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashChars(final char[] arr, final long cumOffsetChars,
final long lengthChars, final long seed) {
return hash(arr, ARRAY_CHAR_BASE_OFFSET + (cumOffsetChars << CHAR_SHIFT),
lengthChars << CHAR_SHIFT, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetInts starting at this offset
* @param lengthInts continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashInts(final int[] arr, final long cumOffsetInts,
final long lengthInts, final long seed) {
return hash(arr, ARRAY_INT_BASE_OFFSET + (cumOffsetInts << INT_SHIFT),
lengthInts << INT_SHIFT, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetLongs starting at this offset
* @param lengthLongs continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashLongs(final long[] arr, final long cumOffsetLongs,
final long lengthLongs, final long seed) {
return hash(arr, ARRAY_LONG_BASE_OFFSET + (cumOffsetLongs << LONG_SHIFT),
lengthLongs << LONG_SHIFT, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetFloats starting at this offset
* @param lengthFloats continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashFloats(final float[] arr, final long cumOffsetFloats,
final long lengthFloats, final long seed) {
return hash(arr, ARRAY_FLOAT_BASE_OFFSET + (cumOffsetFloats << FLOAT_SHIFT),
lengthFloats << FLOAT_SHIFT, seed);
}

/**
* Hash the given arr starting at the given offset and continuing for the given length using the
* given seed.
* @param arr the given array
* @param cumOffsetDoubles starting at this offset
* @param lengthDoubles continuing for this length
* @param seed the given seed
* @return the hash
*/
public static long hashDoubles(final double[] arr, final long cumOffsetDoubles,
final long lengthDoubles, final long seed) {
return hash(arr, ARRAY_DOUBLE_BASE_OFFSET + (cumOffsetDoubles << DOUBLE_SHIFT),
lengthDoubles << DOUBLE_SHIFT, seed);
}

}

@@ -35,7 +35,7 @@ public void testWithSeed() {
* <a href="https://github.com/OpenHFT/Zero-Allocation-Hashing/blob/master/src/test/java/net/openhft/hashing/XxHashTest.java">
* OpenHFT/Zero-Allocation-Hashing</a> to test hash compatibility with that implementation.
*/
public static final long[] HASHES_OF_LOOPING_BYTES_WITH_SEED_42 = {
private static final long[] HASHES_OF_LOOPING_BYTES_WITH_SEED_42 = {
-7444071767201028348L,
-8959994473701255385L,
7116559933691734543L,
@@ -5,6 +5,14 @@

package com.yahoo.memory;

import static com.yahoo.memory.XxHash64.hashBooleans;
import static com.yahoo.memory.XxHash64.hashBytes;
import static com.yahoo.memory.XxHash64.hashChars;
import static com.yahoo.memory.XxHash64.hashDoubles;
import static com.yahoo.memory.XxHash64.hashFloats;
import static com.yahoo.memory.XxHash64.hashInts;
import static com.yahoo.memory.XxHash64.hashLongs;
import static com.yahoo.memory.XxHash64.hashShorts;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;

@@ -104,4 +112,49 @@ public void testXxHash() {
}
}

private static final byte[] barr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};

@Test
public void testArrHashes() {
WritableMemory wmem = WritableMemory.wrap(barr);
long hash0 = wmem.xxHash64(8, 8, 0);
long hash1 = hashBytes(barr, 8, 8, 0);
assertEquals(hash1, hash0);

char[] carr = new char[8];
wmem.getCharArray(0, carr, 0, 8);
hash1 = hashChars(carr, 4, 4, 0);
assertEquals(hash1, hash0);

short[] sarr = new short[8];
wmem.getShortArray(0, sarr, 0, 8);
hash1 = hashShorts(sarr, 4, 4, 0);
assertEquals(hash1, hash0);

int[] iarr = new int[4];
wmem.getIntArray(0, iarr, 0, 4);
hash1 = hashInts(iarr, 2, 2, 0);
assertEquals(hash1, hash0);

float[] farr = new float[4];
wmem.getFloatArray(0, farr, 0, 4);
hash1 = hashFloats(farr, 2, 2, 0);
assertEquals(hash1, hash0);

long[] larr = new long[2];
wmem.getLongArray(0, larr, 0, 2);
hash1 = hashLongs(larr, 1, 1, 0);
assertEquals(hash1, hash0);

double[] darr = new double[2];
wmem.getDoubleArray(0, darr, 0, 2);
hash1 = hashDoubles(darr, 1, 1, 0);
assertEquals(hash1, hash0);

boolean[] blarr = new boolean[16];
wmem.getBooleanArray(0, blarr, 0, 16); //any byte != 0 is true
hash1 = hashBooleans(blarr, 8, 8, 0);
assertEquals(hash1, hash0);
}

}

0 comments on commit db0ce2b

Please sign in to comment.