Permalink
Browse files

Preparing a new release... (still lots of work though).

  • Loading branch information...
1 parent a963a7f commit 42eb1afeb83f62cf9d85002552c858be44062676 @lemire committed Jul 22, 2013
View
@@ -1,6 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/main/java"/>
+ <classpathentry kind="src" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="output" path="bin"/>
</classpath>
@@ -14,6 +14,12 @@
import java.util.ArrayList;
import java.util.Arrays;
+/**
+ *
+ * Simple class meant to compare the speed of different schemes.
+ * @author lemire
+ *
+ */
public class Benchmark {
public static void testCodec(IntegerCODEC c, int[][] data, int Max,
@@ -17,6 +17,7 @@
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
inlength = inlength / 128 * 128;
+ if(inlength == 0) return;
out[outpos.get()] = inlength;
outpos.increment();
@@ -7,20 +7,42 @@
package me.lemire.integercompression;
+/**
+ * Generic class to compute differential coding.
+ *
+ * @author lemire
+ *
+ */
public class Delta {
+ /**
+ * Apply differential coding (in-place).
+ *
+ * @param data data to be modified
+ */
public static void delta(int[] data) {
for (int i = data.length - 1; i > 0; --i) {
data[i] -= data[i - 1];
}
}
+ /**
+ * Undo differential coding (in-place). Effectively
+ * computes a prefix sum.
+ *
+ * @param data to be modified.
+ */
public static void inverseDelta(int[] data) {
for (int i = 1; i < data.length; ++i) {
data[i] += data[i - 1];
}
}
+ /**
+ * Like inverseDelta, only faster.
+ *
+ * @param data to be modified
+ */
public static void fastinverseDelta(int[] data) {
int sz0 = data.length / 4 * 4;
int i = 1;
@@ -30,27 +30,41 @@
int[][] datatobepacked = new int[32][];
ByteBuffer bytecontainer;
+ /**
+ * Construct the FastPFOR CODEC.
+ * @param pagesize the desired page size (for expert use)
+ */
public FastPFOR(int pagesize) {
PageSize = pagesize;
initArrays();
}
+ /**
+ * Construct the fastPFOR CODEC with default parameters.
+ */
public FastPFOR() {
PageSize = 65536;
initArrays();
}
- public void initArrays() {
+ private void initArrays() {
bytecontainer = ByteBuffer.allocateDirect(3 * PageSize / BlockSize
+ PageSize);
for (int k = 1; k < 32; ++k)
datatobepacked[k] = new int[PageSize / 32 * 4];// heuristic
}
+ /**
+ * Compress data in blocks of 128 integers (if fewer than 128 integers are
+ * provided, nothing is done).
+ *
+ * @see IntegerCODEC#compress(int[], IntWrapper, int, int[], IntWrapper)
+ */
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
inlength = inlength / 128 * 128;
+ if(inlength == 0) return;
final int finalinpos = inpos.get() + inlength;
out[outpos.get()] = inlength;
@@ -62,7 +76,7 @@ public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
}
}
- public static void getBestBFromData(int[] in, int pos,
+ private static void getBestBFromData(int[] in, int pos,
byte[] bestbbestcexceptmaxb) {
int freqs[] = new int[32];
for (int k = pos; k < BlockSize + pos; ++k)
@@ -166,10 +180,16 @@ private void encodePage(int[] in, IntWrapper inpos, int thissize,
outpos.set(tmpoutpos);
}
+ /**
+ * Uncompress data in blocks of 128 integers. In this
+ * particular case, the inlength parameter is ignored:
+ * it is deduced from the compressed data.
+ *
+ * @see IntegerCODEC#compress(int[], IntWrapper, int, int[], IntWrapper)
+ */
@Override
public void uncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
int mynvalue = in[inpos.get()];
inpos.increment();
int finalout = outpos.get() + mynvalue;
@@ -188,7 +208,6 @@ private void decodePage(int[] in, IntWrapper inpos, int[] out,
int inexcept = initpos + wheremeta;
final int bytesize = in[inexcept++];
bytecontainer.clear();
- // Util.assertTrue((bytesize & 3) == 0);
bytecontainer.asIntBuffer().put(in, inexcept, bytesize / 4);
inexcept += bytesize / 4;
@@ -10,6 +10,8 @@
public interface IntegerCODEC {
/**
+ * Compress data from an array to another array.
+ *
* Both inpos and outpos are modified to represent how much
* data was read and written to
* if 12 ints (inlength = 12) are compressed to 3 ints, then
@@ -19,7 +21,17 @@
*/
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos);
- // returns how many integers we decompressed
+ /**
+ * Uncompress data from an array to another array.
+ *
+ * Both inpos and outpos parameters are modified to indicate new positions after read/write.
+ *
+ * @param in array containing data in compressed form
+ * @param inpos where to start reading in the array
+ * @param inlength length of the compressed data (ignored by some schemes)
+ * @param out array where to write the compressed output
+ * @param outpos where to write the compressed output in out
+ */
public void uncompress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos);
@@ -12,6 +12,7 @@
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
inlength = inlength / 128 * 128;
+ if(inlength == 0) return;
out[outpos.get()] = inlength;
outpos.increment();
int tmpoutpos = outpos.get();
@@ -32,9 +32,8 @@ public NewPFD() {
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
inlength = inlength / BlockSize * BlockSize;
-
+ if(inlength == 0) return;
final int finalinpos = inpos.get() + inlength;
out[outpos.get()] = inlength;
outpos.increment();
@@ -32,9 +32,8 @@ public NewPFDS9() {
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
inlength = inlength / BlockSize * BlockSize;
-
+ if(inlength == 0) return;
final int finalinpos = inpos.get() + inlength;
out[outpos.get()] = inlength;
outpos.increment();
@@ -32,9 +32,8 @@ public OptPFD() {
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
inlength = inlength / BlockSize * BlockSize;
-
+ if(inlength == 0) return;
final int finalinpos = inpos.get() + inlength;
out[outpos.get()] = inlength;
outpos.increment();
@@ -31,8 +31,8 @@ public OptPFDS9() {
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
inlength = inlength / BlockSize * BlockSize;
+ if(inlength == 0) return;
final int finalinpos = inpos.get() + inlength;
out[outpos.get()] = inlength;
@@ -13,12 +13,10 @@
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int out[],
IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
int tmpoutpos = outpos.get();
int currentPos = inpos.get();
out[tmpoutpos++] = inlength;
final int finalin = currentPos + inlength;
- // Util.assertTrue(finalin <= in.length);
outer:
while (currentPos < finalin - 28) {
mainloop:
@@ -69,25 +67,20 @@ public void compress(int[] in, IntWrapper inpos, int inlength, int out[],
continue outer;
}
final int selector = 8;
- // Util.assertTrue(codeNum[selector] == 1);
if (in[currentPos] >= 1 << bitLength[selector])
throw new RuntimeException("Too big a number");
out[tmpoutpos++] = in[currentPos++] | (selector << 28);
}
- // Util.assertTrue(currentPos == finalin);
inpos.set(currentPos);
outpos.set(tmpoutpos);
}
- // @Override
@Override
public void uncompress(int[] in, IntWrapper inpos, int inlength,
int[] out, IntWrapper outpos) {
- // Util.assertTrue(inpos.get()+inlength <= in.length);
int currentPos = outpos.get();
int tmpinpos = inpos.get();
final int finalout = currentPos + in[tmpinpos++];
- // Util.assertTrue(finalout <= out.length);
while (currentPos < finalout - 28) {
int val = in[tmpinpos++];
int header = val >>> 28;
@@ -6,15 +6,21 @@
*/
package me.lemire.integercompression;
+/**
+ * Routine utility functions.
+ *
+ * @author lemire
+ *
+ */
public class Util {
- public static int maxbits(int[] i, int pos, int length) {
+ protected static int maxbits(int[] i, int pos, int length) {
int mask = 0;
for (int k = pos; k < pos + length; ++k)
mask |= i[k];
return bits(mask);
}
- public static int maxdiffbits(int initoffset, int[] i, int pos, int length) {
+ protected static int maxdiffbits(int initoffset, int[] i, int pos, int length) {
int mask = 0;
mask |= (i[pos] - initoffset);
for (int k = pos + 1; k < pos + length; ++k) {
@@ -23,16 +29,8 @@ public static int maxdiffbits(int initoffset, int[] i, int pos, int length) {
return bits(mask);
}
- public static int bits(int i) {
+ protected static int bits(int i) {
return 32 - Integer.numberOfLeadingZeros(i);
}
- public static void assertTrue(boolean b) {
- if (!b) throw new RuntimeException("bug!");
- }
-
- public static void assertTrue(boolean b, String message) {
- if (!b) throw new RuntimeException(message);
- }
-
}

0 comments on commit 42eb1af

Please sign in to comment.