Skip to content
Browse files

change PForDelta to be backward compatible to 3.0.1 bf13927, changing…

… in the compressed block: the int(b and expNum), the second int(blockSize), combining the expPos and expHighBits
  • Loading branch information...
1 parent 386f855 commit e8c872b37414048c5cbea030e57c9b4a8b22bf5d hao yan committed Sep 14, 2012
View
3 pom.xml
@@ -9,7 +9,7 @@
<groupId>com.linkedin.kamikaze</groupId>
<artifactId>kamikaze</artifactId>
<packaging>jar</packaging>
- <version>3.0.6</version>
+ <version>3.0.7</version>
<name>kamikaze</name>
<description>information retrival utility package for enhancing Lucene</description>
<url>http://sna-projects.com/kamikaze</url>
@@ -70,6 +70,7 @@
-Xms256m -Xmx4g <!-- -Xdebug -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y -->
</argLine>
<excludes>
+ <exclude>com/kamikaze/lucenecode/test/*.java</exclude>
<exclude>com/kamikaze/test/perf/*.java</exclude>
<exclude>**/*$*</exclude>
</excludes>
View
2 src/main/java/com/kamikaze/docidset/impl/PForDeltaDocIdSet.java
@@ -46,7 +46,7 @@
transient private int[] currentNoCompBlock; // the memory used to store the uncompressed elements. Once the block is full, all its elements are compressed into sequencOfCompBlock and the block is cleared.
transient private int sizeOfCurrentNoCompBlock = 0; // the number of uncompressed elements that is hold in the currentNoCompBlock
- private int version = 1;
+ transient private int version = 1;
public PForDeltaDocIdSet() {
sequenceOfCompBlocks = new PForDeltaIntSegmentArray();
View
49 src/main/java/com/kamikaze/pfordelta/PForDelta.java
@@ -36,7 +36,7 @@
// Max number of bits to store an uncompressed value
private static final int MAX_BITS = 32;
// Header records the value of b and the number of exceptions in the block
- private static final int HEADER_NUM = 1;
+ private static final int HEADER_NUM = 2;
// Header size in bits
private static final int HEADER_SIZE = MAX_BITS * HEADER_NUM;
@@ -97,12 +97,11 @@
*/
public static int decompressOneBlock(int[] outBlock, int[] inBlock, int blockSize)
{
- int[] expPos = new int[blockSize];
- int[] expHighBits = new int[blockSize];
-
- int expNum = inBlock[0] & MASK[31-POSSIBLE_B_BITS];
- int bits = (inBlock[0]>>>(31-POSSIBLE_B_BITS)) & (0x1f);
+ int[] expAux = new int[blockSize*2];
+ int expNum = inBlock[0] & 0x3ff;
+ int bits = (inBlock[0]>>>10) & (0x1f);
+
// decompress the b-bit slots
int offset = HEADER_SIZE;
int compressedBits = 0;
@@ -121,15 +120,13 @@ public static int decompressOneBlock(int[] outBlock, int[] inBlock, int blockSiz
// decompress exceptions
if(expNum>0)
{
- compressedBits = decompressBlockByS16(expPos, inBlock, offset, expNum);
- offset += compressedBits;
- compressedBits = decompressBlockByS16(expHighBits, inBlock, offset, expNum);
+ compressedBits = decompressBlockByS16(expAux, inBlock, offset, expNum*2);
offset += compressedBits;
for (int i = 0; i < expNum; i++)
{
- int curExpPos = expPos[i] ;
- int curHighBits = expHighBits[i];
+ int curExpPos = expAux[i] ;
+ int curHighBits = expAux[i+expNum];
outBlock[curExpPos] = (outBlock[curExpPos] & MASK[bits]) | ((curHighBits & MASK[32-bits] ) << bits);
}
}
@@ -188,16 +185,24 @@ public static boolean checkBigNumbers(int[] inputBlock, int bits, int blockSize)
*/
public static int[] compressOneBlock(int[] inputBlock, int bits, int blockSize) throws IllegalArgumentException {
- int[] expPos = new int[blockSize];
- int[] expHighBits = new int[blockSize];
-
+ int[] expAux = new int[blockSize*2];
+
int maxCompBitSize = HEADER_SIZE + blockSize * (MAX_BITS + MAX_BITS + MAX_BITS) + 32;
int[] tmpCompressedBlock = new int[(maxCompBitSize>>>5)];
int outputOffset = HEADER_SIZE;
int expUpperBound = 1<<bits;
int expNum = 0;
+ for(int elem : inputBlock)
+ {
+ if(elem >= expUpperBound)
+ {
+ expNum++;
+ }
+ }
+
+ int expIndex = 0;
// compress the b-bit slots
for (int i = 0; i<blockSize; ++i)
{
@@ -210,23 +215,23 @@ public static boolean checkBigNumbers(int[] inputBlock, int bits, int blockSize)
// store the lower bits-bits of the exception
writeBits(tmpCompressedBlock, inputBlock[i] & MASK[bits], outputOffset, bits);
// write the position of exception
- expPos[expNum] = i;
+ expAux[expIndex] = i;
// write the higher 32-bits bits of the exception
- expHighBits[expNum] = (inputBlock[i] >>> bits) & MASK[32-bits];
- expNum++;
+ expAux[expIndex + expNum] = (inputBlock[i] >>> bits) & MASK[32-bits];
+ expIndex++;
}
outputOffset += bits;
}
// the first int in the compressed block stores the value of b and the number of exceptions
- tmpCompressedBlock[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31-POSSIBLE_B_BITS)) | (expNum & MASK[31-POSSIBLE_B_BITS]);
-
+ //tmpCompressedBlock[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31-POSSIBLE_B_BITS)) | (expNum & MASK[31-POSSIBLE_B_BITS]);
+ tmpCompressedBlock[0] = ((bits & MASK[10]) << 10) | (expNum & 0x3ff);
+ tmpCompressedBlock[1] = inputBlock[blockSize-1];
+
// compress exceptions
if(expNum>0)
{
- int compressedBitSize = compressBlockByS16(tmpCompressedBlock, outputOffset, expPos, expNum);
- outputOffset += compressedBitSize;
- compressedBitSize = compressBlockByS16(tmpCompressedBlock, outputOffset, expHighBits, expNum);
+ int compressedBitSize = compressBlockByS16(tmpCompressedBlock, outputOffset, expAux, expNum*2);
outputOffset += compressedBitSize;
}

0 comments on commit e8c872b

Please sign in to comment.
Something went wrong with that request. Please try again.