Permalink
Browse files

improved the hardcode of pfordelta and simple16, update the contrib t…

…o the latest PForDeltaFixedIntBlockWithIntBuffer codec
  • Loading branch information...
1 parent d990eab commit 020f234e1d9363251ce6b5d3402ae00b5a9705ab @hyan hyan committed Feb 19, 2011
View
3 .classpath
@@ -1,9 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
+ <classpathentry kind="src" path="contrib/luceneCodec/test"/>
+ <classpathentry excluding="com/kamikaze/lucecodec/util/" kind="src" path="contrib/luceneCodec/src"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+ <classpathentry kind="lib" path="/Users/hyan/workspace/ConvertLuceneFrom3To4/lib/master/lucene-core-4.0-SNAPSHOT.jar"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
View
7 contrib/luceneCodec/MANIFEST.MF
@@ -2,7 +2,8 @@ Manifest-Version: 1.0
Ant-Version: Apache Ant 1.7.1
Created-By: 17.1-b03-307 (Apple Inc.)
Built-By: hyan
-Git-Version: 3dbe9dcf8e7a5eca8371dbca6f0d6b00ee38aca6 removed unnecc f
- iles
-Built-Date: 2010-12-21 18:09:27
+Git-Version: d990eab504e1f071fd16b4caffc9d8b2e1848e86 Added profile, w
+ hich signs artifacts during deployment. Changed ids of release and sn
+ apshot repositories.
+Built-Date: 2011-02-18 19:41:06
View
48 ...enecodec/PForDeltaFixedIntBlockCodec.java → ...DeltaFixedIntBlockWithIntBufferCodec.java
@@ -1,23 +1,5 @@
package com.kamikaze.lucenecodec;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
import java.io.IOException;
import java.util.Set;
@@ -29,8 +11,8 @@
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
-import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
-import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
+import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -48,13 +30,13 @@
* used here writes each block as data encoded by PForDelta.
*/
-public class PForDeltaFixedIntBlockCodec extends Codec {
+public class PForDeltaFixedIntBlockWithIntBufferCodec extends Codec {
private final int blockSize;
- public PForDeltaFixedIntBlockCodec(int blockSize) {
+ public PForDeltaFixedIntBlockWithIntBufferCodec(int blockSize) {
this.blockSize = blockSize;
- name = "NewPForDelta";
+ name = "PatchedFrameOfRef4";
}
@Override
@@ -65,12 +47,12 @@ public String toString() {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new PForDeltaFixedIntBlockFactory(blockSize));
+ PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new PForDeltaFixedIntBlockWithIntBufferFactory(blockSize));
boolean success = false;
TermsIndexWriterBase indexWriter;
try {
- indexWriter = new FixedGapTermsIndexWriter(state);
+ indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNTermSelector(state.termIndexInterval));
success = true;
} finally {
if (!success) {
@@ -99,16 +81,16 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
state.segmentInfo,
state.readBufferSize,
- new PForDeltaFixedIntBlockFactory(blockSize), state.codecId);
+ new PForDeltaFixedIntBlockWithIntBufferFactory(blockSize), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- state.termsIndexDivisor,
- BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId);
+ indexReader = new VariableGapTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
+ state.codecId);
success = true;
} finally {
if (!success) {
@@ -144,14 +126,14 @@ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException
public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
SepPostingsReaderImpl.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+ VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
}
@Override
public void getExtensions(Set<String> extensions) {
SepPostingsWriterImpl.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
- FixedGapTermsIndexReader.getIndexExtensions(extensions);
+ VariableGapTermsIndexReader.getIndexExtensions(extensions);
}
}
View
11 ...ecodec/PForDeltaFixedIntBlockFactory.java → ...ltaFixedIntBlockWithIntBufferFactory.java
@@ -1,6 +1,5 @@
package com.kamikaze.lucenecodec;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -25,21 +24,21 @@
import java.io.IOException;
-public class PForDeltaFixedIntBlockFactory extends IntStreamFactory {
+public class PForDeltaFixedIntBlockWithIntBufferFactory extends IntStreamFactory {
private final int blockSize;
/** blockSize is only used when creating the
* IntIndexOutput */
- public PForDeltaFixedIntBlockFactory(int blockSize) {
+ public PForDeltaFixedIntBlockWithIntBufferFactory(int blockSize) {
this.blockSize = blockSize;
}
public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
- return new PForDeltaFixedIntBlockIndexInput(dir, fileName, readBufferSize);
+ return new PForDeltaFixedIntBlockWithIntBufferIndexInput(dir, fileName, readBufferSize);
}
public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
- return new PForDeltaFixedIntBlockIndexOutput(dir, fileName, blockSize);
+ return new PForDeltaFixedIntBlockWithIntBufferIndexOutput(dir, fileName, blockSize);
}
-}
+}
View
59 ...dec/PForDeltaFixedIntBlockIndexInput.java → ...FixedIntBlockWithIntBufferIndexInput.java
@@ -1,6 +1,5 @@
package com.kamikaze.lucenecodec;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -20,26 +19,44 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.pfor2.LCPForDelta;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
-import com.kamikaze.pfordelta.LCPForDelta;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
-public class PForDeltaFixedIntBlockIndexInput extends FixedIntBlockIndexInput {
+public class PForDeltaFixedIntBlockWithIntBufferIndexInput extends FixedIntBlockIndexInput {
- public PForDeltaFixedIntBlockIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException {
+ public PForDeltaFixedIntBlockWithIntBufferIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException {
super(dir.openInput(fileName, readBufferSize));
+
}
private static class BlockReader implements FixedIntBlockIndexInput.BlockReader {
private final LCPForDelta decompressor;
private final IndexInput input;
- private final int[] decompBuffer;
+ private final int[] decompBlock;
+
+ private final ByteBuffer byteCompBuffer;
+ private final IntBuffer intCompBuffer;
+ private final byte[] byteCompBlock;
+ private final int[] expPosIntBlock;
+ private final int[] expHighBitIntBlock;
+
+ private static final int MAX_BLOCK_SIZE = 128;
public BlockReader(IndexInput in, int[] buffer) {
decompressor = new LCPForDelta();
input = in;
- decompBuffer = buffer;
+ decompBlock = buffer;
+
+ byteCompBuffer = ByteBuffer.allocate(MAX_BLOCK_SIZE*4*4);
+ byteCompBlock = byteCompBuffer.array();
+ intCompBuffer = byteCompBuffer.asIntBuffer();
+
+ expPosIntBlock = new int[MAX_BLOCK_SIZE];
+ expHighBitIntBlock = new int[MAX_BLOCK_SIZE];
}
public void seek(long pos) throws IOException {
@@ -51,28 +68,16 @@ public void readBlock() throws IOException {
// read the compressed data
final int compressedSizeInInt = input.readInt();
- // two ways to read the data
- // first way
-// int[] compBuffer = new int[compressedSizeInInt];
-// for(int i=0;i<compressedSizeInInt;i++) {
-// compBuffer[i] = input.readInt();
-// }
-
- // second way
- byte[] byteBuffer = new byte[compressedSizeInInt*4];
- input.readBytes(byteBuffer, 0, compressedSizeInInt*4);
- // convert the byte array into int array
- int[] compBuffer = new int[compressedSizeInInt];
- int i,j;
- for(i=0,j=0; j<compressedSizeInInt; j++)
- {
- compBuffer[j] = ((byteBuffer[i++] & 0xff)<<24) | ((byteBuffer[i++] & 0xff)<<16)
- | ((byteBuffer[i++] & 0xff)<<8) | (byteBuffer[i++] & 0xff);
- }
+ int blockSize = 128;
+ input.readBytes(byteCompBlock, 0, compressedSizeInInt*4);
+ intCompBuffer.rewind();
- // decompress
- decompressor.decompressOneBlock(decompBuffer, compBuffer);
- compBuffer = null;
+ decompressor.decompressOneBlockWithSizeWithIntBuffer(decompBlock, intCompBuffer, blockSize, expPosIntBlock, expHighBitIntBlock, compressedSizeInInt);
+ }
+
+ public void skipBlock() throws IOException {
+ int numInts = input.readInt(); // nocommit: should PFOR use vint header?
+ input.seek(input.getFilePointer() + numInts*4); // seek past block
}
}
View
37 ...ec/PForDeltaFixedIntBlockIndexOutput.java → ...ixedIntBlockWithIntBufferIndexOutput.java
@@ -20,14 +20,15 @@
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
import org.apache.lucene.store.Directory;
-import com.kamikaze.pfordelta.LCPForDelta;
-
+import org.apache.lucene.util.pfor2.LCPForDelta;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
-public class PForDeltaFixedIntBlockIndexOutput extends FixedIntBlockIndexOutput {
+public class PForDeltaFixedIntBlockWithIntBufferIndexOutput extends FixedIntBlockIndexOutput {
private final LCPForDelta compressor;
private final int blockSize;
- public PForDeltaFixedIntBlockIndexOutput(Directory dir, String fileName, int blockSize) throws IOException {
+ public PForDeltaFixedIntBlockWithIntBufferIndexOutput(Directory dir, String fileName, int blockSize) throws IOException {
super(dir.createOutput(fileName), blockSize);
this.blockSize = blockSize;
compressor = new LCPForDelta();
@@ -39,29 +40,15 @@ protected void flushBlock() throws IOException {
// write out the compressed size in ints
out.writeInt(compressedSizeInInts);
- // we can use either of the following two ways to write out the compressed data
- // first way
-// int[] compBuffer = compressor.getCompBuffer();
-// for(int i=0;i<compressedSizeInInts;i++) {
-// out.writeInt(compBuffer[i]);
-// }
-
- // second way
- int[] compBuffer = compressor.getCompBuffer();
- // convert int array to byte array
- byte[] byteBuffer = new byte[compressedSizeInInts*4];
- int i, j;
- for(i=0, j=0; j<compressedSizeInInts; i+=4, j++)
- {
- int val = compBuffer[j];
- byteBuffer[i] = (byte)(val >>> 24);
- byteBuffer[i+1] = (byte)(val >>> 16);
- byteBuffer[i+2] = (byte)(val >>> 8);
- byteBuffer[i+3] = (byte)(val);
- }
- out.writeBytes(byteBuffer, byteBuffer.length);
+ int[] compBlock = compressor.getCompBuffer();
+ ByteBuffer byteCompBuffer = ByteBuffer.allocate(compressedSizeInInts*4);
+ byte[] byteCompBlock = byteCompBuffer.array();
+ IntBuffer intCompBuffer = byteCompBuffer.asIntBuffer();
+ intCompBuffer.put(compBlock, 0, compressedSizeInInts);
+ out.writeBytes(byteCompBlock, byteCompBlock.length);
compressor.setCompBuffer(null);
}
}
+
View
2 contrib/luceneCodec/test/com/kamikaze/lucenecodec/test/TestLuceneCodecSuite.java
@@ -4,6 +4,8 @@
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
+import com.kamikaze.lucenecodec.test.TestLucenePForDeltaCodec;
+
@RunWith(Suite.class)
@SuiteClasses( { TestLucenePForDeltaCodec.class})
public class TestLuceneCodecSuite {
View
9 contrib/luceneCodec/test/com/kamikaze/lucenecodec/test/TestLucenePForDeltaCodec.java
@@ -4,25 +4,22 @@
import org.apache.lucene.index.BulkPostingsEnum;
import org.apache.lucene.index.codecs.sep.*;
-import org.apache.lucene.store.*;
-
-import org.junit.Ignore;
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.junit.After;
import org.junit.Before;
import java.util.Random;
-import com.kamikaze.lucenecodec.PForDeltaFixedIntBlockFactory;
+import com.kamikaze.lucenecodec.PForDeltaFixedIntBlockWithIntBufferFactory;
public class TestLucenePForDeltaCodec extends TestCase{
public void testPForDeltaSimpleIntBlocks() throws Exception {
System.out.println("running test case : testPForDeltaSimpleIntBlocks for PForDeltaFixedIntBlockCodec");
Directory dir = new RAMDirectory();
int blockSize = 128;
- IntStreamFactory f = new PForDeltaFixedIntBlockFactory(blockSize);
+ IntStreamFactory f = new PForDeltaFixedIntBlockWithIntBufferFactory(blockSize);
int testDataSize = 80024;
int[] testData = new int[testDataSize];
Random random = new Random(0);
@@ -62,7 +59,7 @@ public void testPForDeltaEmptySimpleIntBlocks() throws Exception {
System.out.println("running test case : testPForDeltaEmptySimpleIntBlocks for PForDeltaFixedIntBlockCodec");
Directory dir = new RAMDirectory();
int blockSize = 128;
- IntStreamFactory f = new PForDeltaFixedIntBlockFactory(blockSize);
+ IntStreamFactory f = new PForDeltaFixedIntBlockWithIntBufferFactory(blockSize);
IntIndexOutput out = f.createOutput(dir, "test");
// write no ints
View
134 pom.xml
@@ -29,11 +29,11 @@
</scm>
<developers>
- <developer>
- <id>javasoze</id>
- <name>John Wang</name>
- <email>john.wang@gmail.com</email>
- </developer>
+ <developer>
+<id>hyan</id>
+<name>hao yan</name>
+<email>hyan2008@gmail.com</email>
+</developer>
</developers>
<!-- Set the compiler to java6 -->
@@ -50,11 +50,11 @@
</configuration>
</plugin>
<plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.2</version>
- <configuration>
- <encoding>UTF-8</encoding>
- </configuration>
+<artifactId>maven-resources-plugin</artifactId>
+<version>2.2</version>
+<configuration>
+ <encoding>UTF-8</encoding>
+</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -70,30 +70,30 @@
</excludes>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-sources</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+<plugin>
+<groupId>org.apache.maven.plugins</groupId>
+<artifactId>maven-source-plugin</artifactId>
+<executions>
+<execution>
+<id>attach-sources</id>
+<goals>
+<goal>jar</goal>
+</goals>
+</execution>
+</executions>
+</plugin>
+<plugin>
+<groupId>org.apache.maven.plugins</groupId>
+<artifactId>maven-javadoc-plugin</artifactId>
+<executions>
+<execution>
+<id>attach-javadocs</id>
+<goals>
+<goal>jar</goal>
+</goals>
+</execution>
+</executions>
+</plugin>
</plugins>
</build>
@@ -109,54 +109,28 @@
<version>2.9.1</version>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.5</version>
- <scope>test</scope>
+<groupId>junit</groupId>
+<artifactId>junit</artifactId>
+<version>4.5</version>
+<scope>test</scope>
</dependency>
</dependencies>
- <distributionManagement>
- <repository>
- <id>sonatype-nexus-staging</id>
- <name>sonatype nexus staging repository</name>
- <url>http://oss.sonatype.org/service/local/staging/deploy/maven2</url>
- </repository>
- <snapshotRepository>
- <id>sonatype-nexus-snapshots</id>
- <name>sonatype nexus snapshots repository</name>
- <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
-
- <profiles>
- <profile>
- <id>release-sign-artifacts</id>
- <activation>
- <property>
- <name>performRelease</name>
- <value>true</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-gpg-plugin</artifactId>
- <version>1.1</version>
- <executions>
- <execution>
- <id>sign-artifacts</id>
- <phase>verify</phase>
- <goals>
- <goal>sign</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
+
+<distributionManagement>
+ <snapshotRepository>
+ <id>ossKamikaze</id>
+ <name>SNA snapshot repo</name>
+ <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
+ </snapshotRepository>
+
+<!--
+<repository>
+<id>nexusKamikazeRelease</id>
+<name>Nexus staging Repo</name>
+<url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
+</repository>
+-->
+ </distributionManagement>
</project>
View
88 src/main/java/com/kamikaze/docidset/utils/GenerateUnpackClass.java
@@ -4,41 +4,41 @@
import java.io.IOException;
import java.io.PrintWriter;
+/**
+ * Generate the hardwired unpack code to read every k bits from 128 integers .
+ * This is highly optimized to speed up the decompression speed of
+ * decoding b-bit slots (that is, non-exceptions) in PForDelta algorithm
+ * @author hyan
+ *
+ */
public class GenerateUnpackClass {
public static void main(String[] args)
{
try
{
- generatePForDeltaUnpackClass(128, "/Users/hyan/workspace/UnpackTmp.txt");
+ generatePForDeltaUnpackClass("com.kamikaze.pfordelta", 128, "./PForDeltaUnpack128.txt");
}
catch(Exception e)
{
e.printStackTrace();
}
}
- /*example usage
- public void writeUnpackFile() throws IOException
- {
- GenerateUnpackClass.generatePForDeltaUnpackClass(256, "/Users/hyan/workspace/UnpackTmp.txt");
- }
- */
- static public void generatePForDeltaUnpackClass(int blockSize, String filename) throws IOException
+ static public void generatePForDeltaUnpackClass(String packageName, int blockSize, String filename) throws IOException
{
PrintWriter pw = new PrintWriter(new FileOutputStream(filename));
- pw.println("package com.kamikaze.docidset.compression;");
+ pw.println("package " + packageName + ";");
pw.println(" ");
pw.println(" ");
- pw.println("public class PForDeltaUnpack{");
+ pw.println("public class PForDeltaUnpack128{");
pw.println(" ");
generatePForDeltaFunctionSelectionFile(pw);
int HEADER_BITS = 32 ; // two int header
for(int i=0; i<POSSIBLE_B.length; ++i)
{
pw.println(" ");
- //generatePForDeltaUnpackFile(pw, HEADER_BITS, blockSize, POSSIBLE_B[i]);
generatePForDeltaUnpackFileEach32(pw, HEADER_BITS, blockSize, POSSIBLE_B[i]);
}
@@ -64,57 +64,60 @@ static private void generatePForDeltaUnpackFileEach32(PrintWriter pw, int inOffs
if(bits>0)
{
pw.println(" int i, w;");
+ pw.println(" int outOffset = " + 0 + ";");
+ pw.println(" final int mask = " + MASK[bits] + ";");
int index,skip;
pw.println(" for(i=0, w=" + (inOffset>>>5) + "; i<" + (n/32) + "; ++i, w+=" + bits + "){");
int localInOffset = 0;
+
+ int prevIndex = -1;
+ for(int i=0 ; i<32; ++i, localInOffset+=bits)
+ {
+ index = localInOffset >>> 5;
+ if(index != prevIndex)
+ {
+ if(index == 0)
+ {
+ pw.println(" int curInputValue" + index + " = in[w];");
+ }
+ else
+ {
+ pw.println(" int curInputValue" + index + " = in[w+" + index +"];");
+ }
+ prevIndex = index;
+ }
+ }
+
+ prevIndex = -1;
+ localInOffset = 0;
for(int i=0 ; i<32; ++i, localInOffset+=bits)
{
index = localInOffset >>> 5;
skip = localInOffset & 0x1f;
if(skip == 0)
- pw.println(" out[" + i + "+(i<<5)] = ((in[w+" + index + "]) & " + MASK[bits] + ");");
+ pw.println(" out[" + i + "+outOffset] = curInputValue" + index + " & mask;");
else
{
if (32 - skip < bits) {
- pw.println(" out[" + i + "+(i<<5)] = ((in[w+" + index + "] >>> " + (skip) + ") & " + MASK[bits] + ");");
- pw.println(" out[" + i + "+(i<<5)] |= ((in[w+" + (index+1) + "] << " + (32-skip) + ") & " + MASK[bits] + ");");
+ pw.println(" out[" + i + "+outOffset] = ((curInputValue" + index + " >>> " + (skip) + ") | (curInputValue" + (index+1) + " << " + (32-skip) + ")) & mask;");
+ }
+ else if(32-skip == bits)
+ {
+ pw.println(" out[" + i + "+outOffset] = curInputValue" + index + " >>> " + (skip) + ";");
}
else
- pw.println(" out[" + i + "+(i<<5)] = ((in[w+" + index + "] >>> " + (skip) + ") & " + MASK[bits] + ");");
+ {
+ pw.println(" out[" + i + "+outOffset] = (curInputValue" + index + " >>> " + (skip) + ") & mask;");
+ }
}
}
+ pw.println(" outOffset += 32;");
pw.println(" }");
}
pw.println(" }");
}
- static private void generatePForDeltaUnpackFile(PrintWriter pw, int inOffset, int n, int bits)
- {
- pw.println(" static private void unpack" + bits + "(int[] out, int[] in)");
- pw.println(" {");
-
- if(bits>0)
- {
- for(int i=0; i<n; ++i, inOffset+=bits)
- {
- //out[outStart+i] = readBits(in, inStart+i*bits, bits);
-
- final int index = inOffset >>> 5;
- final int skip = inOffset & 0x1f;
- if(skip == 0)
- pw.println(" out[" + i + "] = ((in[" + (index) + "]) & " + MASK[bits] + ");");
- else
- pw.println(" out[" + i + "] = ((in[" + (index) + "] >>> " + (skip) + ") & " + MASK[bits] + ");");
-
- if (32 - skip < bits) {
- pw.println(" out[" + i + "] |= ((in[" + (index+1) + "] << " + (32-skip) + ") & " + MASK[bits] + ");");
- }
- }
- }
- pw.println(" }");
- }
-
static private void generatePForDeltaFunctionSelectionFile(PrintWriter pw)
{
pw.println(" static public void unpack(int[] out, int[] in, int bits) {" );
@@ -129,7 +132,4 @@ static private void generatePForDeltaFunctionSelectionFile(PrintWriter pw)
pw.println(" }");
pw.println(" }");
}
-
-
-
}
View
130 src/main/java/com/kamikaze/docidset/utils/GenerateUnpackClass2.java
@@ -0,0 +1,130 @@
+package com.kamikaze.docidset.utils;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * This class is basically same as GenerateUnpackClass except that the input is IntBuffer instead of int[] .
+ * @author hyan
+ *
+ */
+public class GenerateUnpackClass2 {
+
+ public static void main(String[] args)
+ {
+ try
+ {
+ generatePForDeltaUnpackClass("com.kamikaze.pfordelta", 128, "./PForDeltaUnpack128.txt");
+ }
+ catch(Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ static public void generatePForDeltaUnpackClass(String packageName, int blockSize, String filename) throws IOException
+ {
+ PrintWriter pw = new PrintWriter(new FileOutputStream(filename));
+ pw.println("package " + packageName + ";");
+ pw.println(" ");
+ pw.println("import java.nio.IntBuffer;");
+ pw.println(" ");
+
+ pw.println("public class PForDeltaUnpack{");
+ pw.println(" ");
+ generatePForDeltaFunctionSelectionFile(pw);
+ int HEADER_BITS = 32 ; // two int header
+ for(int i=0; i<POSSIBLE_B.length; ++i)
+ {
+ pw.println(" ");
+ //generatePForDeltaUnpackFile(pw, HEADER_BITS, blockSize, POSSIBLE_B[i]);
+ generatePForDeltaUnpackFileEach32(pw, HEADER_BITS, blockSize, POSSIBLE_B[i]);
+ }
+
+ pw.println("}");
+ pw.close();
+
+ }
+
+ private static int[] POSSIBLE_B = {0, 1,2,3,4,5,6,7,8,9,10,11,12,13,16,20,28};
+
+ private static final int[] MASK = {0x00000000,
+ 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f,
+ 0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
+ 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff,
+ 0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff,
+ 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff,
+ 0x7fffffff, 0xffffffff};
+
+ static private void generatePForDeltaUnpackFileEach32(PrintWriter pw, int inOffset, int n, int bits)
+ {
+ pw.println(" static private void unpack" + bits + "(int[] out, IntBuffer in)");
+ pw.println(" {");
+ if(bits>0)
+ {
+ pw.println(" int outOffset = " + 0 + ";");
+ pw.println(" final int mask = " + MASK[bits] + ";");
+ int index,skip;
+ pw.println(" for(int i=0; i<" + (n/32) + "; ++i){");
+ int localInOffset = 0;
+
+ int prevIndex = -1;
+ for(int i=0 ; i<32; ++i, localInOffset+=bits)
+ {
+ index = localInOffset >>> 5;
+ if(index != prevIndex)
+ {
+ pw.println(" int curInputValue" + index + " = in.get();");
+ prevIndex = index;
+ }
+ }
+
+ prevIndex = -1;
+ localInOffset = 0;
+ for(int i=0 ; i<32; ++i, localInOffset+=bits)
+ {
+ index = localInOffset >>> 5;
+ skip = localInOffset & 0x1f;
+ if(skip == 0)
+ pw.println(" out[" + i + "+outOffset] = curInputValue" + index + " & mask;");
+ else
+ {
+ if (32 - skip < bits) {
+ pw.println(" out[" + i + "+outOffset] = ((curInputValue" + index + " >>> " + (skip) + ") | (curInputValue" + (index+1) + " << " + (32-skip) + ")) & mask;");
+ }
+ else if(32-skip == bits)
+ {
+ pw.println(" out[" + i + "+outOffset] = curInputValue" + index + " >>> " + (skip) + ";");
+ }
+ else
+ {
+ pw.println(" out[" + i + "+outOffset] = (curInputValue" + index + " >>> " + (skip) + ") & mask;");
+ }
+ }
+
+ }
+ pw.println(" outOffset += 32;");
+ pw.println(" }");
+ }
+ pw.println(" }");
+ }
+
+ static private void generatePForDeltaFunctionSelectionFile(PrintWriter pw)
+ {
+ pw.println(" static public void unpack(int[] out, InputBuffer in, int bits) {" );
+ pw.println(" switch (bits) {");
+
+ for(int i=0; i<POSSIBLE_B.length; i++)
+ {
+ int bits = POSSIBLE_B[i];
+ pw.println(" case " + bits + ": " + "unpack" + bits + "(out, in); break;");
+ }
+ pw.println(" default: break;");
+ pw.println(" }");
+ pw.println(" }");
+ }
+
+
+
+}
View
142 src/main/java/com/kamikaze/docidset/utils/GenerateUnpackClassS16.java
@@ -0,0 +1,142 @@
+package com.kamikaze.docidset.utils;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Generate the the hardwired unpack code to decode an compressed integer in Simple16
+ * @author hyan
+ *
+ */
+
+public class GenerateUnpackClassS16 {
+
+ private static final int S16_NUMSIZE = 16;
+ private static final int S16_BITSSIZE = 28;
+
+ //the possible number of compressed numbers hold in a single 32-bit integer
+ private static final int[] S16_NUM = {28, 21, 21, 21, 14, 9, 8, 7, 6, 6, 5, 5, 4, 3, 2, 1};
+
+ //the possible number of bits used to compress one number
+ private static final int[][] S16_BITS = { {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1},
+ {2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0},
+ {1,1,1,1,1,1,1,2,2,2,2,2,2,2,1,1,1,1,1,1,1,0,0,0,0,0,0,0},
+ {1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,0,0,0,0,0,0,0},
+ {2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {4,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {3,4,4,4,4,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {4,4,4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {5,5,5,5,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {4,4,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {6,6,6,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {5,5,6,6,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {7,7,7,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {10,9,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {14,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+ {28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} };
+
+ public static void main(String[] args)
+ {
+ try
+ {
+ generateS16UnpackFunction("/Users/hyan/workspace/UnpackS16Tmp.java", 0);
+ generateS16UnpackFunction2("/Users/hyan/workspace/UnpackS16Tmp2.java", 0);
+ }
+ catch(Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ static public void generateS16UnpackFunction(String filename, int numIdx) throws IOException
+ {
+ PrintWriter pw = new PrintWriter(new FileOutputStream(filename));
+ pw.println("public static int s16DecompressOneNumberWithHardCodes(int[] out, int outOffset, int value, int numIdx){");
+ pw.println(" switch(numIdx){");
+ int i,j,shift, s16Bits;
+ for(i=0; i<S16_BITS.length; ++i)
+ {
+ int num = S16_NUM[i];
+ pw.println(" case " + i + ":");
+ pw.println(" {");
+ for(j=0, shift=0; j<num; ++j)
+ {
+ s16Bits = S16_BITS[i][j];
+ if(shift>0)
+ {
+ if(j>0)
+ pw.println(" out[outOffset+" + j + "] = (value >>> " + shift + ") & " + MASK[s16Bits] + ";");
+ else
+ pw.println(" out[outOffset] = (value >>> " + shift + ") & " + MASK[s16Bits] + ";");
+ }
+ else
+ {
+ if(j>0)
+ pw.println(" out[outOffset+" + j + "] = value & " + MASK[s16Bits] + ";");
+ else
+ pw.println(" out[outOffset] = value & " + MASK[s16Bits] + ";");
+ }
+ shift += s16Bits;
+ }
+ pw.println(" return " + num + ";");
+ pw.println(" }");
+ }
+ pw.println(" default:");
+ pw.println(" return -1;");
+ pw.println(" }");
+ pw.println("}");
+ pw.close();
+
+ }
+
+ static public void generateS16UnpackFunction2(String filename, int numIdx) throws IOException
+ {
+ PrintWriter pw = new PrintWriter(new FileOutputStream(filename));
+ pw.println("public static int s16DecompressOneNumberWithHardCodesIntegrated(int[] out, int outOffset, int value, int numIdx, int oribits, int[] expPos){");
+ pw.println(" switch(numIdx){");
+ int i,j,shift, s16Bits;
+ for(i=0; i<S16_BITS.length; ++i)
+ {
+ int num = S16_NUM[i];
+ pw.println(" case " + i + ":");
+ pw.println(" {");
+ for(j=0, shift=0; j<num; ++j)
+ {
+ s16Bits = S16_BITS[i][j];
+ if(shift>0)
+ {
+ if(j>0)
+ pw.println(" out[expPos[outOffset+" + j + "]] |= (((value >>> " + shift + ") & " + MASK[s16Bits] + ")<<oribits);");
+ else
+ pw.println(" out[expPos[outOffset]] |= (((value >>> " + shift + ") & " + MASK[s16Bits] + ")<<oribits);");
+ }
+ else
+ {
+ if(j>0)
+ pw.println(" out[expPos[outOffset+" + j + "]] |= ((value & " + MASK[s16Bits] + ")<<oribits);");
+ else
+ pw.println(" out[expPos[outOffset]] |= ((value & " + MASK[s16Bits] + ")<<oribits);");
+ }
+ shift += s16Bits;
+ }
+ pw.println(" return " + num + ";");
+ pw.println(" }");
+ }
+ pw.println(" default:");
+ pw.println(" return -1;");
+ pw.println(" }");
+ pw.println("}");
+ pw.close();
+
+ }
+
+ private static final String[] MASK = {"0x00000000",
+ "0x00000001", "0x00000003", "0x00000007", "0x0000000f", "0x0000001f", "0x0000003f",
+ "0x0000007f", "0x000000ff", "0x000001ff", "0x000003ff", "0x000007ff", "0x00000fff",
+ "0x00001fff", "0x00003fff", "0x00007fff", "0x0000ffff", "0x0001ffff", "0x0003ffff",
+ "0x0007ffff", "0x000fffff", "0x001fffff", "0x003fffff", "0x007fffff", "0x00ffffff",
+ "0x01ffffff", "0x03ffffff", "0x07ffffff", "0x0fffffff", "0x1fffffff", "0x3fffffff",
+ "0x7fffffff", "0xffffffff"};
+
+}
View
197 src/main/java/com/kamikaze/pfordelta/LCPForDelta.java
@@ -19,6 +19,7 @@
*/
import java.util.Arrays;
+import java.nio.IntBuffer;
/**
* Implementation of the optimized PForDelta algorithm for sorted integer arrays. The basic ideas are based on
@@ -45,7 +46,7 @@
public class LCPForDelta{
// NOTE: we expect the blockSize is always < (1<<(31-POSSIBLE_B_BITS)). For example, in the current default settings,
- // the blockSize < (1<<(31-5)), that is, < 2^27, the commonly used block size is 128 or 256.
+ // the blockSize < (1<<(31-5)), that is, < 2^27
//All possible values of b in the PForDelta algorithm
private static final int[] POSSIBLE_B = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,16,20,28};
@@ -55,7 +56,7 @@
// Max number of bits to store an uncompressed value
private static final int MAX_BITS = 32;
// Header records the value of b and the number of exceptions in the block
- private static final int HEADER_NUM = 2;
+ private static final int HEADER_NUM = 1;
// Header size in bits
private static final int HEADER_SIZE = MAX_BITS * HEADER_NUM;
@@ -131,8 +132,10 @@ public int compressOneBlockCore(int[] inputBlock, int blockSize, int bits) throw
int expNum = 0;
int maxCompBitSize = HEADER_SIZE + blockSize * (MAX_BITS + MAX_BITS + MAX_BITS) + 32;
int[] tmpCompBuffer = new int[(maxCompBitSize>>>5)];
+
int[] expPosBuffer = new int[blockSize];
int[] expHighBitsBuffer = new int[blockSize];
+
// compress the b-bit slots
for (int i = 0; i<blockSize; ++i)
{
@@ -154,15 +157,17 @@ public int compressOneBlockCore(int[] inputBlock, int blockSize, int bits) throw
outputOffset += bits;
}
- tmpCompBuffer[0] = blockSize;
- tmpCompBuffer[1] = ((bits & MASK[POSSIBLE_B_BITS]) << (31-POSSIBLE_B_BITS)) | (expNum & MASK[31-POSSIBLE_B_BITS]);
+ tmpCompBuffer[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31-POSSIBLE_B_BITS)) | (expNum & MASK[31-POSSIBLE_B_BITS]);
// compress exceptions
if(expNum>0)
{
- int compressedBitSize = compressBlockByS16(tmpCompBuffer, outputOffset, expPosBuffer, expNum);
+ int compressedBitSize;
+
+ compressedBitSize = compressBlockByS16(tmpCompBuffer, outputOffset, expPosBuffer, expNum, blockSize, inputBlock);
outputOffset += compressedBitSize;
- compressedBitSize = compressBlockByS16(tmpCompBuffer, outputOffset, expHighBitsBuffer, expNum);
+
+ compressedBitSize = compressBlockByS16(tmpCompBuffer, outputOffset, expHighBitsBuffer, expNum, blockSize, inputBlock);
outputOffset += compressedBitSize;
}
@@ -173,18 +178,70 @@ public int compressOneBlockCore(int[] inputBlock, int blockSize, int bits) throw
return compressedSizeInInts;
}
+ public int compressOneBlockCore2(int[] inputBlock, int blockSize, int bits) throws IllegalArgumentException {
+ int outputOffset = HEADER_SIZE;
+ int expUpperBound = 1<<bits;
+ int expNum = 0;
+ int maxCompBitSize = HEADER_SIZE + blockSize * (MAX_BITS + MAX_BITS + MAX_BITS) + 32;
+ int[] tmpCompBuffer = new int[(maxCompBitSize>>>5)];
+
+ int[] expPosBuffer = new int[blockSize];
+ int[] expHighBitsBuffer = new int[blockSize];
+
+ // compress the b-bit slots
+ for (int i = 0; i<blockSize; ++i)
+ {
+ int value = inputBlock[i];
+ if (value < expUpperBound)
+ {
+ writeBits(tmpCompBuffer, value, outputOffset, bits);
+ }
+ else // exp
+ {
+ // store the lower bits-bits of the exception
+ writeBits(tmpCompBuffer, value & MASK[bits], outputOffset, bits);
+ // write the position of exception
+ expPosBuffer[expNum] = i;
+ // write the higher 32-bits bits of the exception
+ expHighBitsBuffer[expNum] = (value >>> bits) & MASK[32-bits];
+ expNum++;
+ }
+ outputOffset += bits;
+ }
+
+ tmpCompBuffer[0] = ((bits & MASK[POSSIBLE_B_BITS]) << (31-POSSIBLE_B_BITS)) | (expNum & MASK[31-POSSIBLE_B_BITS]);
+
+ // compress exceptions
+ if(expNum>0)
+ {
+ int compressedBitSize;
+
+ int[] expBuffer = new int[expNum*2];
+ System.arraycopy(expPosBuffer,0, expBuffer,0,expNum);
+ System.arraycopy(expHighBitsBuffer,0,expBuffer,expNum,expNum);
+
+ compressedBitSize = compressBlockByS16(tmpCompBuffer, outputOffset, expBuffer, expNum*2, blockSize, inputBlock);
+ outputOffset += compressedBitSize;
+ }
+
+ // discard the redundant parts in the tmpCompressedBlock
+ int compressedSizeInInts = (outputOffset+31)>>>5;
+
+ compBuffer = tmpCompBuffer;
+ return compressedSizeInInts;
+ }
+
/**
* Decompress one block using PForDelta
* @param decompBlock the block that was decompressed
* @param inBlock the block to be decompressed
* @param blockSize the number of elements in the decompressed block
*
*/
- public void decompressOneBlock(int[] decompBlock, int[] inBlock)
+ public void decompressOneBlock(int[] decompBlock, int[] inBlock, int blockSize)
{
- int blockSize = inBlock[0];
- int expNum = inBlock[1] & MASK[31-POSSIBLE_B_BITS];
- int bits = (inBlock[1]>>>(31-POSSIBLE_B_BITS)) & (0x1f);
+ int expNum = inBlock[0] & MASK[31-POSSIBLE_B_BITS];
+ int bits = (inBlock[0]>>>(31-POSSIBLE_B_BITS)) & (0x1f);
int[] expPosBuffer = new int[blockSize];
int[] expHighBitsBuffer = new int[blockSize];
@@ -199,7 +256,6 @@ public void decompressOneBlock(int[] decompBlock, int[] inBlock)
else
{
compressedBits = decompressBBitSlots(decompBlock, inBlock, blockSize, bits);
- // hardcoding does not help with -server
//compressedBits = decompressBBitSlotsWithHardCodes(decompBlock, inBlock, blockSize, bits);
}
offset += compressedBits;
@@ -220,6 +276,76 @@ public void decompressOneBlock(int[] decompBlock, int[] inBlock)
}
}
}
+
+ public void decompressOneBlockWithSize(int[] decompBlock, int[] inBlock, int blockSize, int[] expPosBuffer, int[] expHighBitsBuffer, int inBlockLen)
+ {
+ int expNum = inBlock[0] & MASK[31-POSSIBLE_B_BITS];
+ int bits = (inBlock[0]>>>(31-POSSIBLE_B_BITS)) & (0x1f);
+
+ // decompress the b-bit slots
+ int offset = HEADER_SIZE;
+ int compressedBits = 0;
+ if(bits == 0)
+ {
+ Arrays.fill(decompBlock,0, inBlockLen, 0);
+ }
+ else
+ {
+ //compressedBits = decompressBBitSlotsWithHardCodes(decompBlock, inBlock, blockSize, bits);
+ compressedBits = decompressBBitSlots(decompBlock, inBlock, blockSize, bits);
+ }
+ offset += compressedBits;
+
+ // decompress exceptions
+ if(expNum>0)
+ {
+ compressedBits = decompressBlockByS16(expPosBuffer, inBlock, offset, expNum);
+ offset += compressedBits;
+ compressedBits = decompressBlockByS16(expHighBitsBuffer, inBlock, offset, expNum);
+ offset += compressedBits;
+
+ for (int i = 0; i < expNum; i++)
+ {
+ int curExpPos = expPosBuffer[i] ;
+ int curHighBits = expHighBitsBuffer[i];
+ decompBlock[curExpPos] = (decompBlock[curExpPos] & MASK[bits]) | ((curHighBits & MASK[32-bits] ) << bits);
+ }
+ }
+ }
+
+ public void decompressOneBlockWithSizeWithIntBuffer(final int[] decompBlock, final IntBuffer inBlock, final int blockSize, final int[] expPosBuffer, final int[] expHighBitsBuffer, final int inBlockLen)
+ {
+ final int flag = inBlock.get();
+ final int expNum = flag & MASK[31-POSSIBLE_B_BITS];
+ final int bits = (flag>>>(31-POSSIBLE_B_BITS)) & (0x1f);
+ if(bits == 0)
+ {
+ Arrays.fill(decompBlock,0, inBlockLen, 0);
+ }
+ else
+ {
+ PForDeltaUnpack128WIthIntBuffer.unpack(decompBlock, inBlock, bits);
+ }
+
+ if(expNum>0)
+ {
+ // decompress expPos
+ int num, outOffset=0, numLeft;
+ for(numLeft=expNum; numLeft>0; numLeft -= num)
+ {
+ num = Simple16WithHardCodes.s16DecompressWithIntBufferWithHardCodes(expPosBuffer, outOffset, inBlock.get(), numLeft);
+ outOffset += num;
+ }
+
+ // decompress expHighBits and decompBlock at the same time
+ for(outOffset=0, numLeft=expNum; numLeft>0; numLeft -= num)
+ {
+ num = Simple16WithHardCodes.s16DecompressWithIntBufferIntegrated2(decompBlock, outOffset, inBlock.get(), numLeft, expPosBuffer, bits);
+ outOffset += num;
+ }
+ }
+ }
+
/**
* Estimate the compressed size in ints of a block
@@ -288,6 +414,20 @@ public int decompressBBitSlots(int[] outDecompSlots, int[] inCompBlock, int bloc
}
+ public int decompressBBitSlotsWithHardCodes(final int[] outDecompSlots, final int[] inCompBlock, final int blockSize, final int bits)
+ {
+ int compressedBitSize = 0;
+ PForDeltaUnpack128.unpack(outDecompSlots, inCompBlock, bits);
+ compressedBitSize = bits * blockSize;
+ return compressedBitSize;
+ }
+
+ public int decompressBBitSlotsWithHardCodesWithIntBuffer(final int[] outDecompSlots, final IntBuffer inCompBlock, final int blockSize, final int bits)
+ {
+ PForDeltaUnpack128WIthIntBuffer.unpack(outDecompSlots, inCompBlock, bits);
+ return bits * blockSize;
+ }
+
/**
* Compress a block of blockSize integers using Simple16 algorithm
@@ -297,13 +437,13 @@ public int decompressBBitSlots(int[] outDecompSlots, int[] inCompBlock, int bloc
* @param blockSize the block size
* @return the compressed size in bits
*/
- private int compressBlockByS16(int[] outCompBlock, int outStartOffsetInBits, int[] inBlock, int blockSize)
+ private int compressBlockByS16(int[] outCompBlock, int outStartOffsetInBits, int[] inBlock, int blockSize, int oriBlockSize, int[] oriInputBlock)
{
int outOffset = (outStartOffsetInBits+31)>>>5;
int num, inOffset=0, numLeft;
for(numLeft=blockSize; numLeft>0; numLeft -= num)
{
- num = Simple16.s16Compress(outCompBlock, outOffset, inBlock, inOffset, numLeft, blockSize);
+ num = Simple16WithHardCodes.s16Compress(outCompBlock, outOffset, inBlock, inOffset, numLeft, blockSize, oriBlockSize, oriInputBlock);
outOffset++;
inOffset += num;
}
@@ -333,6 +473,26 @@ public int decompressBlockByS16(int[] outDecompBlock, int[] inCompBlock, int inS
return compressedBitSize;
}
+ public void decompressBlockByS16WithIntBuffer(final int[] outDecompBlock, final IntBuffer inCompBlock, final int blockSize)
+ {
+ int num, outOffset=0, numLeft;
+ for(numLeft=blockSize; numLeft>0; numLeft -= num)
+ {
+ num = Simple16WithHardCodes.s16DecompressWithIntBuffer(outDecompBlock, outOffset, inCompBlock.get(), numLeft);
+ outOffset += num;
+ }
+ }
+
+ public void decompressBlockByS16WithIntBufferIntegrated(final int[] outDecompBlock, final IntBuffer inCompBlock, final int blockSize, int[] expPosBuffer, int oribits)
+ {
+ int num, outOffset=0, numLeft;
+ for(numLeft=blockSize; numLeft>0; numLeft -= num)
+ {
+ num = Simple16WithHardCodes.s16DecompressWithIntBufferIntegrated(outDecompBlock, outOffset, inCompBlock.get(), numLeft, expPosBuffer, oribits);
+ outOffset += num;
+ }
+ }
+
/**
* Write a certain number of bits of an integer into an integer array starting from the given start offset
@@ -373,5 +533,16 @@ public static final int readBits(int[] in, final int inOffset, final int bits) {
return val & (0xffffffff >>> (32 - bits));
}
+ public static final int readBitsWithBuffer(int[] in, final int inOffset, final int bits) {
+ final int index = inOffset >>> 5;
+ final int skip = inOffset & 0x1f;
+ int val = in[index] >>> skip;
+ if (32 - skip < bits) {
+ val |= (in[index + 1] << (32 - skip));
+ }
+ return val & (0xffffffff >>> (32 - bits));
+ }
+
}
+
View
4 src/main/java/com/kamikaze/pfordelta/PForDelta.java
@@ -113,6 +113,8 @@ public static int decompressOneBlock(int[] outBlock, int[] inBlock, int blockSiz
else
{
compressedBits = decompressBBitSlots(outBlock, inBlock, blockSize, bits);
+ // Note that blocksize must be ==128 in order to use decompressBBitSlotsWithHardCodes
+ //compressedBits = decompressBBitSlotsWithHardCodes(outBlock, inBlock, blockSize, bits);
}
offset += compressedBits;
@@ -334,7 +336,7 @@ public static final void writeBits(int[] out, int val, int outOffset, int bits)
* @param bits the value of b
* @return the processed data size (the number of bits in the compressed form)
*/
- private int decompressBBitSlotsWithHardCodes(int[] decompressedSlots, int[] compBlock, int blockSize, int bits)
+ static int decompressBBitSlotsWithHardCodes(int[] decompressedSlots, int[] compBlock, int blockSize, int bits)
{
int compressedBitSize = 0;
PForDeltaUnpack128.unpack(decompressedSlots, compBlock, bits);
View
1,329 src/main/java/com/kamikaze/pfordelta/PForDeltaUnpack128.java
@@ -33,726 +33,827 @@ static private void unpack0(int[] out, int[] in)
static private void unpack1(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 1;
for(i=0, w=1; i<4; ++i, w+=1){
- out[0+(i<<5)] = ((in[w+0]) & 1);
- out[1+(i<<5)] = ((in[w+0] >>> 1) & 1);
- out[2+(i<<5)] = ((in[w+0] >>> 2) & 1);
- out[3+(i<<5)] = ((in[w+0] >>> 3) & 1);
- out[4+(i<<5)] = ((in[w+0] >>> 4) & 1);
- out[5+(i<<5)] = ((in[w+0] >>> 5) & 1);
- out[6+(i<<5)] = ((in[w+0] >>> 6) & 1);
- out[7+(i<<5)] = ((in[w+0] >>> 7) & 1);
- out[8+(i<<5)] = ((in[w+0] >>> 8) & 1);
- out[9+(i<<5)] = ((in[w+0] >>> 9) & 1);
- out[10+(i<<5)] = ((in[w+0] >>> 10) & 1);
- out[11+(i<<5)] = ((in[w+0] >>> 11) & 1);
- out[12+(i<<5)] = ((in[w+0] >>> 12) & 1);
- out[13+(i<<5)] = ((in[w+0] >>> 13) & 1);
- out[14+(i<<5)] = ((in[w+0] >>> 14) & 1);
- out[15+(i<<5)] = ((in[w+0] >>> 15) & 1);
- out[16+(i<<5)] = ((in[w+0] >>> 16) & 1);
- out[17+(i<<5)] = ((in[w+0] >>> 17) & 1);
- out[18+(i<<5)] = ((in[w+0] >>> 18) & 1);
- out[19+(i<<5)] = ((in[w+0] >>> 19) & 1);
- out[20+(i<<5)] = ((in[w+0] >>> 20) & 1);
- out[21+(i<<5)] = ((in[w+0] >>> 21) & 1);
- out[22+(i<<5)] = ((in[w+0] >>> 22) & 1);
- out[23+(i<<5)] = ((in[w+0] >>> 23) & 1);
- out[24+(i<<5)] = ((in[w+0] >>> 24) & 1);
- out[25+(i<<5)] = ((in[w+0] >>> 25) & 1);
- out[26+(i<<5)] = ((in[w+0] >>> 26) & 1);
- out[27+(i<<5)] = ((in[w+0] >>> 27) & 1);
- out[28+(i<<5)] = ((in[w+0] >>> 28) & 1);
- out[29+(i<<5)] = ((in[w+0] >>> 29) & 1);
- out[30+(i<<5)] = ((in[w+0] >>> 30) & 1);
- out[31+(i<<5)] = ((in[w+0] >>> 31) & 1);
+ int curInputValue0 = in[w];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 1) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 2) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 3) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 4) & mask;
+ out[5+outOffset] = (curInputValue0 >>> 5) & mask;
+ out[6+outOffset] = (curInputValue0 >>> 6) & mask;
+ out[7+outOffset] = (curInputValue0 >>> 7) & mask;
+ out[8+outOffset] = (curInputValue0 >>> 8) & mask;
+ out[9+outOffset] = (curInputValue0 >>> 9) & mask;
+ out[10+outOffset] = (curInputValue0 >>> 10) & mask;
+ out[11+outOffset] = (curInputValue0 >>> 11) & mask;
+ out[12+outOffset] = (curInputValue0 >>> 12) & mask;
+ out[13+outOffset] = (curInputValue0 >>> 13) & mask;
+ out[14+outOffset] = (curInputValue0 >>> 14) & mask;
+ out[15+outOffset] = (curInputValue0 >>> 15) & mask;
+ out[16+outOffset] = (curInputValue0 >>> 16) & mask;
+ out[17+outOffset] = (curInputValue0 >>> 17) & mask;
+ out[18+outOffset] = (curInputValue0 >>> 18) & mask;
+ out[19+outOffset] = (curInputValue0 >>> 19) & mask;
+ out[20+outOffset] = (curInputValue0 >>> 20) & mask;
+ out[21+outOffset] = (curInputValue0 >>> 21) & mask;
+ out[22+outOffset] = (curInputValue0 >>> 22) & mask;
+ out[23+outOffset] = (curInputValue0 >>> 23) & mask;
+ out[24+outOffset] = (curInputValue0 >>> 24) & mask;
+ out[25+outOffset] = (curInputValue0 >>> 25) & mask;
+ out[26+outOffset] = (curInputValue0 >>> 26) & mask;
+ out[27+outOffset] = (curInputValue0 >>> 27) & mask;
+ out[28+outOffset] = (curInputValue0 >>> 28) & mask;
+ out[29+outOffset] = (curInputValue0 >>> 29) & mask;
+ out[30+outOffset] = (curInputValue0 >>> 30) & mask;
+ out[31+outOffset] = curInputValue0 >>> 31;
+ outOffset += 32;
}
}
static private void unpack2(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 3;
for(i=0, w=1; i<4; ++i, w+=2){
- out[0+(i<<5)] = ((in[w+0]) & 3);
- out[1+(i<<5)] = ((in[w+0] >>> 2) & 3);
- out[2+(i<<5)] = ((in[w+0] >>> 4) & 3);
- out[3+(i<<5)] = ((in[w+0] >>> 6) & 3);
- out[4+(i<<5)] = ((in[w+0] >>> 8) & 3);
- out[5+(i<<5)] = ((in[w+0] >>> 10) & 3);
- out[6+(i<<5)] = ((in[w+0] >>> 12) & 3);
- out[7+(i<<5)] = ((in[w+0] >>> 14) & 3);
- out[8+(i<<5)] = ((in[w+0] >>> 16) & 3);
- out[9+(i<<5)] = ((in[w+0] >>> 18) & 3);
- out[10+(i<<5)] = ((in[w+0] >>> 20) & 3);
- out[11+(i<<5)] = ((in[w+0] >>> 22) & 3);
- out[12+(i<<5)] = ((in[w+0] >>> 24) & 3);
- out[13+(i<<5)] = ((in[w+0] >>> 26) & 3);
- out[14+(i<<5)] = ((in[w+0] >>> 28) & 3);
- out[15+(i<<5)] = ((in[w+0] >>> 30) & 3);
- out[16+(i<<5)] = ((in[w+1]) & 3);
- out[17+(i<<5)] = ((in[w+1] >>> 2) & 3);
- out[18+(i<<5)] = ((in[w+1] >>> 4) & 3);
- out[19+(i<<5)] = ((in[w+1] >>> 6) & 3);
- out[20+(i<<5)] = ((in[w+1] >>> 8) & 3);
- out[21+(i<<5)] = ((in[w+1] >>> 10) & 3);
- out[22+(i<<5)] = ((in[w+1] >>> 12) & 3);
- out[23+(i<<5)] = ((in[w+1] >>> 14) & 3);
- out[24+(i<<5)] = ((in[w+1] >>> 16) & 3);
- out[25+(i<<5)] = ((in[w+1] >>> 18) & 3);
- out[26+(i<<5)] = ((in[w+1] >>> 20) & 3);
- out[27+(i<<5)] = ((in[w+1] >>> 22) & 3);
- out[28+(i<<5)] = ((in[w+1] >>> 24) & 3);
- out[29+(i<<5)] = ((in[w+1] >>> 26) & 3);
- out[30+(i<<5)] = ((in[w+1] >>> 28) & 3);
- out[31+(i<<5)] = ((in[w+1] >>> 30) & 3);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 2) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 4) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 6) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 8) & mask;
+ out[5+outOffset] = (curInputValue0 >>> 10) & mask;
+ out[6+outOffset] = (curInputValue0 >>> 12) & mask;
+ out[7+outOffset] = (curInputValue0 >>> 14) & mask;
+ out[8+outOffset] = (curInputValue0 >>> 16) & mask;
+ out[9+outOffset] = (curInputValue0 >>> 18) & mask;
+ out[10+outOffset] = (curInputValue0 >>> 20) & mask;
+ out[11+outOffset] = (curInputValue0 >>> 22) & mask;
+ out[12+outOffset] = (curInputValue0 >>> 24) & mask;
+ out[13+outOffset] = (curInputValue0 >>> 26) & mask;
+ out[14+outOffset] = (curInputValue0 >>> 28) & mask;
+ out[15+outOffset] = curInputValue0 >>> 30;
+ out[16+outOffset] = curInputValue1 & mask;
+ out[17+outOffset] = (curInputValue1 >>> 2) & mask;
+ out[18+outOffset] = (curInputValue1 >>> 4) & mask;
+ out[19+outOffset] = (curInputValue1 >>> 6) & mask;
+ out[20+outOffset] = (curInputValue1 >>> 8) & mask;
+ out[21+outOffset] = (curInputValue1 >>> 10) & mask;
+ out[22+outOffset] = (curInputValue1 >>> 12) & mask;
+ out[23+outOffset] = (curInputValue1 >>> 14) & mask;
+ out[24+outOffset] = (curInputValue1 >>> 16) & mask;
+ out[25+outOffset] = (curInputValue1 >>> 18) & mask;
+ out[26+outOffset] = (curInputValue1 >>> 20) & mask;
+ out[27+outOffset] = (curInputValue1 >>> 22) & mask;
+ out[28+outOffset] = (curInputValue1 >>> 24) & mask;
+ out[29+outOffset] = (curInputValue1 >>> 26) & mask;
+ out[30+outOffset] = (curInputValue1 >>> 28) & mask;
+ out[31+outOffset] = curInputValue1 >>> 30;
+ outOffset += 32;
}
}
static private void unpack3(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 7;
for(i=0, w=1; i<4; ++i, w+=3){
- out[0+(i<<5)] = ((in[w+0]) & 7);
- out[1+(i<<5)] = ((in[w+0] >>> 3) & 7);
- out[2+(i<<5)] = ((in[w+0] >>> 6) & 7);
- out[3+(i<<5)] = ((in[w+0] >>> 9) & 7);
- out[4+(i<<5)] = ((in[w+0] >>> 12) & 7);
- out[5+(i<<5)] = ((in[w+0] >>> 15) & 7);
- out[6+(i<<5)] = ((in[w+0] >>> 18) & 7);
- out[7+(i<<5)] = ((in[w+0] >>> 21) & 7);
- out[8+(i<<5)] = ((in[w+0] >>> 24) & 7);
- out[9+(i<<5)] = ((in[w+0] >>> 27) & 7);
- out[10+(i<<5)] = ((in[w+0] >>> 30) & 7);
- out[10+(i<<5)] |= ((in[w+1] << 2) & 7);
- out[11+(i<<5)] = ((in[w+1] >>> 1) & 7);
- out[12+(i<<5)] = ((in[w+1] >>> 4) & 7);
- out[13+(i<<5)] = ((in[w+1] >>> 7) & 7);
- out[14+(i<<5)] = ((in[w+1] >>> 10) & 7);
- out[15+(i<<5)] = ((in[w+1] >>> 13) & 7);
- out[16+(i<<5)] = ((in[w+1] >>> 16) & 7);
- out[17+(i<<5)] = ((in[w+1] >>> 19) & 7);
- out[18+(i<<5)] = ((in[w+1] >>> 22) & 7);
- out[19+(i<<5)] = ((in[w+1] >>> 25) & 7);
- out[20+(i<<5)] = ((in[w+1] >>> 28) & 7);
- out[21+(i<<5)] = ((in[w+1] >>> 31) & 7);
- out[21+(i<<5)] |= ((in[w+2] << 1) & 7);
- out[22+(i<<5)] = ((in[w+2] >>> 2) & 7);
- out[23+(i<<5)] = ((in[w+2] >>> 5) & 7);
- out[24+(i<<5)] = ((in[w+2] >>> 8) & 7);
- out[25+(i<<5)] = ((in[w+2] >>> 11) & 7);
- out[26+(i<<5)] = ((in[w+2] >>> 14) & 7);
- out[27+(i<<5)] = ((in[w+2] >>> 17) & 7);
- out[28+(i<<5)] = ((in[w+2] >>> 20) & 7);
- out[29+(i<<5)] = ((in[w+2] >>> 23) & 7);
- out[30+(i<<5)] = ((in[w+2] >>> 26) & 7);
- out[31+(i<<5)] = ((in[w+2] >>> 29) & 7);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ int curInputValue2 = in[w+2];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 3) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 6) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 9) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 12) & mask;
+ out[5+outOffset] = (curInputValue0 >>> 15) & mask;
+ out[6+outOffset] = (curInputValue0 >>> 18) & mask;
+ out[7+outOffset] = (curInputValue0 >>> 21) & mask;
+ out[8+outOffset] = (curInputValue0 >>> 24) & mask;
+ out[9+outOffset] = (curInputValue0 >>> 27) & mask;
+ out[10+outOffset] = ((curInputValue0 >>> 30) | (curInputValue1 << 2)) & mask;
+ out[11+outOffset] = (curInputValue1 >>> 1) & mask;
+ out[12+outOffset] = (curInputValue1 >>> 4) & mask;
+ out[13+outOffset] = (curInputValue1 >>> 7) & mask;
+ out[14+outOffset] = (curInputValue1 >>> 10) & mask;
+ out[15+outOffset] = (curInputValue1 >>> 13) & mask;
+ out[16+outOffset] = (curInputValue1 >>> 16) & mask;
+ out[17+outOffset] = (curInputValue1 >>> 19) & mask;
+ out[18+outOffset] = (curInputValue1 >>> 22) & mask;
+ out[19+outOffset] = (curInputValue1 >>> 25) & mask;
+ out[20+outOffset] = (curInputValue1 >>> 28) & mask;
+ out[21+outOffset] = ((curInputValue1 >>> 31) | (curInputValue2 << 1)) & mask;
+ out[22+outOffset] = (curInputValue2 >>> 2) & mask;
+ out[23+outOffset] = (curInputValue2 >>> 5) & mask;
+ out[24+outOffset] = (curInputValue2 >>> 8) & mask;
+ out[25+outOffset] = (curInputValue2 >>> 11) & mask;
+ out[26+outOffset] = (curInputValue2 >>> 14) & mask;
+ out[27+outOffset] = (curInputValue2 >>> 17) & mask;
+ out[28+outOffset] = (curInputValue2 >>> 20) & mask;
+ out[29+outOffset] = (curInputValue2 >>> 23) & mask;
+ out[30+outOffset] = (curInputValue2 >>> 26) & mask;
+ out[31+outOffset] = curInputValue2 >>> 29;
+ outOffset += 32;
}
}
static private void unpack4(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 15;
for(i=0, w=1; i<4; ++i, w+=4){
- out[0+(i<<5)] = ((in[w+0]) & 15);
- out[1+(i<<5)] = ((in[w+0] >>> 4) & 15);
- out[2+(i<<5)] = ((in[w+0] >>> 8) & 15);
- out[3+(i<<5)] = ((in[w+0] >>> 12) & 15);
- out[4+(i<<5)] = ((in[w+0] >>> 16) & 15);
- out[5+(i<<5)] = ((in[w+0] >>> 20) & 15);
- out[6+(i<<5)] = ((in[w+0] >>> 24) & 15);
- out[7+(i<<5)] = ((in[w+0] >>> 28) & 15);
- out[8+(i<<5)] = ((in[w+1]) & 15);
- out[9+(i<<5)] = ((in[w+1] >>> 4) & 15);
- out[10+(i<<5)] = ((in[w+1] >>> 8) & 15);
- out[11+(i<<5)] = ((in[w+1] >>> 12) & 15);
- out[12+(i<<5)] = ((in[w+1] >>> 16) & 15);
- out[13+(i<<5)] = ((in[w+1] >>> 20) & 15);
- out[14+(i<<5)] = ((in[w+1] >>> 24) & 15);
- out[15+(i<<5)] = ((in[w+1] >>> 28) & 15);
- out[16+(i<<5)] = ((in[w+2]) & 15);
- out[17+(i<<5)] = ((in[w+2] >>> 4) & 15);
- out[18+(i<<5)] = ((in[w+2] >>> 8) & 15);
- out[19+(i<<5)] = ((in[w+2] >>> 12) & 15);
- out[20+(i<<5)] = ((in[w+2] >>> 16) & 15);
- out[21+(i<<5)] = ((in[w+2] >>> 20) & 15);
- out[22+(i<<5)] = ((in[w+2] >>> 24) & 15);
- out[23+(i<<5)] = ((in[w+2] >>> 28) & 15);
- out[24+(i<<5)] = ((in[w+3]) & 15);
- out[25+(i<<5)] = ((in[w+3] >>> 4) & 15);
- out[26+(i<<5)] = ((in[w+3] >>> 8) & 15);
- out[27+(i<<5)] = ((in[w+3] >>> 12) & 15);
- out[28+(i<<5)] = ((in[w+3] >>> 16) & 15);
- out[29+(i<<5)] = ((in[w+3] >>> 20) & 15);
- out[30+(i<<5)] = ((in[w+3] >>> 24) & 15);
- out[31+(i<<5)] = ((in[w+3] >>> 28) & 15);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ int curInputValue2 = in[w+2];
+ int curInputValue3 = in[w+3];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 4) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 8) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 12) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 16) & mask;
+ out[5+outOffset] = (curInputValue0 >>> 20) & mask;
+ out[6+outOffset] = (curInputValue0 >>> 24) & mask;
+ out[7+outOffset] = curInputValue0 >>> 28;
+ out[8+outOffset] = curInputValue1 & mask;
+ out[9+outOffset] = (curInputValue1 >>> 4) & mask;
+ out[10+outOffset] = (curInputValue1 >>> 8) & mask;
+ out[11+outOffset] = (curInputValue1 >>> 12) & mask;
+ out[12+outOffset] = (curInputValue1 >>> 16) & mask;
+ out[13+outOffset] = (curInputValue1 >>> 20) & mask;
+ out[14+outOffset] = (curInputValue1 >>> 24) & mask;
+ out[15+outOffset] = curInputValue1 >>> 28;
+ out[16+outOffset] = curInputValue2 & mask;
+ out[17+outOffset] = (curInputValue2 >>> 4) & mask;
+ out[18+outOffset] = (curInputValue2 >>> 8) & mask;
+ out[19+outOffset] = (curInputValue2 >>> 12) & mask;
+ out[20+outOffset] = (curInputValue2 >>> 16) & mask;
+ out[21+outOffset] = (curInputValue2 >>> 20) & mask;
+ out[22+outOffset] = (curInputValue2 >>> 24) & mask;
+ out[23+outOffset] = curInputValue2 >>> 28;
+ out[24+outOffset] = curInputValue3 & mask;
+ out[25+outOffset] = (curInputValue3 >>> 4) & mask;
+ out[26+outOffset] = (curInputValue3 >>> 8) & mask;
+ out[27+outOffset] = (curInputValue3 >>> 12) & mask;
+ out[28+outOffset] = (curInputValue3 >>> 16) & mask;
+ out[29+outOffset] = (curInputValue3 >>> 20) & mask;
+ out[30+outOffset] = (curInputValue3 >>> 24) & mask;
+ out[31+outOffset] = curInputValue3 >>> 28;
+ outOffset += 32;
}
}
static private void unpack5(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 31;
for(i=0, w=1; i<4; ++i, w+=5){
- out[0+(i<<5)] = ((in[w+0]) & 31);
- out[1+(i<<5)] = ((in[w+0] >>> 5) & 31);
- out[2+(i<<5)] = ((in[w+0] >>> 10) & 31);
- out[3+(i<<5)] = ((in[w+0] >>> 15) & 31);
- out[4+(i<<5)] = ((in[w+0] >>> 20) & 31);
- out[5+(i<<5)] = ((in[w+0] >>> 25) & 31);
- out[6+(i<<5)] = ((in[w+0] >>> 30) & 31);
- out[6+(i<<5)] |= ((in[w+1] << 2) & 31);
- out[7+(i<<5)] = ((in[w+1] >>> 3) & 31);
- out[8+(i<<5)] = ((in[w+1] >>> 8) & 31);
- out[9+(i<<5)] = ((in[w+1] >>> 13) & 31);
- out[10+(i<<5)] = ((in[w+1] >>> 18) & 31);
- out[11+(i<<5)] = ((in[w+1] >>> 23) & 31);
- out[12+(i<<5)] = ((in[w+1] >>> 28) & 31);
- out[12+(i<<5)] |= ((in[w+2] << 4) & 31);
- out[13+(i<<5)] = ((in[w+2] >>> 1) & 31);
- out[14+(i<<5)] = ((in[w+2] >>> 6) & 31);
- out[15+(i<<5)] = ((in[w+2] >>> 11) & 31);
- out[16+(i<<5)] = ((in[w+2] >>> 16) & 31);
- out[17+(i<<5)] = ((in[w+2] >>> 21) & 31);
- out[18+(i<<5)] = ((in[w+2] >>> 26) & 31);
- out[19+(i<<5)] = ((in[w+2] >>> 31) & 31);
- out[19+(i<<5)] |= ((in[w+3] << 1) & 31);
- out[20+(i<<5)] = ((in[w+3] >>> 4) & 31);
- out[21+(i<<5)] = ((in[w+3] >>> 9) & 31);
- out[22+(i<<5)] = ((in[w+3] >>> 14) & 31);
- out[23+(i<<5)] = ((in[w+3] >>> 19) & 31);
- out[24+(i<<5)] = ((in[w+3] >>> 24) & 31);
- out[25+(i<<5)] = ((in[w+3] >>> 29) & 31);
- out[25+(i<<5)] |= ((in[w+4] << 3) & 31);
- out[26+(i<<5)] = ((in[w+4] >>> 2) & 31);
- out[27+(i<<5)] = ((in[w+4] >>> 7) & 31);
- out[28+(i<<5)] = ((in[w+4] >>> 12) & 31);
- out[29+(i<<5)] = ((in[w+4] >>> 17) & 31);
- out[30+(i<<5)] = ((in[w+4] >>> 22) & 31);
- out[31+(i<<5)] = ((in[w+4] >>> 27) & 31);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ int curInputValue2 = in[w+2];
+ int curInputValue3 = in[w+3];
+ int curInputValue4 = in[w+4];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 5) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 10) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 15) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 20) & mask;
+ out[5+outOffset] = (curInputValue0 >>> 25) & mask;
+ out[6+outOffset] = ((curInputValue0 >>> 30) | (curInputValue1 << 2)) & mask;
+ out[7+outOffset] = (curInputValue1 >>> 3) & mask;
+ out[8+outOffset] = (curInputValue1 >>> 8) & mask;
+ out[9+outOffset] = (curInputValue1 >>> 13) & mask;
+ out[10+outOffset] = (curInputValue1 >>> 18) & mask;
+ out[11+outOffset] = (curInputValue1 >>> 23) & mask;
+ out[12+outOffset] = ((curInputValue1 >>> 28) | (curInputValue2 << 4)) & mask;
+ out[13+outOffset] = (curInputValue2 >>> 1) & mask;
+ out[14+outOffset] = (curInputValue2 >>> 6) & mask;
+ out[15+outOffset] = (curInputValue2 >>> 11) & mask;
+ out[16+outOffset] = (curInputValue2 >>> 16) & mask;
+ out[17+outOffset] = (curInputValue2 >>> 21) & mask;
+ out[18+outOffset] = (curInputValue2 >>> 26) & mask;
+ out[19+outOffset] = ((curInputValue2 >>> 31) | (curInputValue3 << 1)) & mask;
+ out[20+outOffset] = (curInputValue3 >>> 4) & mask;
+ out[21+outOffset] = (curInputValue3 >>> 9) & mask;
+ out[22+outOffset] = (curInputValue3 >>> 14) & mask;
+ out[23+outOffset] = (curInputValue3 >>> 19) & mask;
+ out[24+outOffset] = (curInputValue3 >>> 24) & mask;
+ out[25+outOffset] = ((curInputValue3 >>> 29) | (curInputValue4 << 3)) & mask;
+ out[26+outOffset] = (curInputValue4 >>> 2) & mask;
+ out[27+outOffset] = (curInputValue4 >>> 7) & mask;
+ out[28+outOffset] = (curInputValue4 >>> 12) & mask;
+ out[29+outOffset] = (curInputValue4 >>> 17) & mask;
+ out[30+outOffset] = (curInputValue4 >>> 22) & mask;
+ out[31+outOffset] = curInputValue4 >>> 27;
+ outOffset += 32;
}
}
static private void unpack6(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 63;
for(i=0, w=1; i<4; ++i, w+=6){
- out[0+(i<<5)] = ((in[w+0]) & 63);
- out[1+(i<<5)] = ((in[w+0] >>> 6) & 63);
- out[2+(i<<5)] = ((in[w+0] >>> 12) & 63);
- out[3+(i<<5)] = ((in[w+0] >>> 18) & 63);
- out[4+(i<<5)] = ((in[w+0] >>> 24) & 63);
- out[5+(i<<5)] = ((in[w+0] >>> 30) & 63);
- out[5+(i<<5)] |= ((in[w+1] << 2) & 63);
- out[6+(i<<5)] = ((in[w+1] >>> 4) & 63);
- out[7+(i<<5)] = ((in[w+1] >>> 10) & 63);
- out[8+(i<<5)] = ((in[w+1] >>> 16) & 63);
- out[9+(i<<5)] = ((in[w+1] >>> 22) & 63);
- out[10+(i<<5)] = ((in[w+1] >>> 28) & 63);
- out[10+(i<<5)] |= ((in[w+2] << 4) & 63);
- out[11+(i<<5)] = ((in[w+2] >>> 2) & 63);
- out[12+(i<<5)] = ((in[w+2] >>> 8) & 63);
- out[13+(i<<5)] = ((in[w+2] >>> 14) & 63);
- out[14+(i<<5)] = ((in[w+2] >>> 20) & 63);
- out[15+(i<<5)] = ((in[w+2] >>> 26) & 63);
- out[16+(i<<5)] = ((in[w+3]) & 63);
- out[17+(i<<5)] = ((in[w+3] >>> 6) & 63);
- out[18+(i<<5)] = ((in[w+3] >>> 12) & 63);
- out[19+(i<<5)] = ((in[w+3] >>> 18) & 63);
- out[20+(i<<5)] = ((in[w+3] >>> 24) & 63);
- out[21+(i<<5)] = ((in[w+3] >>> 30) & 63);
- out[21+(i<<5)] |= ((in[w+4] << 2) & 63);
- out[22+(i<<5)] = ((in[w+4] >>> 4) & 63);
- out[23+(i<<5)] = ((in[w+4] >>> 10) & 63);
- out[24+(i<<5)] = ((in[w+4] >>> 16) & 63);
- out[25+(i<<5)] = ((in[w+4] >>> 22) & 63);
- out[26+(i<<5)] = ((in[w+4] >>> 28) & 63);
- out[26+(i<<5)] |= ((in[w+5] << 4) & 63);
- out[27+(i<<5)] = ((in[w+5] >>> 2) & 63);
- out[28+(i<<5)] = ((in[w+5] >>> 8) & 63);
- out[29+(i<<5)] = ((in[w+5] >>> 14) & 63);
- out[30+(i<<5)] = ((in[w+5] >>> 20) & 63);
- out[31+(i<<5)] = ((in[w+5] >>> 26) & 63);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ int curInputValue2 = in[w+2];
+ int curInputValue3 = in[w+3];
+ int curInputValue4 = in[w+4];
+ int curInputValue5 = in[w+5];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 6) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 12) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 18) & mask;
+ out[4+outOffset] = (curInputValue0 >>> 24) & mask;
+ out[5+outOffset] = ((curInputValue0 >>> 30) | (curInputValue1 << 2)) & mask;
+ out[6+outOffset] = (curInputValue1 >>> 4) & mask;
+ out[7+outOffset] = (curInputValue1 >>> 10) & mask;
+ out[8+outOffset] = (curInputValue1 >>> 16) & mask;
+ out[9+outOffset] = (curInputValue1 >>> 22) & mask;
+ out[10+outOffset] = ((curInputValue1 >>> 28) | (curInputValue2 << 4)) & mask;
+ out[11+outOffset] = (curInputValue2 >>> 2) & mask;
+ out[12+outOffset] = (curInputValue2 >>> 8) & mask;
+ out[13+outOffset] = (curInputValue2 >>> 14) & mask;
+ out[14+outOffset] = (curInputValue2 >>> 20) & mask;
+ out[15+outOffset] = curInputValue2 >>> 26;
+ out[16+outOffset] = curInputValue3 & mask;
+ out[17+outOffset] = (curInputValue3 >>> 6) & mask;
+ out[18+outOffset] = (curInputValue3 >>> 12) & mask;
+ out[19+outOffset] = (curInputValue3 >>> 18) & mask;
+ out[20+outOffset] = (curInputValue3 >>> 24) & mask;
+ out[21+outOffset] = ((curInputValue3 >>> 30) | (curInputValue4 << 2)) & mask;
+ out[22+outOffset] = (curInputValue4 >>> 4) & mask;
+ out[23+outOffset] = (curInputValue4 >>> 10) & mask;
+ out[24+outOffset] = (curInputValue4 >>> 16) & mask;
+ out[25+outOffset] = (curInputValue4 >>> 22) & mask;
+ out[26+outOffset] = ((curInputValue4 >>> 28) | (curInputValue5 << 4)) & mask;
+ out[27+outOffset] = (curInputValue5 >>> 2) & mask;
+ out[28+outOffset] = (curInputValue5 >>> 8) & mask;
+ out[29+outOffset] = (curInputValue5 >>> 14) & mask;
+ out[30+outOffset] = (curInputValue5 >>> 20) & mask;
+ out[31+outOffset] = curInputValue5 >>> 26;
+ outOffset += 32;
}
}
static private void unpack7(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 127;
for(i=0, w=1; i<4; ++i, w+=7){
- out[0+(i<<5)] = ((in[w+0]) & 127);
- out[1+(i<<5)] = ((in[w+0] >>> 7) & 127);
- out[2+(i<<5)] = ((in[w+0] >>> 14) & 127);
- out[3+(i<<5)] = ((in[w+0] >>> 21) & 127);
- out[4+(i<<5)] = ((in[w+0] >>> 28) & 127);
- out[4+(i<<5)] |= ((in[w+1] << 4) & 127);
- out[5+(i<<5)] = ((in[w+1] >>> 3) & 127);
- out[6+(i<<5)] = ((in[w+1] >>> 10) & 127);
- out[7+(i<<5)] = ((in[w+1] >>> 17) & 127);
- out[8+(i<<5)] = ((in[w+1] >>> 24) & 127);
- out[9+(i<<5)] = ((in[w+1] >>> 31) & 127);
- out[9+(i<<5)] |= ((in[w+2] << 1) & 127);
- out[10+(i<<5)] = ((in[w+2] >>> 6) & 127);
- out[11+(i<<5)] = ((in[w+2] >>> 13) & 127);
- out[12+(i<<5)] = ((in[w+2] >>> 20) & 127);
- out[13+(i<<5)] = ((in[w+2] >>> 27) & 127);
- out[13+(i<<5)] |= ((in[w+3] << 5) & 127);
- out[14+(i<<5)] = ((in[w+3] >>> 2) & 127);
- out[15+(i<<5)] = ((in[w+3] >>> 9) & 127);
- out[16+(i<<5)] = ((in[w+3] >>> 16) & 127);
- out[17+(i<<5)] = ((in[w+3] >>> 23) & 127);
- out[18+(i<<5)] = ((in[w+3] >>> 30) & 127);
- out[18+(i<<5)] |= ((in[w+4] << 2) & 127);
- out[19+(i<<5)] = ((in[w+4] >>> 5) & 127);
- out[20+(i<<5)] = ((in[w+4] >>> 12) & 127);
- out[21+(i<<5)] = ((in[w+4] >>> 19) & 127);
- out[22+(i<<5)] = ((in[w+4] >>> 26) & 127);
- out[22+(i<<5)] |= ((in[w+5] << 6) & 127);
- out[23+(i<<5)] = ((in[w+5] >>> 1) & 127);
- out[24+(i<<5)] = ((in[w+5] >>> 8) & 127);
- out[25+(i<<5)] = ((in[w+5] >>> 15) & 127);
- out[26+(i<<5)] = ((in[w+5] >>> 22) & 127);
- out[27+(i<<5)] = ((in[w+5] >>> 29) & 127);
- out[27+(i<<5)] |= ((in[w+6] << 3) & 127);
- out[28+(i<<5)] = ((in[w+6] >>> 4) & 127);
- out[29+(i<<5)] = ((in[w+6] >>> 11) & 127);
- out[30+(i<<5)] = ((in[w+6] >>> 18) & 127);
- out[31+(i<<5)] = ((in[w+6] >>> 25) & 127);
+ int curInputValue0 = in[w];
+ int curInputValue1 = in[w+1];
+ int curInputValue2 = in[w+2];
+ int curInputValue3 = in[w+3];
+ int curInputValue4 = in[w+4];
+ int curInputValue5 = in[w+5];
+ int curInputValue6 = in[w+6];
+ out[0+outOffset] = curInputValue0 & mask;
+ out[1+outOffset] = (curInputValue0 >>> 7) & mask;
+ out[2+outOffset] = (curInputValue0 >>> 14) & mask;
+ out[3+outOffset] = (curInputValue0 >>> 21) & mask;
+ out[4+outOffset] = ((curInputValue0 >>> 28) | (curInputValue1 << 4)) & mask;
+ out[5+outOffset] = (curInputValue1 >>> 3) & mask;
+ out[6+outOffset] = (curInputValue1 >>> 10) & mask;
+ out[7+outOffset] = (curInputValue1 >>> 17) & mask;
+ out[8+outOffset] = (curInputValue1 >>> 24) & mask;
+ out[9+outOffset] = ((curInputValue1 >>> 31) | (curInputValue2 << 1)) & mask;
+ out[10+outOffset] = (curInputValue2 >>> 6) & mask;
+ out[11+outOffset] = (curInputValue2 >>> 13) & mask;
+ out[12+outOffset] = (curInputValue2 >>> 20) & mask;
+ out[13+outOffset] = ((curInputValue2 >>> 27) | (curInputValue3 << 5)) & mask;
+ out[14+outOffset] = (curInputValue3 >>> 2) & mask;
+ out[15+outOffset] = (curInputValue3 >>> 9) & mask;
+ out[16+outOffset] = (curInputValue3 >>> 16) & mask;
+ out[17+outOffset] = (curInputValue3 >>> 23) & mask;
+ out[18+outOffset] = ((curInputValue3 >>> 30) | (curInputValue4 << 2)) & mask;
+ out[19+outOffset] = (curInputValue4 >>> 5) & mask;
+ out[20+outOffset] = (curInputValue4 >>> 12) & mask;
+ out[21+outOffset] = (curInputValue4 >>> 19) & mask;
+ out[22+outOffset] = ((curInputValue4 >>> 26) | (curInputValue5 << 6)) & mask;
+ out[23+outOffset] = (curInputValue5 >>> 1) & mask;
+ out[24+outOffset] = (curInputValue5 >>> 8) & mask;
+ out[25+outOffset] = (curInputValue5 >>> 15) & mask;
+ out[26+outOffset] = (curInputValue5 >>> 22) & mask;
+ out[27+outOffset] = ((curInputValue5 >>> 29) | (curInputValue6 << 3)) & mask;
+ out[28+outOffset] = (curInputValue6 >>> 4) & mask;
+ out[29+outOffset] = (curInputValue6 >>> 11) & mask;
+ out[30+outOffset] = (curInputValue6 >>> 18) & mask;
+ out[31+outOffset] = curInputValue6 >>> 25;
+ outOffset += 32;
}
}
static private void unpack8(int[] out, int[] in)
{
int i, w;
+ int outOffset = 0;
+ final int mask = 255;
for(i=0, w=1; i<4; ++i, w+=8){
- out[0+(i<<5)] = ((in[w+0]) & 255);
- out[1+(i<<5)] = ((in[w+0] >>> 8) & 255);
- out[2+(i<<5)] = ((in[w+0] >>> 16) & 255);
- out[3+(i<<5)] = ((in[w+0] >>> 24) & 255);
- out[4+(i<<5)] = ((in[w+1]) & 255);
- out[5+(i<<5)] = ((in[w+1] >>> 8) & 255);
- out[6+(i<<5)] = ((in[w+1] >>> 16) & 255);
- out[7+(i<<5)] = ((in[w+1] >>> 24) & 255);
- out[8+(i<<5)] = ((in[w+2]) & 255);
- out[9+(i<<5)] = ((in[w+2] >>> 8) & 255);
- out[10+(i<<5)] = ((in[w+2] >>> 16) & 255);
- out[11+(i<<5)] = ((in[w+2] >>> 24) & 255);
- out[12+(i<<5)] = ((in[w+3]) & 255);
- out[13+(i<<5)] = ((in[w+3] >>> 8) & 255);
- out[14+(i<<5)] = ((in[w+3] >>> 16) & 255);
- out[15+(i<<