Calculate max compressed length

example.java

@@ -104,11 +104,7 @@ public static void basicExampleHeadless() {
         // be processed using variable byte
         SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
                 new IntegratedVariableByte());
-        // output vector should be large enough...
-        int[] compressed = new int[data.length + 1024];
-        // compressed might not be large enough in some cases
-        // if you get java.lang.ArrayIndexOutOfBoundsException, try
-        // allocating more memory
+        int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
 
         /**
          *
@@ -267,10 +263,12 @@ public static void headlessDemo() {
         int[] uncompressed1 = {1,2,1,3,1};
         int[] uncompressed2 = {3,2,4,6,1};
 
-        int[] compressed = new int[uncompressed1.length+uncompressed2.length+1024];
-
         SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
 
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed1.length)
+                                  + codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed2.length);
+        int[] compressed = new int[maxCompressedLength];
+
         // compressing
         IntWrapper outPos = new IntWrapper();
 

src/main/java/me/lemire/integercompression/BinaryPacking.java

@@ -37,8 +37,9 @@
  * @author Daniel Lemire
  */
 public final class BinaryPacking implements IntegerCODEC, SkippableIntegerCODEC {
-        final static int BLOCK_SIZE = 32;
-
+        public final static int BLOCK_SIZE = 32;
+        private static final int MAX_BIT_WIDTH = Integer.SIZE;
+
         @Override
         public void compress(int[] in, IntWrapper inpos, int inlength,
                 int[] out, IntWrapper outpos) {
@@ -131,7 +132,16 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
             outpos.add(outlength);
             inpos.set(tmpinpos);
         }
-
+
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            int blockCount = inlength / BLOCK_SIZE;
+            int headersSizeInInts = blockCount / Integer.BYTES + (blockCount % Integer.BYTES);
+            int blocksSizeInInts = blockCount * MAX_BIT_WIDTH;
+            compressedPositions.add(blockCount * BLOCK_SIZE);
+            return headersSizeInInts + blocksSizeInInts;
+        }
+
         @Override
         public String toString() {
                 return this.getClass().getSimpleName();

src/main/java/me/lemire/integercompression/FastPFOR.java

@@ -40,6 +40,13 @@
  */
 public class FastPFOR implements IntegerCODEC,SkippableIntegerCODEC {
         final static int OVERHEAD_OF_EACH_EXCEPT = 8;
+        private static final int OVERHEAD_OF_EACH_PAGE_IN_INTS = 36; // 1 int for the header
+                                                                     // 1 int for the byte array size
+                                                                     // 1 int for the bitmap
+                                                                     // 1 int for byte array padding (to align to 4 bytes)
+                                                                     // 32 to have enough space to bit-pack the exceptions
+        private static final int OVERHEAD_OF_EACH_BLOCK_IN_INTS = 1; // 1 byte for the number of bits allocated per truncated integer
+                                                                     // 1 byte for the number of exceptions
         /**
          *
          */
@@ -65,7 +72,7 @@ public class FastPFOR implements IntegerCODEC,SkippableIntegerCODEC {
          * @param pagesize
          *                the desired page size (recommended value is FastPFOR.DEFAULT_PAGE_SIZE)
          */
-        private FastPFOR(int pagesize) {
+        FastPFOR(int pagesize) {
             pageSize = pagesize;
             // Initiate arrrays.
             byteContainer = makeBuffer(3 * pageSize
@@ -230,6 +237,18 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 }
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+
+            int pageCount = (inlength + pageSize - 1) / pageSize;
+            int blockCount = inlength / BLOCK_SIZE;
+
+            // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+            int blockSizeInInts = OVERHEAD_OF_EACH_BLOCK_IN_INTS + BLOCK_SIZE;
+            return OVERHEAD_OF_EACH_PAGE_IN_INTS * pageCount + blockSizeInInts * blockCount + 24;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 final int initpos = inpos.get();

src/main/java/me/lemire/integercompression/FastPFOR128.java

@@ -23,6 +23,13 @@
  */
 public class FastPFOR128 implements IntegerCODEC,SkippableIntegerCODEC {
         final static int OVERHEAD_OF_EACH_EXCEPT = 8;
+        private static final int OVERHEAD_OF_EACH_PAGE_IN_INTS = 36; // 1 int for the header
+                                                                     // 1 int for the byte array size
+                                                                     // 1 int for the bitmap
+                                                                     // 1 int for byte array padding (to align to 4 bytes)
+                                                                     // 32 to have enough space to bit-pack the exceptions
+        private static final int OVERHEAD_OF_EACH_BLOCK_IN_INTS = 1; // 1 byte for the number of bits allocated per truncated integer
+                                                                     // 1 byte for the number of exceptions
         /**
          *
          */
@@ -209,6 +216,18 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 }
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+
+            int pageCount = (inlength + pageSize - 1) / pageSize;
+            int blockCount = inlength / BLOCK_SIZE;
+
+            // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+            int blockSizeInInts = OVERHEAD_OF_EACH_BLOCK_IN_INTS + BLOCK_SIZE;
+            return OVERHEAD_OF_EACH_PAGE_IN_INTS * pageCount + blockSizeInInts * blockCount + 24;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 final int initpos = inpos.get();

src/main/java/me/lemire/integercompression/GroupSimple9.java

@@ -3549,4 +3549,10 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         inpos.set(tmpinpos);
 
     }
-}
+
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        compressedPositions.add(inlength);
+        return inlength;
+    }
+}

src/main/java/me/lemire/integercompression/IntCompressor.java

@@ -33,21 +33,14 @@ public IntCompressor() {
      * 
      * @param input array to be compressed
      * @return compressed array
-     * @throws UncompressibleInputException if the data is too poorly compressible
      */
     public  int[] compress(int[] input) {
-        int[] compressed = new int[input.length + input.length / 100 + 1024];
+        int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+        int[] compressed = new int[maxCompressedLength + 1]; // +1 to store the length of the input
         // Store at index=0 the length of the input, hence enabling .headlessCompress
         compressed[0] = input.length;
         IntWrapper outpos = new IntWrapper(1);
-        try {
-            codec.headlessCompress(input, new IntWrapper(0),
-                    input.length, compressed, outpos);
-        } catch (IndexOutOfBoundsException ioebe) {
-            throw new
-            UncompressibleInputException("Your input is too poorly compressible "
-                    + "with the current codec : "+codec);
-        }
+        codec.headlessCompress(input, new IntWrapper(0), input.length, compressed, outpos);
         compressed = Arrays.copyOf(compressed,outpos.intValue());
         return compressed;
     }

src/main/java/me/lemire/integercompression/JustCopy.java

@@ -42,6 +42,12 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
 
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+            compressedPositions.add(inlength);
+            return inlength;
+        }
+
         @Override
         public void compress(int[] in, IntWrapper inpos, int inlength,
                 int[] out, IntWrapper outpos) {

src/main/java/me/lemire/integercompression/Kamikaze.java

@@ -38,6 +38,11 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         }
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        throw new UnsupportedOperationException("Calculating the max compressed length is not supported yet.");
+    }
+
     @Override
     public String toString() {
         return "Kamikaze's PForDelta";
@@ -64,4 +69,4 @@ public void uncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
         headlessUncompress(in, inpos, inlength, out, outpos, outlength);
 
     }
-}
+}

src/main/java/me/lemire/integercompression/NewPFD.java

@@ -132,6 +132,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();

src/main/java/me/lemire/integercompression/NewPFDS16.java

@@ -131,6 +131,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();

src/main/java/me/lemire/integercompression/NewPFDS9.java

@@ -130,6 +130,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();

src/main/java/me/lemire/integercompression/OptPFD.java

@@ -147,6 +147,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();

src/main/java/me/lemire/integercompression/OptPFDS16.java

@@ -147,6 +147,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();
@@ -197,4 +208,4 @@ public String toString() {
                 return this.getClass().getSimpleName();
         }
 
-}
+}

src/main/java/me/lemire/integercompression/OptPFDS9.java

@@ -146,6 +146,17 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
                 decodePage(in, inpos, out, outpos, mynvalue);
         }
 
+        @Override
+        public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+                inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+                int blockCount = inlength / BLOCK_SIZE;
+                // +1 for the header
+                // getBestBFromData limits the memory used for exceptions so that the total size of the block does not exceed BLOCK_SIZE integers.
+                int maxBlockSizeInInts = 1 + BLOCK_SIZE;
+                compressedPositions.add(inlength);
+                return maxBlockSizeInInts * blockCount;
+        }
+
         private void decodePage(int[] in, IntWrapper inpos, int[] out,
                 IntWrapper outpos, int thissize) {
                 int tmpoutpos = outpos.get();
@@ -197,4 +208,4 @@ public String toString() {
                 return this.getClass().getSimpleName();
         }
 
-}
+}

src/main/java/me/lemire/integercompression/Simple16.java

@@ -103,6 +103,12 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         outpos.set(i_outpos);
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        compressedPositions.add(inlength);
+        return inlength;
+    }
+
     /**
      * Uncompress data from an array to another array.
      * 
@@ -182,4 +188,4 @@ public String toString() {
             { 7, 7, 7, 7 }, { 10, 9, 9, }, { 14, 14 }, { 28 } };
     private static final int[][] SHIFTED_S16_BITS = shiftme(S16_BITS);
 
-}
+}

src/main/java/me/lemire/integercompression/Simple9.java

@@ -268,6 +268,12 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
 
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        compressedPositions.add(inlength);
+        return inlength;
+    }
+
     @Override
     public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
         if (inlength == 0)

src/main/java/me/lemire/integercompression/SkippableComposition.java

@@ -61,6 +61,16 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
         F2.headlessUncompress(in, inpos, inlength, out, outpos, num);
     }
 
+    @Override
+    public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+        int init = compressedPositions.get();
+        int maxLength = F1.maxHeadlessCompressedLength(compressedPositions, inlength);
+        maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+        inlength -= compressedPositions.get() - init;
+        maxLength += F2.maxHeadlessCompressedLength(compressedPositions, inlength);
+        return maxLength;
+    }
+
     @Override
     public String toString() {
         return F1.toString() + "+" + F2.toString();

src/main/java/me/lemire/integercompression/SkippableIntegerCODEC.java

@@ -69,4 +69,21 @@ public void headlessCompress(int[] in, IntWrapper inpos, int inlength, int[] out
     public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
             IntWrapper outpos, int num);
 
+    /**
+     * Compute the maximum number of integers that might be required to store
+     * the compressed form of a given input array segment, without headers.
+     * <p>
+     * This is useful to pre-allocate the output buffer before calling
+     * {@link #headlessCompress(int[], IntWrapper, int, int[], IntWrapper)}.
+     * </p>
+     *
+     * @param compressedPositions
+     *        since not all schemes compress every input integer, this parameter
+     *        returns how many input integers will actually be compressed.
+     *        This is useful when composing multiple schemes.
+     * @param inlength
+     *            number of integers to be compressed
+     * @return the maximum number of integers needed in the output array
+     */
+    int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
 }