Merge 5302a7c into 1f99ab6

roaringbitmap/src/main/java/org/roaringbitmap/RoaringArray.java

@@ -101,6 +101,7 @@ protected int advanceUntil(short x, int pos) {
   }
 
   protected void append(short key, Container value) {
+    assert !value.isEmpty();
     extendArray(1);
     this.keys[this.size] = key;
     this.values[this.size] = value;

roaringbitmap/src/main/java/org/roaringbitmap/RunContainer.java

@@ -2217,6 +2217,7 @@ void smartAppend(short start, short length) {
     if ((nbrruns == 0) || (toIntUnsigned(start) > (oldend =
         toIntUnsigned(getValue(nbrruns - 1)) + toIntUnsigned(getLength(nbrruns - 1)))
         + 1)) { // we add a new one
+      ensureCapacity(nbrruns + 1);
       valueslength[2 * nbrruns] = start;
       valueslength[2 * nbrruns + 1] = length;
       nbrruns++;

roaringbitmap/src/main/java/org/roaringbitmap/Util.java

@@ -39,7 +39,7 @@ public static  Container[] addOffset(Container source, short offsets) {
       for(int k = 0; k < c.cardinality; k++) {
         int val =  Util.toIntUnsigned(c.content[k]);
         val += offset;
-        if(val < 0xFFFF) {
+        if(val <= 0xFFFF) {
           low.content[low.cardinality++] = (short) val;
         } else {
           high.content[high.cardinality++] = (short) (val & 0xFFFF);
@@ -52,41 +52,41 @@ public static  Container[] addOffset(Container source, short offsets) {
       BitmapContainer high = new BitmapContainer();
       low.cardinality = -1;
       high.cardinality = -1;
-      final int b = offset / 64;
+      final int b = offset >>> 6;
       final int i = offset % 64;
       if(i == 0) {
         System.arraycopy(c.bitmap, 0, low.bitmap, b, 1024 - b);
         System.arraycopy(c.bitmap, 1024 - b, high.bitmap, 0, b );
       } else {
         low.bitmap[b + 0] = c.bitmap[0] << i;
         for(int k = 1; k < 1024 - b; k++) {
-          low.bitmap[b + k] = (c.bitmap[k] << i) | (c.bitmap[k - 1] >> (64-i));
+          low.bitmap[b + k] = (c.bitmap[k] << i) | (c.bitmap[k - 1] >>> (64-i));
         }
         for(int k = 1024 - b; k < 1024 ; k++) {
           high.bitmap[k - (1024 - b)] = 
              (c.bitmap[k] << i)  
-             | (c.bitmap[k - 1] >> (64-i));
+             | (c.bitmap[k - 1] >>> (64-i));
         }
-        high.bitmap[b] =  (c.bitmap[1024 - 1] >> (64-i));
+        high.bitmap[b] =  (c.bitmap[1024 - 1] >>> (64-i));
       }
-      return new Container[] {low, high};
+      return new Container[] {low.repairAfterLazy(), high.repairAfterLazy()};
     } else if (source instanceof RunContainer) {
-      RunContainer c = (RunContainer) source;
+      RunContainer input = (RunContainer) source;
       RunContainer low = new RunContainer();
       RunContainer high = new RunContainer();
-      for(int k = 0 ; k < c.nbrruns; k++) {
-        int val =  Util.toIntUnsigned(c.getValue(k));
+      for(int k = 0 ; k < input.nbrruns; k++) {
+        int val =  Util.toIntUnsigned(input.getValue(k));
         val += offset;
-        int finalval =  val + Util.toIntUnsigned(c.getLength(k));
-        if(val < 0xFFFF) {
-          if(finalval < 0xFFFF) {
-            low.smartAppend((short)val,c.getLength(k));
+        int finalval =  val + Util.toIntUnsigned(input.getLength(k));
+        if(val <= 0xFFFF) {
+          if(finalval <= 0xFFFF) {
+            low.smartAppend((short)val,input.getLength(k));
           } else {
             low.smartAppend((short)val,(short)(0xFFFF-val));
             high.smartAppend((short) 0,(short)(finalval & 0xFFFF));
           }
         } else {
-          high.smartAppend((short)(val & 0xFFFF),c.getLength(k));
+          high.smartAppend((short)(val & 0xFFFF),input.getLength(k));
         }
       }
       return new Container[] {low, high};

roaringbitmap/src/test/java/org/roaringbitmap/Fuzzer.java

@@ -288,41 +288,4 @@ public void xorInvariance() {
             (l, r) -> RoaringBitmap.xor(l, r),
             (l, r) -> RoaringBitmap.andNot(RoaringBitmap.or(l, r), RoaringBitmap.and(l, r)));
   }
-
-  @Test
-  public void naiveOrPriorityQueueOrInvariance() {
-    verifyInvarianceArray(
-            bitmaps -> FastAggregation.naive_or(bitmaps),
-            bitmaps -> FastAggregation.priorityqueue_or(bitmaps));
-  }
-
-
-  @Test
-  public void naiveOrPriorityQueueOrInvarianceIterator() {
-    verifyInvarianceIterator(
-            bitmaps -> FastAggregation.naive_or(bitmaps),
-            bitmaps -> FastAggregation.priorityqueue_or(bitmaps));
-  }
-
-
-  @Test
-  public void naiveXorPriorityQueueXorInvariance() {
-    verifyInvarianceArray(
-            bitmaps -> FastAggregation.naive_xor(bitmaps),
-            bitmaps -> FastAggregation.priorityqueue_xor(bitmaps));
-  }
-
-  @Test
-  public void parallelOrVsFastOr() {
-    verifyInvarianceArray(
-            bitmaps -> ParallelAggregation.or(bitmaps),
-            bitmaps -> FastAggregation.priorityqueue_or(bitmaps));
-  }
-
-  @Test
-  public void parallelXorVsFastXor() {
-    verifyInvarianceArray(
-            bitmaps -> ParallelAggregation.xor(bitmaps),
-            bitmaps -> FastAggregation.priorityqueue_xor(bitmaps));
-  }
 }

roaringbitmap/src/test/java/org/roaringbitmap/TestConcatenation.java

@@ -0,0 +1,104 @@
+package org.roaringbitmap;
+
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.roaringbitmap.RandomisedTestData.TestDataSet.testCase;
+
+@RunWith(Parameterized.class)
+public class TestConcatenation {
+
+  @Parameterized.Parameters
+  public static Object[][] params() {
+    return new Object[][]
+            {
+                    // the data set reported in issue #260
+                    {read("src/test/resources/testdata/testIssue260.txt"), 5950},
+                    {read("src/test/resources/testdata/offset_failure_case_1.txt"), 20},
+                    {read("src/test/resources/testdata/offset_failure_case_2.txt"), 20},
+                    {read("src/test/resources/testdata/offset_failure_case_3.txt"), 20},
+                    // a range of test cases with offsets being divisors of 65536
+                    {testCase().withBitmapAt(0).withRunAt(1).withArrayAt(2).build(), 1 << 16},
+                    {testCase().withRunAt(0).withBitmapAt(1).withBitmapAt(2).build(), 1 << 16},
+                    {testCase().withBitmapAt(0).withBitmapAt(1).withRunAt(2).build(), 1 << 16},
+                    {testCase().withBitmapAt(0).withRunAt(2).withArrayAt(4).build(), 1 << 16},
+                    {testCase().withRunAt(0).withBitmapAt(2).withBitmapAt(4).build(), 1 << 16},
+                    {testCase().withArrayAt(0).withBitmapAt(2).withRunAt(4).build(), 1 << 16},
+                    // awkward offsets
+                    {testCase().withBitmapAt(0).build(), 20},
+                    {testCase().withRunAt(0).build(), 20},
+                    {testCase().withArrayAt(0).build(), 20},
+                    {testCase().withBitmapAt(0).withRunAt(1).build(), 20},
+                    {testCase().withRunAt(0).withBitmapAt(1).build(), 20},
+                    {testCase().withArrayAt(0).withBitmapAt(1).build(), 20},
+                    {testCase().withBitmapAt(0).withRunAt(2).build(), 20},
+                    {testCase().withRunAt(0).withBitmapAt(2).build(), 20},
+                    {testCase().withArrayAt(0).withBitmapAt(2).build(), 20},
+                    {testCase().withBitmapAt(0).withRunAt(1).withArrayAt(2).build(), 20},
+                    {testCase().withRunAt(0).withBitmapAt(1).withBitmapAt(2).build(), 20},
+                    {testCase().withArrayAt(0).withBitmapAt(1).withRunAt(2).build(), 20},
+                    {testCase().withBitmapAt(0).withRunAt(2).withArrayAt(4).build(), 20},
+                    {testCase().withRunAt(0).withBitmapAt(2).withBitmapAt(4).build(), 20},
+                    {testCase().withArrayAt(0).withBitmapAt(2).withRunAt(4).build(), 20},
+                    {testCase().withRange(0, 1 << 16).build(), 20}
+            };
+  }
+
+  private final RoaringBitmap bitmap;
+  private final int offset;
+
+  public TestConcatenation(RoaringBitmap bitmap, int offset) {
+    this.bitmap = bitmap;
+    this.offset = offset;
+  }
+
+  @Test
+  public void offsetAppliedCorrectly() {
+    int[] array1 = bitmap.toArray();
+    for (int i = 0; i < array1.length; ++i) {
+      array1[i] += offset;
+    }
+    String inputBitmapAsString = Arrays.toString(bitmap.toArray());
+    RoaringBitmap shifted = RoaringBitmap.addOffset(bitmap, offset);
+    assertEquals(inputBitmapAsString, bitmap.getCardinality(), shifted.getCardinality());
+    assertArrayEquals(inputBitmapAsString, array1, shifted.toArray());
+  }
+
+
+  @Test
+  public void canSerializeAndDeserialize() throws IOException {
+    RoaringBitmap shifted = RoaringBitmap.addOffset(bitmap, offset);
+    ByteArrayDataOutput out = ByteStreams.newDataOutput();
+    shifted.serialize(out);
+
+    RoaringBitmap deserialized = new RoaringBitmap();
+    deserialized.deserialize(ByteStreams.newDataInput(out.toByteArray()));
+
+    assertEquals(shifted, deserialized);
+
+  }
+
+  private static RoaringBitmap read(String classPathResource) {
+    try {
+      OrderedWriter writer = new OrderedWriter();
+      Arrays.stream(Files.readFirstLine(new File(classPathResource), Charset.forName("UTF-8")).split(","))
+              .mapToInt(Integer::parseInt)
+              .forEach(writer::add);
+      writer.flush();
+      return writer.getUnderlying();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

roaringbitmap/src/test/java/org/roaringbitmap/buffer/BufferFuzzer.java

@@ -253,21 +253,6 @@ public void xorInvariance() {
             (l, r) -> MutableRoaringBitmap.andNot(MutableRoaringBitmap.or(l, r), MutableRoaringBitmap.and(l, r)));
   }
 
-
-  @Test
-  public void parallelOrVsFastOr() {
-    verifyInvarianceArray(
-            bitmaps -> BufferParallelAggregation.or(bitmaps),
-            bitmaps -> BufferFastAggregation.priorityqueue_or(bitmaps));
-  }
-
-  @Test
-  public void parallelXorVsFastXor() {
-    verifyInvarianceArray(
-            bitmaps -> BufferParallelAggregation.xor(bitmaps),
-            bitmaps -> BufferFastAggregation.priorityqueue_xor(bitmaps));
-  }
-
   private static MutableRoaringBitmap randomBitmap(int maxKeys) {
     return RandomisedTestData.randomBitmap(maxKeys).toMutableRoaringBitmap();
   }