Merge 68dc90f into f223387

jmh/src/main/java/org/roaringbitmap/RandomData.java

@@ -8,24 +8,17 @@ public class RandomData {
 
   private static final ThreadLocal<long[]> bits = ThreadLocal.withInitial(() -> new long[1 << 10]);
 
+  public static RoaringBitmap randomContiguousBitmap(int startKey, int numKeys, double rleLimit, double denseLimit) {
+    int[] keys = new int[numKeys];
+    for (int i = 0; i < numKeys; ++i) {
+      keys[i] = i + startKey;
+    }
+    return forKeys(keys, rleLimit, denseLimit);
+  }
+
   public static RoaringBitmap randomBitmap(int maxKeys, double rleLimit, double denseLimit) {
     int[] keys = createSorted16BitInts(maxKeys);
-    OrderedWriter writer = new OrderedWriter();
-    IntStream.of(keys)
-            .forEach(key -> {
-              double choice = ThreadLocalRandom.current().nextDouble();
-              final IntStream stream;
-              if (choice < rleLimit) {
-                stream = rleRegion();
-              } else if (choice < denseLimit) {
-                stream = denseRegion();
-              } else {
-                stream = sparseRegion();
-              }
-              stream.map(i -> (key << 16) | i).forEach(writer::add);
-            });
-    writer.flush();
-    return writer.getUnderlying();
+    return forKeys(keys, rleLimit, denseLimit);
   }
 
   public static IntStream rleRegion() {
@@ -68,4 +61,23 @@ private static int[] createSorted16BitInts(int howMany) {
     }
     return keys;
   }
+
+  private static RoaringBitmap forKeys(int[] keys, double rleLimit, double denseLimit) {
+    OrderedWriter writer = new OrderedWriter();
+    IntStream.of(keys)
+            .forEach(key -> {
+              double choice = ThreadLocalRandom.current().nextDouble();
+              final IntStream stream;
+              if (choice < rleLimit) {
+                stream = rleRegion();
+              } else if (choice < denseLimit) {
+                stream = denseRegion();
+              } else {
+                stream = sparseRegion();
+              }
+              stream.map(i -> (key << 16) | i).forEach(writer::add);
+            });
+    writer.flush();
+    return writer.getUnderlying();
+  }
 }

jmh/src/main/java/org/roaringbitmap/iteration/Concatenation.java

@@ -0,0 +1,110 @@
+package org.roaringbitmap.iteration;
+
+import org.openjdk.jmh.annotations.*;
+import org.roaringbitmap.*;
+
+import java.util.BitSet;
+import java.util.stream.IntStream;
+
+@State(Scope.Benchmark)
+@Fork(value = 1, jvmArgsPrepend =
+        {
+                "-XX:+UseG1GC",
+                "-XX:-TieredCompilation",
+                "-XX:+AlwaysPreTouch",
+                "-ms4G",
+                "-mx4G"
+        })
+public class Concatenation {
+
+  @Param({"1", "16", "32"})
+  int count;
+
+  @Param({"8", "32", "2048"})
+  int size;
+
+
+  @Setup(Level.Trial)
+  public void init() {
+    bitSets = IntStream.range(0, count)
+            .mapToObj(i -> {
+              RoaringBitmap rb = RandomData.randomContiguousBitmap(0, size, 0.1, 0.8);
+              return new BitSetWithOffset(rb, toBitSet(rb), i * size * 65536);
+            }).toArray(BitSetWithOffset[]::new);
+  }
+
+  private BitSetWithOffset[] bitSets;
+
+
+  private static class BitSetWithOffset {
+    public final RoaringBitmap bitmap;
+    public final BitSet bitset;
+    public final int offset;
+
+    public BitSetWithOffset(RoaringBitmap bitmap, BitSet bitset, int offset) {
+      this.bitmap = bitmap;
+      this.bitset = bitset;
+      this.offset = offset;
+    }
+  }
+
+  public static void main(String[] args) {
+    Concatenation bench = new Concatenation();
+    bench.count = 10;
+    bench.size = 32;
+    bench.init();
+    assert bench.roaringBatchOrderedWriter().equals(bench.roaringBatchOrderedWriter());
+  }
+
+  @Benchmark
+  public BitSet bitset() {
+    BitSet result = new BitSet();
+    for(int i = 0; i < bitSets.length; ++i) {
+      BitSetWithOffset bit = bitSets[i];
+      int currentBit = bit.bitset.nextSetBit(0);
+      while(currentBit != -1) {
+        result.set(currentBit + bit.offset);
+        currentBit = bit.bitset.nextSetBit(currentBit + 1);
+      }
+    }
+    return result;
+  }
+
+  @Benchmark
+  public RoaringBitmap roaringNaive() {
+    RoaringBitmap result = new RoaringBitmap();
+    for(int i = 0; i < bitSets.length; ++i) {
+      BitSetWithOffset bit = bitSets[i];
+      PeekableIntIterator peekableIter = bit.bitmap.getIntIterator();
+      while(peekableIter.hasNext()){
+        int currentBit = peekableIter.next();
+        result.add(currentBit + bit.offset);
+      }
+    }
+    return result;
+  }
+
+  @Benchmark
+  public RoaringBitmap roaringBatchOrderedWriter() {
+    int[] buffer = new int[256];
+    OrderedWriter writer = new OrderedWriter();
+      for(int i = 0; i < bitSets.length; ++i) {
+        BitSetWithOffset bit = bitSets[i];
+        RoaringBatchIterator iterator = bit.bitmap.getBatchIterator();
+        while (iterator.hasNext()) {
+          int count = iterator.nextBatch(buffer);
+          for (int j = 0; j < count; ++j) {
+            writer.add(buffer[j] + bit.offset);
+          }
+        }
+    }
+    writer.flush();
+    return writer.getUnderlying();
+  }
+
+  private static BitSet toBitSet(RoaringBitmap rb) {
+    BitSet bs = new BitSet();
+    rb.forEach((IntConsumer) bs::set);
+    return bs;
+  }
+}