Merge bd2cfca into 50515ad

roaringbitmap/src/main/java/org/roaringbitmap/ArrayBatchIterator.java

@@ -0,0 +1,29 @@
+package org.roaringbitmap;
+
+import static org.roaringbitmap.Util.toIntUnsigned;
+
+public class ArrayBatchIterator implements ContainerBatchIterator {
+
+  private int index = 0;
+  private final ArrayContainer array;
+
+  public ArrayBatchIterator(ArrayContainer array) {
+    this.array = array;
+  }
+
+  @Override
+  public int next(int key, int[] buffer) {
+    int consumed = 0;
+    short[] data = array.content;
+    while (consumed < buffer.length && index < array.getCardinality()) {
+      buffer[consumed++] = key | toIntUnsigned(data[index++]);
+    }
+    assert consumed <= buffer.length;
+    return consumed;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return index < array.getCardinality();
+  }
+}

roaringbitmap/src/main/java/org/roaringbitmap/ArrayContainer.java

@@ -430,6 +430,11 @@ public PeekableShortIterator getShortIterator() {
     return new ArrayContainerShortIterator(this);
   }
 
+  @Override
+  public ContainerBatchIterator getBatchIterator() {
+    return new ArrayBatchIterator(this);
+  }
+
 
   @Override
   public int getSizeInBytes() {

roaringbitmap/src/main/java/org/roaringbitmap/BatchIterator.java

@@ -0,0 +1,19 @@
+package org.roaringbitmap;
+
+public interface BatchIterator {
+
+  /**
+   * Writes the next batch of integers onto the buffer,
+   * and returns how many were written. Aims to fill
+   * the buffer.
+   * @param buffer - the target to write onto
+   * @return how many values were written during the call.
+   */
+  int nextBatch(int[] buffer);
+
+  /**
+   * Returns true is there are more values to get.
+   * @return whether the iterator is exhaused or not.
+   */
+  boolean hasNext();
+}

roaringbitmap/src/main/java/org/roaringbitmap/BitmapBatchIterator.java

@@ -0,0 +1,37 @@
+package org.roaringbitmap;
+
+import static java.lang.Long.numberOfTrailingZeros;
+
+public class BitmapBatchIterator implements ContainerBatchIterator {
+
+  private int wordIndex = 0;
+  private long word;
+  private final BitmapContainer bitmap;
+
+  public BitmapBatchIterator(BitmapContainer bitmap) {
+    this.bitmap = bitmap;
+    word = bitmap.bitmap[0];
+  }
+
+  @Override
+  public int next(int key, int[] buffer) {
+    int consumed = 0;
+    while (consumed < buffer.length) {
+      while (word == 0) {
+        ++wordIndex;
+        if (wordIndex == 1024) {
+          return consumed;
+        }
+        word = bitmap.bitmap[wordIndex];
+      }
+      buffer[consumed++] = key + (64 * wordIndex) + numberOfTrailingZeros(word);
+      word &= (word - 1);
+    }
+    return consumed;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return wordIndex < 1024;
+  }
+}

roaringbitmap/src/main/java/org/roaringbitmap/BitmapContainer.java

@@ -507,6 +507,11 @@ public PeekableShortIterator getShortIterator() {
     return new BitmapContainerShortIterator(this.bitmap);
   }
 
+  @Override
+  public ContainerBatchIterator getBatchIterator() {
+    return new BitmapBatchIterator(this);
+  }
+
   @Override
   public int getSizeInBytes() {
     return this.bitmap.length * 8;

roaringbitmap/src/main/java/org/roaringbitmap/Container.java

@@ -321,6 +321,12 @@ public String getContainerName() {
    */
   public abstract PeekableShortIterator getShortIterator();
 
+  /**
+   * Gets an iterator to visit the contents of the container in batches
+   * @return iterator
+   */
+  public abstract ContainerBatchIterator getBatchIterator();
+
   /**
    * Computes an estimate of the memory usage of this container. The estimate is not meant to be
    * exact.

roaringbitmap/src/main/java/org/roaringbitmap/ContainerBatchIterator.java

@@ -0,0 +1,20 @@
+package org.roaringbitmap;
+
+public interface ContainerBatchIterator {
+
+  /**
+   * Fills the buffer with values prefixed by the key,
+   * and returns how much of the buffer was used.
+   * @param key the prefix of the values
+   * @param buffer the buffer to write values onto
+   * @return how many values were written.
+   */
+  int next(int key, int[] buffer);
+
+  /**
+   * Whether the underlying container is exhausted or not
+   * @return true if there is data remaining
+   */
+  boolean hasNext();
+
+}

roaringbitmap/src/main/java/org/roaringbitmap/ImmutableBitmapDataProvider.java

@@ -7,6 +7,7 @@
 import java.io.DataOutput;
 import java.io.IOException;
 import java.util.NoSuchElementException;
+import java.util.stream.IntStream;
 
 /**
  * Interface representing an immutable bitmap.
@@ -20,23 +21,23 @@ public interface ImmutableBitmapDataProvider {
    * @param x integer value
    * @return whether the integer value is included.
    */
-  public boolean contains(int x);
+  boolean contains(int x);
 
   /**
    * Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
    * Internally, this is computed as a 64-bit number.
    *
    * @return the cardinality
    */
-  public int getCardinality();
+  int getCardinality();
 
   /**
    * Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
    * This returns a full 64-bit result.
    *
    * @return the cardinality
    */
-  public long getLongCardinality();
+  long getLongCardinality();
 
   /**
    * Visit all values in the bitmap and pass them to the consumer.
@@ -56,18 +57,24 @@ public interface ImmutableBitmapDataProvider {
    * </pre>
    * @param ic the consumer
    */
-  public void forEach(IntConsumer ic);
+  void forEach(IntConsumer ic);
 
   /**
    * For better performance, consider the Use the {@link #forEach forEach} method.
    * @return a custom iterator over set bits, the bits are traversed in ascending sorted order
    */
-  public PeekableIntIterator getIntIterator();
+  PeekableIntIterator getIntIterator();
 
   /**
    * @return a custom iterator over set bits, the bits are traversed in descending sorted order
    */
-  public IntIterator getReverseIntIterator();
+  IntIterator getReverseIntIterator();
+
+  /**
+   * This iterator may be faster than others
+   * @return iterator which works on batches of data.
+   */
+  BatchIterator getBatchIterator();
 
   /**
    * Estimate of the memory usage of this data structure.
@@ -76,30 +83,30 @@ public interface ImmutableBitmapDataProvider {
    *
    * @return estimated memory usage.
    */
-  public int getSizeInBytes();
+  int getSizeInBytes();
 
   /**
    * Estimate of the memory usage of this data structure. Provides
    * full 64-bit number.
    *
    * @return estimated memory usage.
    */
-  public long getLongSizeInBytes();
+  long getLongSizeInBytes();
 
   /**
    * Checks whether the bitmap is empty.
    *
    * @return true if this bitmap contains no set bit
    */
-  public boolean isEmpty();
+  boolean isEmpty();
 
   /**
    * Create a new bitmap of the same class, containing at most maxcardinality integers.
    *
    * @param x maximal cardinality
    * @return a new bitmap with cardinality no more than maxcardinality
    */
-  public ImmutableBitmapDataProvider limit(int x);
+  ImmutableBitmapDataProvider limit(int x);
 
   /**
    * Rank returns the number of integers that are smaller or equal to x (rank(infinity) would be
@@ -112,7 +119,7 @@ public interface ImmutableBitmapDataProvider {
    * @return the rank
    * @see <a href="https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics">Ranking in statistics</a> 
    */
-  public int rank(int x);
+  int rank(int x);
 
   /**
    * Rank returns the number of integers that are smaller or equal to x (rankLong(infinity) would be
@@ -124,7 +131,7 @@ public interface ImmutableBitmapDataProvider {
    * @return the rank
    * @see <a href="https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics">Ranking in statistics</a> 
    */
-  public long rankLong(int x);
+  long rankLong(int x);
 
   /**
    * Return the jth value stored in this bitmap. The provided value 
@@ -137,23 +144,23 @@ public interface ImmutableBitmapDataProvider {
    * @return the value
    * @see <a href="https://en.wikipedia.org/wiki/Selection_algorithm">Selection algorithm</a> 
    */
-  public int select(int j);
+  int select(int j);
 
   /**
    * Get the first (smallest) integer in this RoaringBitmap,
    * that is, returns the minimum of the set.
    * @return the first (smallest) integer
    * @throws NoSuchElementException if empty
    */
-  public int first();
+  int first();
 
   /**
    * Get the last (largest) integer in this RoaringBitmap,
    * that is, returns the maximum of the set.
    * @return the last (largest) integer
    * @throws NoSuchElementException if empty
    */
-  public int last();
+  int last();
 
   /**
    * Serialize this bitmap.
@@ -163,7 +170,7 @@ public interface ImmutableBitmapDataProvider {
    * @param out the DataOutput stream
    * @throws IOException Signals that an I/O exception has occurred.
    */
-  public void serialize(DataOutput out) throws IOException;
+  void serialize(DataOutput out) throws IOException;
 
   /**
    * Report the number of bytes required to serialize this bitmap. This is the number of bytes
@@ -172,13 +179,13 @@ public interface ImmutableBitmapDataProvider {
    *
    * @return the size in bytes
    */
-  public int serializedSizeInBytes();
+  int serializedSizeInBytes();
 
   /**
    * Return the set values as an array. The integer values are in sorted order.
    *
    * @return array representing the set values.
    */
-  public int[] toArray();
+  int[] toArray();
 
 }

roaringbitmap/src/main/java/org/roaringbitmap/RoaringBatchIterator.java

@@ -0,0 +1,44 @@
+package org.roaringbitmap;
+
+public class RoaringBatchIterator implements BatchIterator {
+
+  private final RoaringArray highLowContainer;
+  int index = 0;
+  int key;
+  ContainerBatchIterator iterator;
+
+  public RoaringBatchIterator(RoaringArray highLowContainer) {
+    this.highLowContainer = highLowContainer;
+    nextIterator();
+  }
+
+  @Override
+  public int nextBatch(int[] buffer) {
+    int consumed = 0;
+    if (iterator.hasNext()) {
+      consumed += iterator.next(key, buffer);
+    } else {
+      ++index;
+      nextIterator();
+      if (null != iterator) {
+        return nextBatch(buffer);
+      }
+    }
+    assert consumed <= buffer.length;
+    return consumed;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return null != iterator;
+  }
+
+  private void nextIterator() {
+    if (index < highLowContainer.size()) {
+      iterator = highLowContainer.getContainerAtIndex(index).getBatchIterator();
+      key = highLowContainer.getKeyAtIndex(index) << 16;
+    } else {
+      iterator = null;
+    }
+  }
+}

roaringbitmap/src/main/java/org/roaringbitmap/RoaringBitmap.java

@@ -13,6 +13,7 @@
 import java.io.Serializable;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
+import java.util.stream.IntStream;
 
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 import org.roaringbitmap.buffer.MappeableContainerPointer;
@@ -1566,6 +1567,12 @@ public IntIterator getReverseIntIterator() {
     return new RoaringReverseIntIterator();
   }
 
+
+  @Override
+  public RoaringBatchIterator getBatchIterator() {
+    return new RoaringBatchIterator(highLowContainer);
+  }
+
   /**
    * Estimate of the memory usage of this data structure. This can be expected to be within 1% of
    * the true memory usage.