Merge branch 'master' into sean/bump-github-actions

.github/workflows/java-all-versions.yml

@@ -1,4 +1,4 @@
-name: Java 8, 11, 17 CI
+name: Java 8, 11, 17, 21 CI
 
 on: [push,pull_request]
 
@@ -8,14 +8,15 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDKs 8, 11, 17
+      - name: Set up JDKs 8, 11, 17, 21
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: |
             8
             11
             17
+            21
       - name: Build with Gradle
         run: ./gradlew assemble
       - name: Style check
@@ -26,3 +27,5 @@ jobs:
         run: ./gradlew test -PtestOnJava=11 --stacktrace
       - name: Test with Java 17
         run: ./gradlew test -PtestOnJava=17 --stacktrace
+      - name: Test with Java 21
+        run: ./gradlew test -PtestOnJava=21 --stacktrace

README.md

@@ -40,7 +40,7 @@ This library is used by
 
 
 The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. [Apache Lucene](http://lucene.apache.org/) uses  Roaring bitmaps, though they have their own [independent implementation](https://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java?view=markup&pathrev=1629606). Derivatives of Lucene such as Solr and Elastic also use Roaring bitmaps.
-Other platforms such as [Whoosh](https://pypi.python.org/pypi/Whoosh/), [Microsoft Visual Studio Team Services (VSTS)](https://www.visualstudio.com/team-services/) and [Pilosa](https://github.com/pilosa/pilosa) also use Roaring bitmaps with their own implementations. You find Roaring bitmaps in [InfluxDB](https://www.influxdata.com), [Bleve](http://www.blevesearch.com), [Cloud Torrent](https://github.com/jpillora/cloud-torrent), and so forth.
+Other platforms such as [Whoosh](https://pypi.python.org/pypi/Whoosh/), [Microsoft Visual Studio Team Services (VSTS)](https://www.visualstudio.com/team-services/) and [Pilosa](https://github.com/pilosa/pilosa) also use Roaring bitmaps with their own implementations. You find Roaring bitmaps in [InfluxDB](https://www.influxdata.com), [Bleve](http://www.blevesearch.com), [Cloud Torrent](https://github.com/jpillora/cloud-torrent), [Redpanda](https://github.com/redpanda-data/redpanda), and so forth.
 
 
 [There is a serialized format specification for interoperability between implementations](https://github.com/RoaringBitmap/RoaringFormatSpec/).

RoaringBitmap/src/main/java/org/roaringbitmap/ArrayBatchIterator.java

@@ -13,11 +13,11 @@ public ArrayBatchIterator(ArrayContainer array) {
   }
 
   @Override
-  public int next(int key, int[] buffer) {
+  public int next(int key, int[] buffer, int offset) {
     int consumed = 0;
     char[] data = array.content;
-    while (consumed < buffer.length && index < array.getCardinality()) {
-      buffer[consumed++] = key + (data[index++]);
+    while ((offset + consumed) < buffer.length && index < array.getCardinality()) {
+      buffer[offset + consumed++] = key + (data[index++]);
     }
     return consumed;
   }

RoaringBitmap/src/main/java/org/roaringbitmap/ArrayContainer.java

@@ -1129,6 +1129,14 @@ public BitmapContainer toBitmapContainer() {
     return bc;
   }
 
+  @Override
+  public void copyBitmapTo(long[] dest, int position) {
+    for (int k = 0; k < cardinality; ++k) {
+      final char x = content[k];
+      dest[position + x/64] |= 1L << x;
+    }
+  }
+
   @Override
   public int nextValue(char fromValue) {
     int index = Util.advanceUntil(content, -1, cardinality, fromValue);

RoaringBitmap/src/main/java/org/roaringbitmap/BitSetUtil.java

@@ -14,14 +14,99 @@
  *
  */
 public class BitSetUtil {
-  // todo: add a method to convert a RoaringBitmap to a BitSet using BitSet.valueOf
 
   // a block consists has a maximum of 1024 words, each representing 64 bits,
   // thus representing at maximum 65536 bits
   public static final int BLOCK_LENGTH = BitmapContainer.MAX_CAPACITY / Long.SIZE; //
   // 64-bit
   // word
 
+  /**
+   * Convert a {@link RoaringBitmap} to a {@link BitSet}.
+   * <p>
+   * Equivalent to calling {@code BitSet.valueOf(BitSetUtil.toLongArray(bitmap))}.
+   */
+  public static BitSet bitsetOf(RoaringBitmap bitmap) {
+    return BitSet.valueOf(toLongArray(bitmap));
+  }
+
+  /**
+   * Convert a {@link RoaringBitmap} to a {@link BitSet} without copying to an intermediate array.
+   */
+  public static BitSet bitsetOfWithoutCopy(RoaringBitmap bitmap) {
+    if (bitmap.isEmpty()) {
+      return new BitSet(0);
+    }
+    int last = bitmap.last();
+    if (last < 0) {
+      throw new IllegalArgumentException("bitmap has negative bits set");
+    }
+    BitSet bitSet = new BitSet(last);
+    bitmap.forEach((IntConsumer) bitSet::set);
+    return bitSet;
+  }
+
+  /**
+   * Returns an array of little-endian ordered bytes, given a {@link RoaringBitmap}.
+   * <p>
+   * See {@link BitSet#toByteArray()}.
+   */
+  public static byte[] toByteArray(RoaringBitmap bitmap) {
+    long[] words = toLongArray(bitmap);
+    ByteBuffer buffer = ByteBuffer.allocate(words.length * Long.SIZE)
+            .order(ByteOrder.LITTLE_ENDIAN);
+    buffer.asLongBuffer().put(words);
+    return buffer.array();
+  }
+
+  /**
+   * Returns an array of long, given a {@link RoaringBitmap}.
+   * <p>
+   * See {@link BitSet#toLongArray()}.
+   */
+  public static long[] toLongArray(RoaringBitmap bitmap) {
+    if (bitmap.isEmpty()) {
+      return new long[0];
+    }
+
+    int last = bitmap.last();
+    if (last < 0) {
+      throw new IllegalArgumentException("bitmap has negative bits set");
+    }
+    int lastBit = Math.max(last, Long.SIZE);
+    int remainder = lastBit % Long.SIZE;
+    int numBits = remainder > 0 ? lastBit - remainder : lastBit;
+    int wordsInUse = numBits / Long.SIZE + 1;
+    long[] words = new long[wordsInUse];
+
+    ContainerPointer pointer = bitmap.getContainerPointer();
+    int numContainers = Math.max(words.length / BLOCK_LENGTH, 1);
+    int position = 0;
+    for (int i = 0; i <= numContainers; i++) {
+      char key = Util.lowbits(i);
+      if (key == pointer.key()) {
+        Container container = pointer.getContainer();
+        int remaining = wordsInUse - position;
+        int length = Math.min(BLOCK_LENGTH, remaining);
+        if (container instanceof BitmapContainer) {
+          ((BitmapContainer)container).copyBitmapTo(words, position, length);
+        } else {
+          container.copyBitmapTo(words, position);
+        }
+        position += length;
+        pointer.advance();
+        if (pointer.getContainer() == null) {
+          break;
+        }
+      } else {
+        position += BLOCK_LENGTH;
+      }
+    }
+    assert pointer.getContainer() == null;
+    assert position == wordsInUse;
+    return words;
+  }
+
   /**
    * Creates array container's content char buffer.
    *

RoaringBitmap/src/main/java/org/roaringbitmap/BitmapBatchIterator.java

@@ -13,17 +13,17 @@ public BitmapBatchIterator(BitmapContainer bitmap) {
   }
 
   @Override
-  public int next(int key, int[] buffer) {
+  public int next(int key, int[] buffer, int offset) {
     int consumed = 0;
-    while (consumed < buffer.length) {
+    while ((consumed + offset) < buffer.length) {
       while (word == 0) {
         ++wordIndex;
         if (wordIndex == 1024) {
           return consumed;
         }
         word = bitmap.bitmap[wordIndex];
       }
-      buffer[consumed++] = key + (64 * wordIndex) + numberOfTrailingZeros(word);
+      buffer[offset + consumed++] = key + (64 * wordIndex) + numberOfTrailingZeros(word);
       word &= (word - 1);
     }
     return consumed;

RoaringBitmap/src/main/java/org/roaringbitmap/BitmapContainer.java

@@ -1665,6 +1665,15 @@ public BitmapContainer toBitmapContainer() {
     return this;
   }
 
+  @Override
+  public void copyBitmapTo(long[] words, int position) {
+    System.arraycopy(bitmap, 0, words, position, bitmap.length);
+  }
+
+  public void copyBitmapTo(long[] words, int position, int length) {
+    System.arraycopy(bitmap, 0, words, position, length);
+  }
+
   @Override
   public int nextValue(char fromValue) {
     return nextSetBit((fromValue));
@@ -1706,7 +1715,6 @@ public int last() {
     // sizeof(long) * #words from start - number of bits after the last bit set
     return (i + 1) * 64 - Long.numberOfLeadingZeros(bitmap[i]) - 1;
   }
-
 }
 
 

RoaringBitmap/src/main/java/org/roaringbitmap/Container.java

@@ -986,6 +986,17 @@ public Container xor(Container x) {
    */
   public abstract BitmapContainer toBitmapContainer();
 
+  /**
+   * Copy the current container to a destination {@code long[]}. Equivalent to calling
+   * {@link #toBitmapContainer()} and copying the result to the given position. The destination
+   * array should be sized to accomodate the maximum number of words required to represent
+   * the container bitmap.
+   *
+   * @param dest the destination array
+   * @param position the position to copy to
+   */
+  public abstract void copyBitmapTo(long[] dest, int position);
+
   /**
    * Gets the first value greater than or equal to the lower bound, or -1 if no such value exists.
    * @param fromValue the lower bound (inclusive)

RoaringBitmap/src/main/java/org/roaringbitmap/ContainerBatchIterator.java

@@ -7,9 +7,21 @@ public interface ContainerBatchIterator extends Cloneable {
    * and returns how much of the buffer was used.
    * @param key the prefix of the values
    * @param buffer the buffer to write values onto
+   * @param offset the offset into the buffer to write values onto
    * @return how many values were written.
    */
-  int next(int key, int[] buffer);
+  int next(int key, int[] buffer, int offset);
+
+  /**
+   * Fills the buffer with values prefixed by the key,
+   * and returns how much of the buffer was used.
+   * @param key the prefix of the values
+   * @param buffer the buffer to write values onto
+   * @return how many values were written.
+   */
+  default int next(int key, int[] buffer) {
+    return next(key, buffer, 0);
+  }
 
   /**
    * Whether the underlying container is exhausted or not

RoaringBitmap/src/main/java/org/roaringbitmap/RoaringBatchIterator.java

@@ -18,9 +18,9 @@ public RoaringBatchIterator(RoaringArray highLowContainer) {
   @Override
   public int nextBatch(int[] buffer) {
     int consumed = 0;
-    while (iterator != null && consumed == 0) {
-      consumed = iterator.next(key, buffer);
-      if (consumed == 0 || !iterator.hasNext()) {
+    while (iterator != null && consumed < buffer.length) {
+      consumed += iterator.next(key, buffer, consumed);
+      if (consumed < buffer.length || !iterator.hasNext()) {
         nextContainer();
       }
     }

RoaringBitmap/src/main/java/org/roaringbitmap/RunBatchIterator.java

@@ -13,16 +13,17 @@ public RunBatchIterator(RunContainer runs) {
   }
 
   @Override
-  public int next(int key, int[] buffer) {
+  public int next(int key, int[] buffer, int offset) {
     int consumed = 0;
     do {
       int runStart = (runs.getValue(run));
       int runLength = (runs.getLength(run));
       int chunkStart = runStart + cursor;
-      int chunkEnd = chunkStart + Math.min(runLength - cursor, buffer.length - consumed - 1);
+      int usableBufferLength = buffer.length - offset - consumed;
+      int chunkEnd = chunkStart + Math.min(runLength - cursor, usableBufferLength - 1);
       int chunk = chunkEnd - chunkStart + 1;
       for (int i = 0; i < chunk; ++i) {
-        buffer[consumed + i] = key + chunkStart + i;
+        buffer[offset + consumed + i] = key + chunkStart + i;
       }
       consumed += chunk;
       if (runStart + runLength == chunkEnd) {
@@ -31,7 +32,7 @@ public int next(int key, int[] buffer) {
       } else {
         cursor += chunk;
       }
-    } while (consumed < buffer.length && run != runs.numberOfRuns());
+    } while ((offset + consumed) < buffer.length && run != runs.numberOfRuns());
     return consumed;
   }
 

RoaringBitmap/src/main/java/org/roaringbitmap/RunContainer.java

@@ -1550,7 +1550,7 @@ public Container ior(RunContainer x) {
       this.smartAppend(x.getValue(xrlepos), x.getLength(xrlepos));
       ++xrlepos;
     }
-    return this.toBitmapIfNeeded();
+    return this.toEfficientContainer();
   }
 
   @Override
@@ -1986,7 +1986,7 @@ public Container or(RunContainer x) {
     if (answer.isFull()) {
       return full();
     }
-    return answer.toBitmapIfNeeded();
+    return answer.toEfficientContainer();
   }
 
   // Prepend a value length with all values starting from a given value
@@ -2295,16 +2295,6 @@ private void smartAppendExclusive(char start, char length) {
     }
   }
 
-  // convert to bitmap *if needed* (useful if you know it can't be an array)
-  private Container toBitmapIfNeeded() {
-    int sizeAsRunContainer = RunContainer.serializedSizeInBytes(this.nbrruns);
-    int sizeAsBitmapContainer = BitmapContainer.serializedSizeInBytes(0);
-    if (sizeAsBitmapContainer > sizeAsRunContainer) {
-      return this;
-    }
-    return toBitmapContainer();
-  }
-
   /**
    * Convert the container to either a Bitmap or an Array Container, depending on the cardinality.
    *
@@ -2646,17 +2636,29 @@ public void forAllInRange(char startValue, char endValue, final RelativeRangeCon
 
   @Override
   public BitmapContainer toBitmapContainer() {
-    int card = this.getCardinality();
+    int card = 0;
     BitmapContainer answer = new BitmapContainer();
     for (int rlepos = 0; rlepos < this.nbrruns; ++rlepos) {
       int start = (this.getValue(rlepos));
       int end = start + (this.getLength(rlepos)) + 1;
+      card += end - start;
       Util.setBitmapRange(answer.bitmap, start, end);
     }
+    assert card == this.getCardinality();
     answer.cardinality = card;
     return answer;
   }
 
+  @Override
+  public void copyBitmapTo(long[] dest, int position) {
+    int offset = position * Long.SIZE;
+    for (int rlepos = 0; rlepos < this.nbrruns; ++rlepos) {
+      int start = offset + this.getValue(rlepos);
+      int end = start + this.getLength(rlepos) + 1;
+      Util.setBitmapRange(dest, start, end);
+    }
+  }
+
   @Override
   public int nextValue(char fromValue) {
     int index = unsignedInterleavedBinarySearch(valueslength, 0, nbrruns, fromValue);

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/ArrayBatchIterator.java

@@ -17,11 +17,11 @@ public ArrayBatchIterator(MappeableArrayContainer array) {
   }
 
   @Override
-  public int next(int key, int[] buffer) {
+  public int next(int key, int[] buffer, int offset) {
     int consumed = 0;
     CharBuffer data = array.content;
-    while (consumed < buffer.length && index < array.getCardinality()) {
-      buffer[consumed++] = key + (data.get(index++));
+    while ((offset + consumed) < buffer.length && index < array.getCardinality()) {
+      buffer[offset + consumed++] = key + (data.get(index++));
     }
     return consumed;
   }

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/BitmapBatchIterator.java

@@ -15,17 +15,17 @@ public BitmapBatchIterator(MappeableBitmapContainer bitmap) {
   }
 
   @Override
-  public int next(int key, int[] buffer) {
+  public int next(int key, int[] buffer, int offset) {
     int consumed = 0;
-    while (consumed < buffer.length) {
+    while ((offset + consumed) < buffer.length) {
       while (word == 0) {
         ++wordIndex;
         if (wordIndex == 1024) {
           return consumed;
         }
         word = bitmap.bitmap.get(wordIndex);
       }
-      buffer[consumed++] = key + (64 * wordIndex) + numberOfTrailingZeros(word);
+      buffer[offset + consumed++] = key + (64 * wordIndex) + numberOfTrailingZeros(word);
       word &= (word - 1);
     }
     return consumed;