port to buffer, test direct buffers

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/BufferFastAggregation.java

@@ -4,6 +4,8 @@
 
 package org.roaringbitmap.buffer;
 
+import java.nio.CharBuffer;
+import java.nio.LongBuffer;
 import java.util.*;
 
 
@@ -24,7 +26,7 @@ public final class BufferFastAggregation {
    * @return aggregated bitmap
    */
   public static MutableRoaringBitmap and(ImmutableRoaringBitmap... bitmaps) {
-    return naive_and(bitmaps);
+    return workShyAnd(bitmaps);
   }
 
   /**
@@ -309,6 +311,80 @@ public static MutableRoaringBitmap naive_and(MutableRoaringBitmap... bitmaps) {
     return answer;
   }
 
+  /**
+   * Computes the intersection by first intersecting the keys, avoids
+   * materialising containers.
+   *
+   * @param bitmaps the inputs
+   * @return the intersection of the bitmaps
+   */
+  public static MutableRoaringBitmap workShyAnd(ImmutableRoaringBitmap... bitmaps) {
+    long[] words = new long[1024];
+    ImmutableRoaringBitmap first = bitmaps[0];
+    for (int i = 0; i < first.highLowContainer.size(); ++i) {
+      char key = first.highLowContainer.getKeyAtIndex(i);
+      words[key >>> 6] |= 1L << key;
+    }
+    int numContainers = first.highLowContainer.size();
+    for (int i = 1; i < bitmaps.length && numContainers > 0; ++i) {
+      final char[] keys;
+      if (bitmaps[i].highLowContainer instanceof MutableRoaringArray) {
+        keys = ((MutableRoaringArray) bitmaps[i].highLowContainer).keys;
+      } else {
+        keys = new char[bitmaps[i].highLowContainer.size()];
+        for (int j = 0; j < keys.length; ++j) {
+          keys[j] = bitmaps[i].highLowContainer.getKeyAtIndex(j);
+        }
+      }
+      numContainers = BufferUtil.intersectArrayIntoBitmap(words,
+              CharBuffer.wrap(keys),
+              bitmaps[i].highLowContainer.size());
+    }
+    if (numContainers == 0) {
+      return new MutableRoaringBitmap();
+    }
+    char[] keys = new char[numContainers];
+    int base = 0;
+    int pos = 0;
+    for (long word : words) {
+      while (word != 0L) {
+        keys[pos++] = (char)(base + Long.numberOfTrailingZeros(word));
+        word &= (word - 1);
+      }
+      base += 64;
+    }
+    MappeableContainer[][] containers = new MappeableContainer[numContainers][bitmaps.length];
+    for (int i = 0; i < bitmaps.length; ++i) {
+      ImmutableRoaringBitmap bitmap = bitmaps[i];
+      int position = 0;
+      for (int j = 0; j < bitmap.highLowContainer.size(); ++j) {
+        char key = bitmap.highLowContainer.getKeyAtIndex(j);
+        if ((words[key >>> 6] & (1L << key)) != 0) {
+          containers[position++][i] = bitmap.highLowContainer.getContainerAtIndex(j);
+        }
+      }
+    }
+
+    MutableRoaringArray array =
+            new MutableRoaringArray(keys, new MappeableContainer[numContainers], 0);
+    for (int i = 0; i < numContainers; ++i) {
+      MappeableContainer[] slice = containers[i];
+      Arrays.fill(words, -1L);
+      MappeableContainer tmp = new MappeableBitmapContainer(LongBuffer.wrap(words), -1);
+      for (MappeableContainer container : slice) {
+        MappeableContainer and = tmp.iand(container);
+        if (and != tmp) {
+          tmp = and;
+        }
+      }
+      tmp = tmp.repairAfterLazy();
+      if (!tmp.isEmpty()) {
+        array.append(keys[i], tmp instanceof MappeableBitmapContainer ? tmp.clone() : tmp);
+      }
+    }
+    return new MutableRoaringBitmap(array);
+  }
+
 
 
   /**

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/BufferUtil.java

@@ -10,6 +10,7 @@
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.LongBuffer;
+import java.util.Arrays;
 
 import static java.lang.Long.numberOfTrailingZeros;
 
@@ -666,6 +667,84 @@ protected static int unsignedDifference(final CharBuffer set1, final int length1
     return pos;
   }
 
+
+  /**
+   * Intersects the bitmap with the array, returning the cardinality of the result
+   * @param bitmap the bitmap, modified
+   * @param array the array, not modified
+   * @param length how much of the array to consume
+   * @return the size of the intersection, i.e. how many bits still set in the bitmap
+   */
+  public static int intersectArrayIntoBitmap(long[] bitmap, CharBuffer array, int length) {
+    int lastWordIndex = 0;
+    int wordIndex = 0;
+    long word = 0L;
+    int cardinality = 0;
+    for (int i = 0; i < length; ++i) {
+      wordIndex = array.get(i) >>> 6;
+      if (wordIndex != lastWordIndex) {
+        bitmap[lastWordIndex] &= word;
+        cardinality += Long.bitCount(bitmap[lastWordIndex]);
+        word = 0L;
+        Arrays.fill(bitmap, lastWordIndex + 1, wordIndex, 0L);
+        lastWordIndex = wordIndex;
+      }
+      word |= 1L << array.get(i);
+    }
+    if (word != 0L) {
+      bitmap[wordIndex] &= word;
+      cardinality += Long.bitCount(bitmap[lastWordIndex]);
+    }
+    if (wordIndex < bitmap.length) {
+      Arrays.fill(bitmap, wordIndex + 1, bitmap.length, 0L);
+    }
+    return cardinality;
+  }
+
+  /**
+   * Intersects the bitmap with the array, returning the cardinality of the result
+   * @param bitmap the bitmap, modified
+   * @param array the array, not modified
+   * @param length how much of the array to consume
+   * @return the size of the intersection, i.e. how many bits still set in the bitmap
+   */
+  public static int intersectArrayIntoBitmap(LongBuffer bitmap, CharBuffer array, int length) {
+    if (isBackedBySimpleArray(bitmap)) {
+      return intersectArrayIntoBitmap(bitmap.array(), array, length);
+    }
+    int lastWordIndex = 0;
+    int wordIndex = 0;
+    long word = 0L;
+    int cardinality = 0;
+    for (int i = 0; i < length; ++i) {
+      wordIndex = array.get(i) >>> 6;
+      if (wordIndex != lastWordIndex) {
+        long lastWord = bitmap.get(lastWordIndex);
+        lastWord &= word;
+        bitmap.put(lastWordIndex, lastWord);
+        cardinality += Long.bitCount(lastWord);
+        word = 0L;
+        for (int j = lastWordIndex + 1; j < wordIndex; ++j) {
+          bitmap.put(j, 0L);
+        }
+        lastWordIndex = wordIndex;
+      }
+      word |= 1L << array.get(i);
+    }
+    if (word != 0L) {
+      long currentWord = bitmap.get(wordIndex);
+      currentWord &= word;
+      bitmap.put(wordIndex, currentWord);
+      cardinality += Long.bitCount(currentWord);
+    }
+    if (wordIndex < bitmap.limit()) {
+      for (int j = wordIndex + 1; j < bitmap.limit(); ++j) {
+        bitmap.put(j, 0L);
+      }
+    }
+    return cardinality;
+  }
+
   protected static int unsignedExclusiveUnion2by2(final CharBuffer set1, final int length1,
       final CharBuffer set2, final int length2, final char[] buffer) {
     int pos = 0;

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/MappeableBitmapContainer.java

@@ -601,7 +601,12 @@ public MappeableContainer iadd(int begin, int end) {
 
   @Override
   public MappeableContainer iand(final MappeableArrayContainer b2) {
-    return b2.and(this);// no inplace possible
+    if (-1 == cardinality) {
+      BufferUtil.intersectArrayIntoBitmap(bitmap, b2.content, b2.cardinality);
+      return this;
+    } else {
+      return b2.and(this);// no inplace possible
+    }
   }
 
 
@@ -613,21 +618,28 @@ public MappeableContainer iand(final MappeableBitmapContainer b2) {
       long[] tb = this.bitmap.array();
       long[] tb2 = b2.bitmap.array();
       int len = this.bitmap.limit();
-      for (int k = 0; k < len; ++k) {
-        newCardinality += Long.bitCount(tb[k] & tb2[k]);
-      }
-      if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
+      if (-1 == cardinality) {
         for (int k = 0; k < len; ++k) {
           tb[k] &= tb2[k];
         }
-        this.cardinality = newCardinality;
         return this;
+      } else {
+        for (int k = 0; k < len; ++k) {
+          newCardinality += Long.bitCount(tb[k] & tb2[k]);
+        }
+        if (newCardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
+          for (int k = 0; k < len; ++k) {
+            tb[k] &= tb2[k];
+          }
+          this.cardinality = newCardinality;
+          return this;
+        }
+        final MappeableArrayContainer ac = new MappeableArrayContainer(newCardinality);
+        BufferUtil.fillArrayAND(ac.content.array(), this.bitmap, b2.bitmap);
+        ac.cardinality = newCardinality;
+        return ac;
       }
-      final MappeableArrayContainer ac = new MappeableArrayContainer(newCardinality);
-      BufferUtil.fillArrayAND(ac.content.array(), this.bitmap, b2.bitmap);
-      ac.cardinality = newCardinality;
-      return ac;
-    } else {
+    } else if (-1 != cardinality) {
       int newCardinality = 0;
       int len = this.bitmap.limit();
       for (int k = 0; k < len; ++k) {
@@ -644,14 +656,20 @@ public MappeableContainer iand(final MappeableBitmapContainer b2) {
       BufferUtil.fillArrayAND(ac.content.array(), this.bitmap, b2.bitmap);
       ac.cardinality = newCardinality;
       return ac;
+    } else { // lazy operation, direct buffer
+      int len = this.bitmap.limit();
+      for (int k = 0; k < len; ++k) {
+        this.bitmap.put(k, this.bitmap.get(k) & b2.bitmap.get(k));
+      }
+      return this;
     }
   }
 
   @Override
   public MappeableContainer iand(final MappeableRunContainer x) {
     final int card = x.getCardinality();
-    if (card <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
-      // no point in doing it in-place
+    if (-1 != cardinality && card <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
+      // no point in doing it in-place, unless it's a lazy operation
       MappeableArrayContainer answer = new MappeableArrayContainer(card);
       answer.cardinality = 0;
       for (int rlepos = 0; rlepos < x.nbrruns; ++rlepos) {
@@ -667,19 +685,26 @@ public MappeableContainer iand(final MappeableRunContainer x) {
     int start = 0;
     for (int rlepos = 0; rlepos < x.nbrruns; ++rlepos) {
       int end = (x.getValue(rlepos));
-      int prevOnes = cardinalityInRange(start, end);
-      BufferUtil.resetBitmapRange(this.bitmap, start, end);
-      updateCardinality(prevOnes, 0);
+      if (-1 == cardinality) {
+        BufferUtil.resetBitmapRange(this.bitmap, start, end);
+      } else {
+        int prevOnes = cardinalityInRange(start, end);
+        BufferUtil.resetBitmapRange(this.bitmap, start, end);
+        updateCardinality(prevOnes, 0);
+      }
       start = end + (x.getLength(rlepos)) + 1;
     }
-    int ones = cardinalityInRange(start, MAX_CAPACITY);
-    BufferUtil.resetBitmapRange(this.bitmap, start, MAX_CAPACITY);
-    updateCardinality(ones, 0);
-    if (getCardinality() > MappeableArrayContainer.DEFAULT_MAX_SIZE) {
-      return this;
+    if (-1 == cardinality) {
+      BufferUtil.resetBitmapRange(this.bitmap, start, MAX_CAPACITY);
     } else {
-      return toArrayContainer();
+      int ones = cardinalityInRange(start, MAX_CAPACITY);
+      BufferUtil.resetBitmapRange(this.bitmap, start, MAX_CAPACITY);
+      updateCardinality(ones, 0);
+      if (getCardinality() <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
+        return toArrayContainer();
+      }
     }
+    return this;
   }
 
   @Override

RoaringBitmap/src/main/java/org/roaringbitmap/buffer/PointableRoaringArray.java

@@ -7,6 +7,7 @@
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 
 /**
  * Generic interface for the array underlying roaring bitmap classes.