Merge 8f6728e into 0c3c489

RoaringBitmap · Jun 2, 2020 · 8db058c · 8db058c
2 parents 0c3c489 + 8f6728e
commit 8db058c
Show file tree

Hide file tree

Showing 5 changed files with 166 additions and 1 deletion.
diff --git a/RoaringBitmap/src/main/java/org/roaringbitmap/FastAggregation.java b/RoaringBitmap/src/main/java/org/roaringbitmap/FastAggregation.java
@@ -335,6 +335,72 @@ public static RoaringBitmap workShyAnd(RoaringBitmap... bitmaps) {
   }
 
 
+
+  /**
+   * Computes the intersection by first intersecting the keys, avoids
+   * materialising containers, limits memory usage.  You must provide a long[] array
+   * of length at least 1024, initialized with zeroes. We do not check whether the array
+   * is initialized with zeros: it is the caller's responsability.
+   * You should expect this function to be slower than workShyAnd and the reduction
+   * in memory usage might be small.
+   *
+   * @param buffer should be a 1024-long array
+   * @param bitmaps the inputs
+   * @return the intersection of the bitmaps
+   */
+  public static RoaringBitmap workAndMemoryShyAnd(long[] buffer, RoaringBitmap... bitmaps) {
+    if(buffer.length < 1024) {
+      throw new IllegalArgumentException("buffer should have at least 1024 elements.");
+    } 
+    long[] words = buffer;
+    RoaringBitmap first = bitmaps[0];
+    for (int i = 0; i < first.highLowContainer.size; ++i) {
+      char key = first.highLowContainer.keys[i];
+      words[key >>> 6] |= 1L << key;
+    }
+    int numContainers = first.highLowContainer.size;
+    for (int i = 1; i < bitmaps.length && numContainers > 0; ++i) {
+      numContainers = Util.intersectArrayIntoBitmap(words,
+              bitmaps[i].highLowContainer.keys, bitmaps[i].highLowContainer.size);
+    }
+    if (numContainers == 0) {
+      return new RoaringBitmap();
+    }
+    char[] keys = new char[numContainers];
+    int base = 0;
+    int pos = 0;
+    for (long word : words) {
+      while (word != 0L) {
+        keys[pos++] = (char)(base + Long.numberOfTrailingZeros(word));
+        word &= (word - 1);
+      }
+      base += 64;
+    }
+    RoaringArray array =
+            new RoaringArray(keys, new Container[numContainers], 0);
+    for (int i = 0; i < numContainers; ++i) {
+      char MatchingKey = keys[i];
+      Arrays.fill(words, -1L);
+      Container tmp = new BitmapContainer(words, -1);
+      for(RoaringBitmap bitmap: bitmaps) {
+        int idx = bitmap.highLowContainer.getIndex(MatchingKey);
+        if(idx < 0) {
+          continue;
+        }
+        Container container = bitmap.highLowContainer.getContainerAtIndex(idx);
+        Container and = tmp.iand(container);
+        if (and != tmp) {
+          tmp = and;
+        }
+      }
+      tmp = tmp.repairAfterLazy();
+      if (!tmp.isEmpty()) {
+        array.append(keys[i], tmp instanceof BitmapContainer ? tmp.clone() : tmp);
+      }
+    }
+    return new RoaringBitmap(array);
+  }
+
   /**
    * Compute overall OR between bitmaps two-by-two.
    *

diff --git a/RoaringBitmap/src/main/java/org/roaringbitmap/buffer/BufferFastAggregation.java b/RoaringBitmap/src/main/java/org/roaringbitmap/buffer/BufferFastAggregation.java
@@ -388,7 +388,81 @@ public static MutableRoaringBitmap workShyAnd(ImmutableRoaringBitmap... bitmaps)
     return new MutableRoaringBitmap(array);
   }
 
-
+  /**
+   * Computes the intersection by first intersecting the keys, avoids
+   * materialising containers, limits memory usage. You must provide a long[] array
+   * of length at least 1024, initialized with zeroes. We do not check whether the array
+   * is initialized with zeros: it is the caller's responsability. 
+   * You should expect this function to be slower than workShyAnd and the reduction
+   * in memory usage might be small.
+   *
+   * @param buffer should be a 1024-long array
+   * @param bitmaps the inputs
+   * @return the intersection of the bitmaps
+   */
+  public static MutableRoaringBitmap workAndMemoryShyAnd(long[] buffer, 
+      ImmutableRoaringBitmap... bitmaps) {
+    if(buffer.length < 1024) {
+      throw new IllegalArgumentException("buffer should have at least 1024 elements.");
+    } 
+    long[] words = buffer;
+    ImmutableRoaringBitmap first = bitmaps[0];
+    for (int i = 0; i < first.highLowContainer.size(); ++i) {
+      char key = first.highLowContainer.getKeyAtIndex(i);
+      words[key >>> 6] |= 1L << key;
+    }
+    int numContainers = first.highLowContainer.size();
+    for (int i = 1; i < bitmaps.length && numContainers > 0; ++i) {
+      final char[] keys;
+      if (bitmaps[i].highLowContainer instanceof MutableRoaringArray) {
+        keys = ((MutableRoaringArray) bitmaps[i].highLowContainer).keys;
+      } else {
+        keys = new char[bitmaps[i].highLowContainer.size()];
+        for (int j = 0; j < keys.length; ++j) {
+          keys[j] = bitmaps[i].highLowContainer.getKeyAtIndex(j);
+        }
+      }
+      numContainers = BufferUtil.intersectArrayIntoBitmap(words,
+              CharBuffer.wrap(keys),
+              bitmaps[i].highLowContainer.size());
+    }
+    if (numContainers == 0) {
+      return new MutableRoaringBitmap();
+    }
+    char[] keys = new char[numContainers];
+    int base = 0;
+    int pos = 0;
+    for (long word : words) {
+      while (word != 0L) {
+        keys[pos++] = (char)(base + Long.numberOfTrailingZeros(word));
+        word &= (word - 1);
+      }
+      base += 64;
+    }
+    MutableRoaringArray array =
+            new MutableRoaringArray(keys, new MappeableContainer[numContainers], 0);
+    for (int i = 0; i < numContainers; ++i) {
+      char MatchingKey = keys[i];
+      Arrays.fill(words, -1L);
+      MappeableContainer tmp = new MappeableBitmapContainer(LongBuffer.wrap(words), -1);
+      for(ImmutableRoaringBitmap bitmap: bitmaps) {
+        int idx = bitmap.highLowContainer.getIndex(MatchingKey);
+        if(idx < 0) {
+          continue;
+        }
+        MappeableContainer container = bitmap.highLowContainer.getContainerAtIndex(idx);
+        MappeableContainer and = tmp.iand(container);
+        if (and != tmp) {
+          tmp = and;
+        }
+      }
+      tmp = tmp.repairAfterLazy();
+      if (!tmp.isEmpty()) {
+        array.append(keys[i], tmp instanceof MappeableBitmapContainer ? tmp.clone() : tmp);
+      }
+    }
+    return new MutableRoaringBitmap(array);
+  }
 
   /**
    * Compute overall OR between bitmaps two-by-two.

diff --git a/RoaringBitmap/src/test/java/org/roaringbitmap/TestFastAggregation.java b/RoaringBitmap/src/test/java/org/roaringbitmap/TestFastAggregation.java
@@ -249,6 +249,9 @@ public void testWorkShyAnd(List<RoaringBitmap> list) {
         RoaringBitmap result = FastAggregation.workShyAnd(bitmaps);
         RoaringBitmap expected = FastAggregation.naive_and(bitmaps);
         assertEquals(expected, result);
+        long[] buffer = new long[1024];
+        result = FastAggregation.workAndMemoryShyAnd(buffer, bitmaps);
+        assertEquals(expected, result);
     }
 
 }
diff --git a/RoaringBitmap/src/test/java/org/roaringbitmap/buffer/TestFastAggregation.java b/RoaringBitmap/src/test/java/org/roaringbitmap/buffer/TestFastAggregation.java
@@ -242,6 +242,9 @@ public void testWorkShyAnd(List<MutableRoaringBitmap> list) {
     MutableRoaringBitmap result = BufferFastAggregation.workShyAnd(bitmaps);
     MutableRoaringBitmap expected = BufferFastAggregation.naive_and(bitmaps);
     assertEquals(expected, result);
+    long[] buffer = new long[1024];
+    result = BufferFastAggregation.workAndMemoryShyAnd(buffer, bitmaps);
+    assertEquals(expected, result);
   }
 
   @MethodSource("bitmaps")
@@ -254,5 +257,8 @@ public void testWorkShyAndDirect(List<MutableRoaringBitmap> list) {
     MutableRoaringBitmap result = BufferFastAggregation.workShyAnd(bitmaps);
     MutableRoaringBitmap expected = BufferFastAggregation.naive_and(bitmaps);
     assertEquals(expected, result);
+    long[] buffer = new long[1024];
+    result = BufferFastAggregation.workAndMemoryShyAnd(buffer, bitmaps);
+    assertEquals(expected, result);
   }
 }
diff --git a/jmh/src/jmh/java/org/roaringbitmap/aggregation/FastAggregationRLEStressTest.java b/jmh/src/jmh/java/org/roaringbitmap/aggregation/FastAggregationRLEStressTest.java
@@ -7,12 +7,14 @@
 import org.roaringbitmap.buffer.BufferFastAggregation;
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 
+import java.util.Arrays;
 import java.util.SplittableRandom;
 
 public class FastAggregationRLEStressTest {
 
   @State(Scope.Benchmark)
   public static class BitmapState {
+    public long[] buffer = new long[1024];
 
     public enum ConstructionStrategy {
       CONSTANT_MEMORY {
@@ -76,6 +78,7 @@ public void createBitmaps() {
     }
   }
 
+
   @Benchmark
   public RoaringBitmap and(BitmapState state) {
     return FastAggregation.and(state.bitmaps);
@@ -85,4 +88,17 @@ public RoaringBitmap and(BitmapState state) {
   public ImmutableRoaringBitmap andBuffer(BitmapState state) {
     return BufferFastAggregation.and(state.bufferBitmaps);
   }
+
+  @Benchmark
+  public RoaringBitmap andMemoryShy(BitmapState state) {
+    Arrays.fill(state.buffer,0);
+    return FastAggregation.workAndMemoryShyAnd(state.buffer, state.bitmaps);
+  }
+
+  @Benchmark
+  public ImmutableRoaringBitmap andBufferMemoryShy(BitmapState state) {
+    Arrays.fill(state.buffer,0);
+    return BufferFastAggregation.workAndMemoryShyAnd(state.buffer, state.bufferBitmaps);
+  }
+
 }