add more tests,improve the JavaDocs,fix some bugs,add some benchmarks

RoaringBitmap/src/main/java/org/roaringbitmap/art/Art.java

@@ -88,14 +88,16 @@ private Node findByKey(Node node, byte[] key, int depth) {
 
   /**
    * a convenient method to traverse the key space in ascending order.
+   * @param containers input containers
+   * @return the key iterator
    */
   public KeyIterator iterator(Containers containers) {
     return new KeyIterator(this, containers);
   }
 
   /**
    * remove the key from the art if it's there.
-   *
+   * @param key the high 48 bit key
    * @return the corresponding containerIdx or -1 indicating not exist
    */
   public long remove(byte[] key) {
@@ -217,9 +219,15 @@ private Node insert(Node node, byte[] key, int depth, long containerIdx) {
         //split the current internal node, spawn a fresh node4 and let the
         //current internal node as its children.
         Node4.insert(node4, node, node.prefix[mismatchPos]);
-        node.prefixLength = (byte) (node.prefixLength - (mismatchPos + (byte) 1));
+        int nodeOriginalPrefixLength = node.prefixLength;
+        node.prefixLength = (byte) (nodeOriginalPrefixLength - (mismatchPos + (byte) 1));
         //move the remained common prefix of the initial internal node
-        System.arraycopy(node.prefix, mismatchPos + 1, node.prefix, 0, node.prefixLength);
+        if (node.prefixLength > 0) {
+          System.arraycopy(node.prefix, mismatchPos + 1, node.prefix, 0, node.prefixLength);
+        } else {
+          //TODO:to reduce the 0 prefix memory space,we could mark the prefix as null
+          node.prefix = new byte[0];
+        }
         LeafNode leafNode = new LeafNode(key, containerIdx);
         Node4.insert(node4, leafNode, key[mismatchPos + depth]);
         return node4;
@@ -261,18 +269,22 @@ public Node getRoot() {
   }
 
   public void serializeArt(DataOutput dataOutput) throws IOException {
+    dataOutput.writeLong(Long.reverseBytes(keySize));
     serialize(root, dataOutput);
   }
 
   public void deserializeArt(DataInput dataInput) throws IOException {
+    keySize = Long.reverseBytes(dataInput.readLong());
     root = deserialize(dataInput);
   }
 
   public void serializeArt(ByteBuffer byteBuffer) throws IOException {
+    byteBuffer.putLong(keySize);
     serialize(root, byteBuffer);
   }
 
   public void deserializeArt(ByteBuffer byteBuffer) throws IOException {
+    keySize = byteBuffer.getLong();
     root = deserialize(byteBuffer);
   }
 
@@ -359,7 +371,7 @@ private Node deserialize(ByteBuffer byteBuffer) throws IOException {
   }
 
   public long serializeSizeInBytes() {
-    return serializeSizeInBytes(this.root);
+    return serializeSizeInBytes(this.root) + 8;
   }
 
   public long getKeySize() {

RoaringBitmap/src/main/java/org/roaringbitmap/art/ContainerIterator.java

@@ -92,6 +92,7 @@ public long getCurrentContainerIdx() {
 
   /**
    * replace current container
+   * @param container the fresh container which is to replace the current old one
    */
   public void replace(Container container) {
     int secondLevelArrIdx = currentSecondLevelArrIdx - 1;

RoaringBitmap/src/main/java/org/roaringbitmap/art/Node.java

@@ -33,44 +33,55 @@ public Node(NodeType nodeType, int compressedPrefixSize) {
 
   /**
    * get the position of a child corresponding to the input key 'k'
+   * @param k a key value of the byte range
+   * @return the child position corresponding to the key 'k'
    */
   public abstract int getChildPos(byte k);
 
   /**
    * get the child at the specified position in the node, the 'pos' range from 0 to count
+   * @param pos the position
+   * @return a Node corresponding to the input position
    */
   public abstract Node getChild(int pos);
 
   /**
    * replace the position child to the fresh one
+   * @param pos the position
+   * @param freshOne the fresh node to replace the old one
    */
   public abstract void replaceNode(int pos, Node freshOne);
 
   /**
    * get the position of the min element in current node.
+   * @return the minimum key's position
    */
   public abstract int getMinPos();
 
   /**
    * get the next position in the node
    *
-   * @param pos current position
+   * @param pos current position,-1 to start from the min one
+   * @return the next larger byte key's position which is close to 'pos' position,-1 for end
    */
   public abstract int getNextLargerPos(int pos);
 
   /**
    * get the max child's position
+   * @return the max byte key's position
    */
   public abstract int getMaxPos();
 
   /**
    * get the next smaller element's position
+   * @param pos the position,-1 to start from the largest one
+   * @return the next smaller key's position which is close to input 'pos' position,-1 for end
    */
   public abstract int getNextSmallerPos(int pos);
 
   /**
    * remove the specified position child
-   *
+   * @param pos the position to remove
    * @return an adaptive changed fresh node of the current node
    */
   public abstract Node remove(int pos);
@@ -143,28 +154,37 @@ public static Node deserialize(ByteBuffer byteBuffer) throws IOException {
   /**
    * replace the node's children according to the given children parameter while doing the
    * deserialization phase.
+   * @param children all the not null children nodes in key byte ascending order,no null element
    */
-  public abstract void replaceChildren(Node[] children);
+  abstract void replaceChildren(Node[] children);
 
   /**
    * serialize the node's body content
+   * @param dataOutput the DataOutput
+   * @throws IOException exception indicates serialization errors
    */
-  public abstract void serializeNodeBody(DataOutput dataOutput) throws IOException;
+  abstract void serializeNodeBody(DataOutput dataOutput) throws IOException;
 
   /**
    * serialize the node's body content
+   * @param byteBuffer the ByteBuffer
+   * @throws IOException exception indicates serialization errors
    */
-  public abstract void serializeNodeBody(ByteBuffer byteBuffer) throws IOException;
+  abstract void serializeNodeBody(ByteBuffer byteBuffer) throws IOException;
 
   /**
    * deserialize the node's body content
+   * @param dataInput the DataInput
+   * @throws IOException exception indicates deserialization errors
    */
-  public abstract void deserializeNodeBody(DataInput dataInput) throws IOException;
+  abstract void deserializeNodeBody(DataInput dataInput) throws IOException;
 
   /**
    * deserialize the node's body content
+   * @param byteBuffer the ByteBuffer
+   * @throws IOException exception indicates deserialization errors
    */
-  public abstract void deserializeNodeBody(ByteBuffer byteBuffer) throws IOException;
+  abstract void deserializeNodeBody(ByteBuffer byteBuffer) throws IOException;
 
   /**
    * the serialized size except the common node header part
@@ -177,6 +197,9 @@ public static Node deserialize(ByteBuffer byteBuffer) throws IOException {
    * insert the LeafNode as a child of the current internal node
    *
    * @param current current internal node
+   * @param childNode the leaf node
+   * @param key the key byte reference to the child leaf node
+   * @return an adaptive changed node of the input 'current' node
    */
   public static Node insertLeaf(Node current, LeafNode childNode, byte key) {
     switch (current.nodeType) {
@@ -195,6 +218,8 @@ public static Node insertLeaf(Node current, LeafNode childNode, byte key) {
 
   /**
    * copy the prefix between two nodes
+   * @param src the source node
+   * @param dst the destination node
    */
   public static void copyPrefix(Node src, Node dst) {
     dst.prefixLength = src.prefixLength;
@@ -204,8 +229,11 @@ public static void copyPrefix(Node src, Node dst) {
   /**
    * search the position of the input byte key in the node's key byte array part
    *
+   * @param key the input key byte array
    * @param fromIndex inclusive
    * @param toIndex exclusive
+   * @param k the target key byte value
+   * @return the array offset of the target input key 'k' or -1 to not found
    */
   public static int binarySearch(byte[] key, int fromIndex, int toIndex,
       byte k) {
@@ -245,7 +273,7 @@ private void serializeHeader(ByteBuffer byteBuffer) throws IOException {
     byteBuffer.putShort(this.count);
     byteBuffer.put(this.prefixLength);
     if (prefixLength > 0) {
-      byteBuffer.put(this.prefix);
+      byteBuffer.put(this.prefix, 0, prefixLength);
     }
   }
 

RoaringBitmap/src/main/java/org/roaringbitmap/art/Node16.java

@@ -189,7 +189,13 @@ public int serializeNodeBodySizeInBytes() {
 
   @Override
   public void replaceChildren(Node[] children) {
-    this.children = children;
+    int pos = this.getNextLargerPos(ILLEGAL_IDX);
+    int offset = 0;
+    while (pos != ILLEGAL_IDX) {
+      this.children[pos] = children[offset];
+      pos = this.getNextLargerPos(pos);
+      offset++;
+    }
   }
 
   /**

RoaringBitmap/src/main/java/org/roaringbitmap/art/Node256.java

@@ -1,9 +1,12 @@
 package org.roaringbitmap.art;
 
+import static java.lang.Long.numberOfTrailingZeros;
+
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.LongBuffer;
 
 public class Node256 extends Node {
 
@@ -94,6 +97,9 @@ public int getNextSmallerPos(int pos) {
     if (pos == ILLEGAL_IDX) {
       pos = 256;
     }
+    if (pos == 0) {
+      return ILLEGAL_IDX;
+    }
     pos--;
     int longPos = pos >>> 6;
     long longVal = bitmapMask[longPos] & (LONG_MASK >>> -(pos + 1));
@@ -158,36 +164,58 @@ public Node remove(int pos) {
 
   @Override
   public void replaceChildren(Node[] children) {
-    for (int i = 0; i < 256; i++) {
-      if (children[i] != null) {
-        Node256.insert(this, children[i], (byte) i);
+    if (children.length == this.children.length) {
+      //short circuit path
+      this.children = children;
+      return;
+    }
+    int offset = 0;
+    int x = 0;
+    for (long longv : bitmapMask) {
+      int w = Long.bitCount(longv);
+      for (int i = 0; i < w; i++) {
+        int pos = x * 64 + numberOfTrailingZeros(longv);
+        this.children[pos] = children[offset + i];
+        longv &= (longv - 1);
       }
+      offset += w;
+      x++;
     }
   }
 
+
   @Override
   public void serializeNodeBody(DataOutput dataOutput) throws IOException {
-
+    for (long longv : bitmapMask) {
+      dataOutput.writeLong(Long.reverseBytes(longv));
+    }
   }
 
   @Override
   public void serializeNodeBody(ByteBuffer byteBuffer) throws IOException {
-
+    LongBuffer longBuffer = byteBuffer.asLongBuffer();
+    longBuffer.put(bitmapMask);
+    byteBuffer.position(byteBuffer.position() + 4 * 8);
   }
 
   @Override
   public void deserializeNodeBody(DataInput dataInput) throws IOException {
-
+    for (int i = 0; i < 4; i++) {
+      long longv = Long.reverseBytes(dataInput.readLong());
+      bitmapMask[i] = longv;
+    }
   }
 
   @Override
   public void deserializeNodeBody(ByteBuffer byteBuffer) throws IOException {
-
+    LongBuffer longBuffer = byteBuffer.asLongBuffer();
+    longBuffer.get(bitmapMask);
+    byteBuffer.position(byteBuffer.position() + 4 * 8);
   }
 
   @Override
   public int serializeNodeBodySizeInBytes() {
-    return 0;
+    return 4 * 8;
   }
 }
 

RoaringBitmap/src/main/java/org/roaringbitmap/art/Node4.java

@@ -102,7 +102,6 @@ public Node remove(int pos) {
     count--;
     key = IntegerUtil.shiftLeftFromSpecifiedPosition(key, pos, count);
     for (; pos < count; pos++) {
-      //key[pos] = key[pos + 1];
       children[pos] = children[pos + 1];
     }
     if (count == 1) {
@@ -155,7 +154,13 @@ public int serializeNodeBodySizeInBytes() {
 
   @Override
   public void replaceChildren(Node[] children) {
-    this.children = children;
+    int pos = getNextLargerPos(ILLEGAL_IDX);
+    int offset = 0;
+    while (pos != ILLEGAL_IDX) {
+      this.children[pos] = children[offset];
+      pos = getNextLargerPos(pos);
+      offset++;
+    }
   }
 
   /**

RoaringBitmap/src/main/java/org/roaringbitmap/art/Node48.java

@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.LongBuffer;
 import java.util.Arrays;
 import org.roaringbitmap.longlong.LongUtils;
 
@@ -234,10 +235,9 @@ public void serializeNodeBody(DataOutput dataOutput) throws IOException {
 
   @Override
   public void serializeNodeBody(ByteBuffer byteBuffer) throws IOException {
-    for (int i = 0; i < 32; i++) {
-      long longv = childIndex[i];
-      byteBuffer.putLong(longv);
-    }
+    LongBuffer longBuffer = byteBuffer.asLongBuffer();
+    longBuffer.put(childIndex);
+    byteBuffer.position(byteBuffer.position() + 32 * 8);
   }
 
   @Override
@@ -249,9 +249,9 @@ public void deserializeNodeBody(DataInput dataInput) throws IOException {
 
   @Override
   public void deserializeNodeBody(ByteBuffer byteBuffer) throws IOException {
-    for (int i = 0; i < 32; i++) {
-      childIndex[i] = byteBuffer.getLong();
-    }
+    LongBuffer longBuffer = byteBuffer.asLongBuffer();
+    longBuffer.get(childIndex);
+    byteBuffer.position(byteBuffer.position() + 32 * 8);
   }
 
   @Override
@@ -261,13 +261,15 @@ public int serializeNodeBodySizeInBytes() {
 
   @Override
   public void replaceChildren(Node[] children) {
+    int step = 0;
     for (int i = 0; i < 32; i++) {
       long longv = childIndex[i];
       for (int j = 7; j >= 0; j--) {
         byte bytePos = (byte) (longv >>> (j * 8));
         int unsignedPos = Byte.toUnsignedInt(bytePos);
         if (bytePos != EMPTY_VALUE) {
-          this.children[unsignedPos] = children[unsignedPos];
+          this.children[unsignedPos] = children[step];
+          step++;
         }
       }
     }