Research/encoding exp

client-cpp/src/main/Session.h

@@ -176,7 +176,9 @@ namespace TSEncoding {
         AC = (char) 14,
         SPRINTZ = (char) 15,
         RAKE = (char) 16,
-        RLBE = (char) 17
+        RLBE = (char) 17,
+        BUFF = (char) 18,
+        CHIMP = (char) 19
     };
 }
 

client-py/iotdb/utils/IoTDBConstants.py

@@ -68,6 +68,8 @@ class TSEncoding(Enum):
     SPRINTZ = 15
     RAKE = 16
     RLBE = 17
+    BUFF = 18
+    CHIMP = 19
 
     # this method is implemented to avoid the issue reported by:
     # https://bugs.python.org/issue30545

server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java

@@ -71,6 +71,8 @@ private SchemaUtils() {}
     intSet.add(TSEncoding.SPRINTZ);
     intSet.add(TSEncoding.RAKE);
     intSet.add(TSEncoding.RLBE);
+    intSet.add(TSEncoding.BUFF);
+    intSet.add(TSEncoding.CHIMP);
     schemaChecker.put(TSDataType.INT32, intSet);
     schemaChecker.put(TSDataType.INT64, intSet);
 
@@ -84,6 +86,8 @@ private SchemaUtils() {}
     floatSet.add(TSEncoding.SPRINTZ);
     floatSet.add(TSEncoding.RAKE);
     floatSet.add(TSEncoding.RLBE);
+    floatSet.add(TSEncoding.BUFF);
+    floatSet.add(TSEncoding.CHIMP);
     schemaChecker.put(TSDataType.FLOAT, floatSet);
     schemaChecker.put(TSDataType.DOUBLE, floatSet);
 

tsfile/pom.xml

@@ -32,7 +32,7 @@
     <description>A columnar file format designed for time-series data</description>
     <url>https://github.com/thulab/iotdb/tree/master/tsfile</url>
     <properties>
-        <tsfile.test.skip>true</tsfile.test.skip>
+        <tsfile.test.skip>false</tsfile.test.skip>
         <tsfile.it.skip>${tsfile.test.skip}</tsfile.it.skip>
         <tsfile.ut.skip>${tsfile.test.skip}</tsfile.ut.skip>
     </properties>
@@ -94,6 +94,11 @@
             <artifactId>awaitility</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.sourceforge.javacsv</groupId>
+            <artifactId>javacsv</artifactId>
+            <version>2.0</version>
+        </dependency>
         <!-- antlr -->
         <dependency>
             <groupId>org.apache.iotdb</groupId>

tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java

@@ -140,9 +140,8 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
           case INT64:
             return new LongSprintzDecoder();
           case FLOAT:
-            return new FloatSprintzDecoder();
           case DOUBLE:
-            return new DoubleSprintzDecoder();
+            return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType);
           default:
             throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
         }
@@ -153,9 +152,8 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
           case INT64:
             return new LongRAKEDecoder();
           case FLOAT:
-            return new FloatRAKEDecoder();
           case DOUBLE:
-            return new DoubleRAKEDecoder();
+            return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType);
           default:
             throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
         }
@@ -166,9 +164,34 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
           case INT64:
             return new LongRLBEDecoder();
           case FLOAT:
-            return new FloatRLBEDecoder();
           case DOUBLE:
-            return new DoubleRLBEDecoder();
+            return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType);
+          default:
+            throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+        }
+      case BUFF:
+        switch (dataType) {
+          case INT32:
+            return new IntBUFFDecoder();
+          case INT64:
+            return new LongBUFFDecoder();
+          case FLOAT:
+            return new FloatBUFFDecoder();
+          case DOUBLE:
+            return new DoubleBUFFDecoder();
+          default:
+            throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+        }
+      case CHIMP:
+        switch (dataType) {
+          case FLOAT:
+            return new SinglePrecisionChimpDecoder();
+          case DOUBLE:
+            return new DoublePrecisionChimpDecoder();
+          case INT32:
+            return new IntChimpDecoder();
+          case INT64:
+            return new LongChimpDecoder();
           default:
             throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
         }

tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoubleBUFFDecoder.java

@@ -0,0 +1,86 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class DoubleBUFFDecoder extends Decoder {
+
+  private boolean readMeta;
+  private long minValue;
+  private int countA, countB, n;
+
+  private byte buffer = 0;
+  private int bitsLeft = 0;
+
+  public DoubleBUFFDecoder() {
+    super(TSEncoding.BUFF);
+    reset();
+  }
+
+  @Override
+  public boolean hasNext(ByteBuffer in) throws IOException {
+    if (!readMeta) readMeta(in);
+    return n > 0;
+  }
+
+  @Override
+  public double readDouble(ByteBuffer in) {
+    if (!readMeta) readMeta(in);
+    long partA = readBits(in, countA);
+    double partB = 0, base = 1;
+    for (int i = 0; i < countB; i++) {
+      base /= 2;
+      if (readBit(in)) partB += base;
+    }
+    n--;
+    return minValue + partA + partB;
+  }
+
+  @Override
+  public void reset() {
+    readMeta = false;
+
+    buffer = 0;
+    bitsLeft = 0;
+  }
+
+  private void readMeta(ByteBuffer in) {
+    n = (int) readBits(in, Integer.SIZE);
+    if (n > 0) {
+      countA = (int) readBits(in, Integer.SIZE);
+      countB = (int) readBits(in, Integer.SIZE);
+      minValue = (int) readBits(in, Long.SIZE);
+    }
+    readMeta = true;
+  }
+
+  protected long readBits(ByteBuffer in, int len) {
+    long result = 0;
+    for (int i = 0; i < len; i++) {
+      result <<= 1;
+      if (readBit(in)) result |= 1;
+    }
+    return result;
+  }
+
+  /**
+   * Reads the next bit and returns a boolean representing it.
+   *
+   * @return true if the next bit is 1, otherwise 0.
+   */
+  protected boolean readBit(ByteBuffer in) {
+    flipByte(in);
+    boolean bit = ((buffer >> (bitsLeft - 1)) & 1) == 1;
+    bitsLeft--;
+    return bit;
+  }
+
+  protected void flipByte(ByteBuffer in) {
+    if (bitsLeft == 0) {
+      buffer = in.get();
+      bitsLeft = Byte.SIZE;
+    }
+  }
+}

tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DoublePrecisionChimpDecoder.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class includes code modified from Panagiotis Liakos chimp project.
+ *
+ * <p>Copyright: 2022- Panagiotis Liakos, Katia Papakonstantinopoulou and Yannis Kotidis
+ *
+ * <p>Project page: https://github.com/panagiotisl/chimp
+ *
+ * <p>License: http://www.apache.org/licenses/LICENSE-2.0
+ */
+public class DoublePrecisionChimpDecoder extends LongChimpDecoder {
+
+  private static final long CHIMP_ENCODING_ENDING = Double.doubleToRawLongBits(Double.NaN);
+
+  @Override
+  public final double readDouble(ByteBuffer in) {
+    return Double.longBitsToDouble(readLong(in));
+  }
+
+  @Override
+  protected long cacheNext(ByteBuffer in) {
+    readNext(in);
+    if (storedValues[current] == CHIMP_ENCODING_ENDING) {
+      hasNext = false;
+    }
+    return storedValues[current];
+  }
+}

tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatBUFFDecoder.java

@@ -0,0 +1,86 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class FloatBUFFDecoder extends Decoder {
+
+  private boolean readMeta;
+  private int minValue;
+  private int countA, countB, n;
+
+  private byte buffer = 0;
+  private int bitsLeft = 0;
+
+  public FloatBUFFDecoder() {
+    super(TSEncoding.BUFF);
+    reset();
+  }
+
+  @Override
+  public boolean hasNext(ByteBuffer in) throws IOException {
+    if (!readMeta) readMeta(in);
+    return n > 0;
+  }
+
+  @Override
+  public float readFloat(ByteBuffer in) {
+    if (!readMeta) readMeta(in);
+    int partA = readBits(in, countA);
+    float partB = 0, base = 1;
+    for (int i = 0; i < countB; i++) {
+      base /= 2;
+      if (readBit(in)) partB += base;
+    }
+    n--;
+    return minValue + partA + partB;
+  }
+
+  @Override
+  public void reset() {
+    readMeta = false;
+
+    buffer = 0;
+    bitsLeft = 0;
+  }
+
+  private void readMeta(ByteBuffer in) {
+    n = (int) readBits(in, Integer.SIZE);
+    if (n > 0) {
+      countA = (int) readBits(in, Integer.SIZE);
+      countB = (int) readBits(in, Integer.SIZE);
+      minValue = (int) readBits(in, Integer.SIZE);
+    }
+    readMeta = true;
+  }
+
+  protected int readBits(ByteBuffer in, int len) {
+    int result = 0;
+    for (int i = 0; i < len; i++) {
+      result <<= 1;
+      if (readBit(in)) result |= 1;
+    }
+    return result;
+  }
+
+  /**
+   * Reads the next bit and returns a boolean representing it.
+   *
+   * @return true if the next bit is 1, otherwise 0.
+   */
+  protected boolean readBit(ByteBuffer in) {
+    flipByte(in);
+    boolean bit = ((buffer >> (bitsLeft - 1)) & 1) == 1;
+    bitsLeft--;
+    return bit;
+  }
+
+  protected void flipByte(ByteBuffer in) {
+    if (bitsLeft == 0) {
+      buffer = in.get();
+      bitsLeft = Byte.SIZE;
+    }
+  }
+}