Benchmark adds four new data types: STRING BLOB TIMESTAMP DATE (#434)

configuration/conf/config.properties

@@ -230,8 +230,8 @@
 # 浮点数小数位数
 # DOUBLE_LENGTH=2
 
-# 插入数据的数据类型的比例，BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT
-# INSERT_DATATYPE_PROPORTION=1:1:1:1:1:1
+# 插入数据的数据类型的比例，BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT:STRING:BLOB:TIMESTAMP:DATE
+# INSERT_DATATYPE_PROPORTION=1:1:1:1:1:1:0:0:0:0
 
 ################ IoTDB相关元数据参数 #####################
 # 压缩方式 UNCOMPRESSED | SNAPPY | LZ4 (仅对IoTDB有效)
@@ -255,6 +255,17 @@
 # TEXT: PLAIN/DICTIONARY 等，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致
 # ENCODING_TEXT=DICTIONARY
 
+# STRING: PLAIN 暂不支持DICTIONARY，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致
+# ENCODING_STRING=PLAIN
+
+# BLOB: PLAIN  暂不支持DICTIONARY，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致
+# ENCODING_BLOB=PLAIN
+
+# TIMESTAMP: PLAIN/RLE/TS_2DIFF/GORILLA/ZIGZAG/CHIMP/SPRINTZ/RLBE，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致
+# ENCODING_TIMESTAMP=TS_2DIFF
+
+# DATE: PLAIN/RLE/TS_2DIFF/GORILLA/ZIGZAG/CHIMP/SPRINTZ/RLBE，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致
+# ENCODING_DATE=TS_2DIFF
 ################ 真实数据集：测试数据 #####################
 # 如下两个参数，当且仅当BENCHMARK_MODE = verificationWriteMode 和 verificationQueryMode 时生效
 # 数据文件地址

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java

@@ -41,7 +41,8 @@
 
 public class Config {
   private static final Logger LOGGER = LoggerFactory.getLogger(Config.class);
-
+  /** The total number of data types supported by the benchmark */
+  private final int typeNumber = 10;
   // 初始化
   // 初始化：清理数据
   /** Whether to clear old data before test */
@@ -126,10 +127,10 @@ public class Config {
   /** The length of double */
   private int DOUBLE_LENGTH = 2;
   /**
-   * 插入数据的比例 Data Type, D1:D2:D3:D4:D5:D6 D1: BOOLEAN D2: INT32 D3: INT64 D4: FLOAT D5: DOUBLE D6:
-   * TEXT
+   * 插入数据的比例 Data Type, D1:D2:D3:D4:D5:D6:D7:D8:D9:D9:D10 D1: BOOLEAN D2: INT32 D3: INT64 D4: FLOAT
+   * D5: DOUBLE D6:TEXT D7: STRING D8: BLOB D9: TIMESTAMP D10: DATE
    */
-  private String INSERT_DATATYPE_PROPORTION = "1:1:1:1:1:1";
+  private String INSERT_DATATYPE_PROPORTION = "1:1:1:1:1:1:0:0:0:0";
 
   /** The compress of data */
   private String COMPRESSOR = "LZ4";
@@ -145,6 +146,14 @@ public class Config {
   private String ENCODING_DOUBLE = "GORILLA";
   /** The encoding of text */
   private String ENCODING_TEXT = "DICTIONARY";
+  /** The encoding of string */
+  private String ENCODING_STRING = "PLAIN";
+  /** The encoding of blob */
+  private String ENCODING_BLOB = "PLAIN";
+  /** The encoding of timestamp */
+  private String ENCODING_TIMESTAMP = "TS_2DIFF";
+  /** The encoding of date */
+  private String ENCODING_DATE = "TS_2DIFF";
 
   // 测试数据相关参数
 
@@ -511,7 +520,6 @@ public void initSensorFunction() {
 
   /** According to the number of sensors, initialize the sensor number */
   void initSensorCodes() {
-    int typeNumber = 6;
     double[] probabilities = generateProbabilities(typeNumber);
     if (probabilities.length == 0) {
       return;
@@ -529,14 +537,12 @@ void initSensorCodes() {
     }
   }
 
-  /** Generate Probabilities according to proportion(e.g. 1:1:1:1:1:1) */
+  /** Generate Probabilities according to proportion(e.g. 1:1:1:1:1:1:0:0:0:0) */
   private double[] generateProbabilities(int typeNumber) {
     // Probabilities for Types
     double[] probabilities = new double[typeNumber + 1];
     // Origin proportion array
     double[] proportions = new double[typeNumber];
-    LOGGER.info(
-        "Init SensorTypes: BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT= {}", INSERT_DATATYPE_PROPORTION);
 
     String[] split = INSERT_DATATYPE_PROPORTION.split(":");
     if (split.length != typeNumber) {
@@ -560,6 +566,10 @@ private double[] generateProbabilities(int typeNumber) {
     return probabilities;
   }
 
+  public int getTypeNumber() {
+    return typeNumber;
+  }
+
   public String getREST_AUTHORIZATION() {
     return REST_AUTHORIZATION;
   }
@@ -801,6 +811,38 @@ public void setENCODING_TEXT(String ENCODING_TEXT) {
     this.ENCODING_TEXT = ENCODING_TEXT;
   }
 
+  public String getENCODING_STRING() {
+    return ENCODING_STRING;
+  }
+
+  public void setENCODING_STRING(String ENCODING_STRING) {
+    this.ENCODING_STRING = ENCODING_STRING;
+  }
+
+  public String getENCODING_BLOB() {
+    return ENCODING_BLOB;
+  }
+
+  public void setENCODING_BLOB(String ENCODING_BLOB) {
+    this.ENCODING_BLOB = ENCODING_BLOB;
+  }
+
+  public String getENCODING_TIMESTAMP() {
+    return ENCODING_TIMESTAMP;
+  }
+
+  public void setENCODING_TIMESTAMP(String ENCODING_TIMESTAMP) {
+    this.ENCODING_TIMESTAMP = ENCODING_TIMESTAMP;
+  }
+
+  public String getENCODING_DATE() {
+    return ENCODING_DATE;
+  }
+
+  public void setENCODING_DATE(String ENCODING_DATE) {
+    this.ENCODING_DATE = ENCODING_DATE;
+  }
+
   public String getFILE_PATH() {
     return FILE_PATH;
   }

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java

@@ -198,7 +198,6 @@ private void loadProps() {
         config.setINSERT_DATATYPE_PROPORTION(
             properties.getProperty(
                 "INSERT_DATATYPE_PROPORTION", config.getINSERT_DATATYPE_PROPORTION()));
-
         config.setCOMPRESSOR(properties.getProperty("COMPRESSOR", config.getCOMPRESSOR()));
         config.setENCODING_BOOLEAN(
             properties.getProperty("ENCODING_BOOLEAN", config.getENCODING_BOOLEAN()));
@@ -211,6 +210,12 @@ private void loadProps() {
         config.setENCODING_DOUBLE(
             properties.getProperty("ENCODING_DOUBLE", config.getENCODING_DOUBLE()));
         config.setENCODING_TEXT(properties.getProperty("ENCODING_TEXT", config.getENCODING_TEXT()));
+        config.setENCODING_STRING(
+            properties.getProperty("ENCODING_STRING", config.getENCODING_STRING()));
+        config.setENCODING_BLOB(properties.getProperty("ENCODING_BLOB", config.getENCODING_BLOB()));
+        config.setENCODING_TIMESTAMP(
+            properties.getProperty("ENCODING_TIMESTAMP", config.getENCODING_TIMESTAMP()));
+        config.setENCODING_DATE(properties.getProperty("ENCODING_DATE", config.getENCODING_DATE()));
 
         config.setFILE_PATH(properties.getProperty("FILE_PATH", config.getFILE_PATH()));
         config.setBIG_BATCH_SIZE(
@@ -601,6 +606,7 @@ private boolean checkConfig() {
       default:
         break;
     }
+    result &= checkInsertDataTypeProportion();
     result &= checkOperationProportion();
     if (config.getCLIENT_NUMBER() == 0) {
       LOGGER.error("Client number can't be zero");
@@ -744,6 +750,25 @@ private boolean checkDatabaseVerification(DBConfig dbConfig) {
     return result;
   }
 
+  // Only iotdb supports STRING BLOB TIMESTAMP DATE
+  protected boolean checkInsertDataTypeProportion() {
+    DBType dbType = config.getDbConfig().getDB_SWITCH().getType();
+    String[] splits = config.getINSERT_DATATYPE_PROPORTION().split(":");
+    if (dbType != DBType.IoTDB && dbType != DBType.DoubleIoTDB) {
+      // When not iotdb, the last four digits of the data ratio must be 0
+      for (int i = config.getTypeNumber() - 4; i < splits.length; i++) {
+        if (!splits[i].equals("0")) {
+          LOGGER.warn("INSERT_DATATYPE_PROPORTION error, this database do not support those type.");
+          return false;
+        }
+      }
+    }
+    LOGGER.info(
+        "Init SensorTypes: BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT:STRING:BLOB:TIMESTAMP:DATE= {}",
+        config.getINSERT_DATATYPE_PROPORTION());
+    return true;
+  }
+
   /**
    * Compare whether each field of the two objects is the same. This function is not used in the
    * normal operation of the benchmark， but is used when adding parameters or when the default

core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SensorType.java

@@ -25,7 +25,11 @@ public enum SensorType {
   INT64("INT64"),
   FLOAT("FLOAT"),
   DOUBLE("DOUBLE"),
-  TEXT("TEXT");
+  TEXT("TEXT"),
+  STRING("STRING"),
+  BLOB("BLOB"),
+  TIMESTAMP("TIMESTAMP"),
+  DATE("DATE");
 
   public String name;
 

core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java

@@ -26,13 +26,16 @@
 import cn.edu.tsinghua.iot.benchmark.measurement.persistence.PersistenceFactory;
 import cn.edu.tsinghua.iot.benchmark.measurement.persistence.TestDataPersistence;
 import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 public class TestWithDefaultPathMode extends BaseMode {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestWithDefaultPathMode.class);
   private static final Config config = ConfigDescriptor.getInstance().getConfig();
 
   @Override

core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java

@@ -23,14 +23,14 @@
 import cn.edu.tsinghua.iot.benchmark.distribution.PoissonDistribution;
 import cn.edu.tsinghua.iot.benchmark.distribution.ProbTool;
 import cn.edu.tsinghua.iot.benchmark.entity.Sensor;
-import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType;
 import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException;
 import cn.edu.tsinghua.iot.benchmark.function.Function;
 import cn.edu.tsinghua.iot.benchmark.function.FunctionParam;
 import cn.edu.tsinghua.iot.benchmark.utils.TimeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
@@ -152,37 +152,40 @@ private static Object[][] initWorkloadValues() {
           // periodic
           long currentTimestamp = getCurrentTimestampStatic(i);
           Object value;
-          if (sensor.getSensorType() == SensorType.TEXT) {
-            // TEXT case: pick STRING_LENGTH chars to be a String for insertion.
-            StringBuffer builder = new StringBuffer(config.getSTRING_LENGTH());
-            for (int k = 0; k < config.getSTRING_LENGTH(); k++) {
-              builder.append(CHAR_TABLE.charAt(dataRandom.nextInt(CHAR_TABLE.length())));
-            }
-            value = builder.toString();
-          } else {
-            // not TEXT case
-            FunctionParam param = config.getSENSOR_FUNCTION().get(sensor.getName());
-            Number number = Function.getValueByFunctionIdAndParam(param, currentTimestamp);
-            switch (sensor.getSensorType()) {
-              case BOOLEAN:
-                value = number.floatValue() > ((param.getMax() + param.getMin()) / 2);
-                break;
-              case INT32:
-                value = number.intValue();
-                break;
-              case INT64:
-                value = number.longValue();
-                break;
-              case FLOAT:
-                value = number.floatValue();
-                break;
-              case DOUBLE:
-                value = Math.round(number.doubleValue() * ratio) / ratio;
-                break;
-              default:
-                value = null;
-                break;
-            }
+          FunctionParam param = config.getSENSOR_FUNCTION().get(sensor.getName());
+          Number number = Function.getValueByFunctionIdAndParam(param, currentTimestamp);
+          switch (sensor.getSensorType()) {
+            case BOOLEAN:
+              value = number.floatValue() > ((param.getMax() + param.getMin()) / 2);
+              break;
+            case INT32:
+              value = number.intValue();
+              break;
+            case INT64:
+            case TIMESTAMP:
+              value = number.longValue();
+              break;
+            case FLOAT:
+              value = number.floatValue();
+              break;
+            case DOUBLE:
+              value = Math.round(number.doubleValue() * ratio) / ratio;
+              break;
+            case TEXT:
+            case STRING:
+            case BLOB:
+              StringBuffer builder = new StringBuffer(config.getSTRING_LENGTH());
+              for (int k = 0; k < config.getSTRING_LENGTH(); k++) {
+                builder.append(CHAR_TABLE.charAt(dataRandom.nextInt(CHAR_TABLE.length())));
+              }
+              value = builder.toString();
+              break;
+            case DATE:
+              value = LocalDate.ofEpochDay(number.intValue());
+              break;
+            default:
+              throw new UnsupportedOperationException(
+                  sensor.getSensorType() + ": This data type is not supported.");
           }
           workloadValues[sensorIndex][i] = value;
         }

core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java

@@ -226,7 +226,10 @@ private List<DeviceSchema> getQueryDeviceSchemaList(boolean typeAllow) throws Wo
         Sensor sensor = sensors.get(sensorId);
         if (!typeAllow) {
           SensorType sensorType = sensor.getSensorType();
-          if (sensorType == SensorType.BOOLEAN || sensorType == SensorType.TEXT) {
+          if (sensorType == SensorType.BOOLEAN
+              || sensorType == SensorType.TEXT
+              || sensorType == SensorType.STRING
+              || sensorType == SensorType.BLOB) {
             continue;
           }
         }

iotdb-1.3/pom.xml

@@ -47,7 +47,7 @@
   <properties>
     <!-- This was the last version to support Java 8 -->
     <logback.version>1.3.14</logback.version>
-    <iotdb.version>1.3.1</iotdb.version>
+    <iotdb.version>1.3.3-SNAPSHOT</iotdb.version>
     <okhttp3.version>4.12.0</okhttp3.version>
     <gson.version>2.10.1</gson.version>
   </properties>

iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java

@@ -2,8 +2,9 @@
 
 import org.apache.iotdb.rpc.IoTDBConnectionException;
 import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.write.record.Tablet;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
 
 import java.util.List;
 

iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java

@@ -3,7 +3,8 @@
 import org.apache.iotdb.isession.SessionDataSet;
 import org.apache.iotdb.rpc.IoTDBConnectionException;
 import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.tsfile.read.common.RowRecord;
+
+import org.apache.tsfile.read.common.RowRecord;
 
 public interface ISessionDataSet {
   RowRecord next() throws IoTDBConnectionException, StatementExecutionException;