diff --git a/docs/UserGuide/Administration-Management/Multi-Tenancy.md b/docs/UserGuide/Administration-Management/Multi-Tenancy.md index 51f43785c968a..d5dd812b23b68 100644 --- a/docs/UserGuide/Administration-Management/Multi-Tenancy.md +++ b/docs/UserGuide/Administration-Management/Multi-Tenancy.md @@ -23,7 +23,7 @@ # Multi-Tenancy -IoTDB provides multi tenant operations, mainly to limit the resources of the database or the database when users are using it. +Times provides multi tenant operations, mainly to limit the resources of the database or the database when users are using it. ## Space Quota @@ -72,9 +72,9 @@ set space quota timeseries=unlimited on root.sg1; - Show the space quota information of all databases ```SQL -IoTDB> set space quota devices=5,timeseries=10,disk=100g on root.sg1, root.sg2; +Times> set space quota devices=5,timeseries=10,disk=100g on root.sg1, root.sg2; Msg: The statement is executed successfully. -IoTDB> show space quota; +Times> show space quota; +--------+-------------+-------+----+ |database| quotaType| limit|used| +--------+-------------+-------+----+ @@ -92,7 +92,7 @@ It costs 0.067s - Show the space quota information of the specified database ```SQL -IoTDB> show space quota root.sg1; +Times> show space quota root.sg1; +--------+-------------+-------+----+ |database| quotaType| limit|used| +--------+-------------+-------+----+ @@ -162,7 +162,7 @@ We support viewing quota information. The main syntax is as follows: - View the throttle quota information of all users ```SQL -IoTDB> show throttle quota +Times> show throttle quota +-----+---------+---------+----------+ | User|QuotaType| Limit|Read/Write| +-----+---------+---------+----------+ @@ -178,7 +178,7 @@ It costs 0.007s - View the throttle quota information of the specified user ```SQL -IoTDB> show throttle quota user1; +Times> show throttle quota user1; +-----+---------+---------+----------+ | User|QuotaType| Limit|Read/Write| +-----+---------+---------+----------+ diff --git a/docs/zh/UserGuide/Administration-Management/Multi-Tenancy.md b/docs/zh/UserGuide/Administration-Management/Multi-Tenancy.md index cf03c8829ba03..835fa4367721b 100644 --- a/docs/zh/UserGuide/Administration-Management/Multi-Tenancy.md +++ b/docs/zh/UserGuide/Administration-Management/Multi-Tenancy.md @@ -21,7 +21,7 @@ # 多租户 -IoTDB 提供了多租户操作，主要是对数据库，或者用户在使用的过程中，对数据库进行一定的资源限制。 +Times 提供了多租户操作，主要是对数据库，或者用户在使用的过程中，对数据库进行一定的资源限制。 ## 空间限额 @@ -70,9 +70,9 @@ set space quota timeseries=unlimited on root.sg1; - 查看全部数据库的空间限额信息 ```SQL -IoTDB> set space quota devices=5,timeseries=10,disk=100g on root.sg1, root.sg2; +Times> set space quota devices=5,timeseries=10,disk=100g on root.sg1, root.sg2; Msg: The statement is executed successfully. -IoTDB> show space quota; +Times> show space quota; +--------+-------------+-------+----+ |database| quotaType| limit|used| +--------+-------------+-------+----+ @@ -90,7 +90,7 @@ It costs 0.067s - 查看指定数据库的空间限额信息 ```SQL -IoTDB> show space quota root.sg1; +Times> show space quota root.sg1; +--------+-------------+-------+----+ |database| quotaType| limit|used| +--------+-------------+-------+----+ @@ -162,7 +162,7 @@ set throttle quota cpu=2, mem=2G on user1; - 查看全部用户的throttle quota信息 ```SQL -IoTDB> show throttle quota +Times> show throttle quota +-----+---------+---------+----------+ | User|QuotaType| Limit|Read/Write| +-----+---------+---------+----------+ @@ -178,7 +178,7 @@ It costs 0.007s - 查看指定用户的throttle quota信息 ```SQL -IoTDB> show throttle quota user1; +Times> show throttle quota user1; +-----+---------+---------+----------+ | User|QuotaType| Limit|Read/Write| +-----+---------+---------+----------+ diff --git a/isession/src/main/java/org/apache/iotdb/isession/ISession.java b/isession/src/main/java/org/apache/iotdb/isession/ISession.java index 4ab78f128feb3..d7efa79a36ddb 100644 --- a/isession/src/main/java/org/apache/iotdb/isession/ISession.java +++ b/isession/src/main/java/org/apache/iotdb/isession/ISession.java @@ -20,7 +20,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.template.Template; -import org.apache.iotdb.isession.util.SystemStatus; import org.apache.iotdb.isession.util.Version; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -473,10 +472,7 @@ TSBackupConfigurationResp getBackupConfiguration() long getQueryTimeout(); - @Deprecated - default SystemStatus getSystemStatus() { - return SystemStatus.NORMAL; - } + String getSystemStatus() throws IoTDBConnectionException; @Deprecated default void createTimeseriesOfTemplateOnPath(String path) @@ -486,7 +482,6 @@ default void createTimeseriesOfTemplateOnPath(String path) default void deactivateTemplateOn(String templateName, String prefixPath) throws IoTDBConnectionException, StatementExecutionException {} - @Deprecated default void operationSyncTransmit(ByteBuffer buffer) throws IoTDBConnectionException, StatementExecutionException {} diff --git a/isession/src/main/java/org/apache/iotdb/isession/pool/ISessionPool.java b/isession/src/main/java/org/apache/iotdb/isession/pool/ISessionPool.java index 5e5f7d72b1f52..e9eb5560cc22c 100644 --- a/isession/src/main/java/org/apache/iotdb/isession/pool/ISessionPool.java +++ b/isession/src/main/java/org/apache/iotdb/isession/pool/ISessionPool.java @@ -19,7 +19,6 @@ package org.apache.iotdb.isession.pool; import org.apache.iotdb.isession.template.Template; -import org.apache.iotdb.isession.util.SystemStatus; import org.apache.iotdb.isession.util.Version; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -462,6 +461,8 @@ TSBackupConfigurationResp getBackupConfiguration() long getQueryTimeout(); + String getSystemStatus() throws IoTDBConnectionException; + @Deprecated default void createTimeseriesOfTemplateOnPath(String path) throws IoTDBConnectionException, StatementExecutionException {} @@ -477,17 +478,11 @@ default ISessionDataSetWrapper executeRawDataQuery( return null; } - @Deprecated default boolean operationSyncTransmit(ByteBuffer buffer) throws IoTDBConnectionException, StatementExecutionException { return false; } - @Deprecated - default SystemStatus getSystemStatus() throws IoTDBConnectionException { - return SystemStatus.NORMAL; - } - @Deprecated default boolean isEnableCacheLeader() { return false; diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index fafc4a101e51c..dde26d61da13d 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -292,6 +292,9 @@ private IoTDBConstant() {} public static final String AVAILABLE = "Available"; public static final String USED = "Times-Used"; + // Operation Sync folder name + public static final String OPERATION_SYNC_FOLDER_NAME = "operationsync"; + // client version number public enum ClientVersion { V_0_12, diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/utils/PropertiesUtils.java b/node-commons/src/main/java/org/apache/iotdb/commons/utils/PropertiesUtils.java index 415c7f20d7d7e..8bc76c983919f 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/utils/PropertiesUtils.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/utils/PropertiesUtils.java @@ -766,6 +766,7 @@ public static void getXml(List fileNames, String xmlFilePath) { || line.contains("dn_tracing_dir=datanode\\\\tracing") || line.contains("trigger_lib_dir=ext\\\\trigger") || line.contains("udf_lib_dir=ext\\\\udf") + || line.contains("dn_sync_dir=data\\\\operationsync") || split[0].contains("<")) { currentNum++; continue; @@ -915,6 +916,7 @@ public static void getXml(List fileNames, String xmlFilePath) { || " schema_region_ratis_log_max_size".equals(split[0]) || " data_region_ratis_log_max_size".equals(split[0]) || " iot_consensus_throttle_threshold_in_byte".equals(split[0]) + || " operation_sync_max_log_size".equals(split[0]) || " ratis_first_election_timeout_max_ms".equals(split[0])) { text[6] = "long"; } else { diff --git a/server/src/assembly/resources/conf/configuration-cea.xml b/server/src/assembly/resources/conf/configuration-cea.xml index fa89d2ca73f14..d7aa720c605b6 100644 --- a/server/src/assembly/resources/conf/configuration-cea.xml +++ b/server/src/assembly/resources/conf/configuration-cea.xml @@ -719,6 +719,211 @@ iotdb-datanode.properties + + enable_operation_sync + Operation Sync Configuration Is Operation Sync enable DataType: boolean + Operation Sync Configuration + 0 + + + select + true,false + false + false + false + false + + + iotdb-datanode.properties + + + + secondary_address + The destination IoTDB ip DataType: String + Operation Sync Configuration + 0 + + + string + 127.0.0.1 + 127.0.0.1 + 127.0.0.1 + 127.0.0.1 + + + iotdb-datanode.properties + + + + secondary_port + The destination IoTDB port DataType: int + Operation Sync Configuration + 0 + + + int + 6668 + 6668 + 6668 + 6668 + + + iotdb-datanode.properties + + + + secondary_user + The destination IoTDB username DataType: String + Operation Sync Configuration + 0 + + + string + root + root + root + root + + + iotdb-datanode.properties + + + + secondary_password + The destination IoTDB password DataType: String + Operation Sync Configuration + 0 + + + string + root + root + root + root + + + iotdb-datanode.properties + + + + operation_sync_log_dir + OperationSyncLog dir If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/operationsync). If it is absolute, system will save the data in the exact location it points to. If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. For windows platform If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. dn_sync_dir=data\\operationsync For Linux platform If its prefix is "/", then the path is absolute. Otherwise, it is relative. + Operation Sync Configuration + 0 + + + string + data/operationsync + data/operationsync + data/operationsync + data/operationsync + + + iotdb-datanode.properties + + + + operation_sync_producer_cache_size + The max Length of blocking queue in memory, the data in queue may be discarded when restarted + Operation Sync Configuration + 0 + + + int + 1024 + 1024 + 1024 + 1024 + + + iotdb-datanode.properties + + + + operation_sync_log_file_validity_second + The validity of each OperationSyncLog Unit: second DataType: int + Operation Sync Configuration + 0 + + + int + 30 + 30 + 30 + 30 + + + iotdb-datanode.properties + + + + operation_sync_log_file_num + The maximum id of OperationSyncLog DataType: int + Operation Sync Configuration + 0 + + + int + 32767 + 32767 + 32767 + 32767 + + + iotdb-datanode.properties + + + + operation_sync_max_log_size + The max size of all th eOperationSyncLog. Default is 100GB DataType: long + Operation Sync Configuration + 0 + + + long + 107374182400 + 107374182400 + 107374182400 + 107374182400 + + + iotdb-datanode.properties + + + + operation_sync_producer_cache_size + OperationSyncProducer cache size DataType: int + Operation Sync Configuration + 0 + + + int + 1024 + 1024 + 1024 + 1024 + + + iotdb-datanode.properties + + + + operation_sync_producer_cache_num + OperationSyncProducer DML cache number. DataType: int + Operation Sync Configuration + 0 + + + int + 4 + 4 + 4 + 4 + + + iotdb-datanode.properties + + MAX_HEAP_SIZE Maximum heap size @@ -5628,10 +5833,28 @@ string - ./conf/xxx.license - ./conf/xxx.license - ./conf/xxx.license - ./conf/xxx.license + ./conf/xxx.lic + ./conf/xxx.lic + ./conf/xxx.lic + ./conf/xxx.lic + + + iotdb-common.properties + + + + trial_version + whether enable trial or not, if enabled, the server will run at most 90 days, if not enable, it will check whether the license is correct or not. + License + 0 + + + select + true,false + false + false + false + false iotdb-common.properties @@ -5846,6 +6069,10 @@ Quota Configuration Quota Configuration + + CEA + CEA + License License diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties index e23b7dfcb1c71..a386fe0308bbb 100644 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@ -273,4 +273,64 @@ dn_target_config_node_list=127.0.0.1:10710 # The type of internal reporter in metric module # Options: [MEMORY, IOTDB] # Datatype: String -# dn_metric_internal_reporter_type=MEMORY \ No newline at end of file +# dn_metric_internal_reporter_type=MEMORY + +#################### +### Operation Sync Configuration +#################### + +# Is Operation Sync enable +# DataType: boolean +# enable_operation_sync=false + +# The destination IoTDB ip +# DataType: String +# secondary_address=127.0.0.1 + +# The destination IoTDB port +# DataType: int +# secondary_port=6668 + +# The destination IoTDB username +# DataType: String +# secondary_user=root + +# The destination IoTDB password +# DataType: String +# secondary_password=root + +# OperationSyncLog dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/operationsync). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# operation_sync_log_dir=data\\operationsync +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# operation_sync_log_dir=data/operationsync + +# The max Length of blocking queue in memory, the data in queue may be discarded when restarted +# operation_sync_producer_cache_size=1024 + +# The validity of each OperationSyncLog +# Unit: second +# DataType: int +# operation_sync_log_file_validity=30 + +# The maximum id of OperationSyncLog +# DataType: int +# operation_sync_log_file_num=32767 + +# The max size of all th eOperationSyncLog. Default is 100GB +# DataType: long +# operation_sync_max_log_size=107374182400 + +# OperationSyncProducer cache size +# DataType: int +# operation_sync_producer_cache_size=1024 + +# OperationSyncProducer DML cache number +# DataType: int +# operation_sync_producer_cache_num=8 \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java index 42a4a0c83e7f5..082f8ad901268 100644 --- a/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java @@ -189,6 +189,7 @@ private static AuditLogOperation judgeLogOperation(StatementType type) { return AuditLogOperation.DDL; case LOAD_DATA: case INSERT: + case INSERT_ROW: case BATCH_INSERT: case BATCH_INSERT_ROWS: case BATCH_INSERT_ONE_DEVICE: diff --git a/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java b/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java index 598ed232b032f..e14273629a113 100644 --- a/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java +++ b/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java @@ -319,6 +319,7 @@ private static int translateToPermissionId(StatementType type) { case COUNT: return PrivilegeType.READ_TIMESERIES.ordinal(); case INSERT: + case INSERT_ROW: case LOAD_DATA: case CREATE_INDEX: case BATCH_INSERT: diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index aa8b1ce255abc..01f2d7bae5de8 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1074,6 +1074,33 @@ public class IoTDBConfig { /** whether the local write api records audit logs * */ private boolean enableAuditLogForNativeInsertApi = true; + // Operation Sync Config + private boolean enableOperationSync = false; + + // Secondary IoTDB + private String secondaryAddress = "127.0.0.1"; + private int secondaryPort = 6668; + private String secondaryUser = "root"; + private String secondaryPassword = "root"; + + private int secondarySessionPoolMaxSize = 10; + + // OperationSyncLog dir + private String operationSyncLogDir = + IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.OPERATION_SYNC_FOLDER_NAME; + // The validity of each OperationSyncLog + private int operationSyncLogValiditySecond = 30; + // The maximum id of OperationSyncLog + private int operationSyncLogNum = 32767; + // The maximum number of OperationSyncLog files for DML. Default is 100GB + private long operationSyncMaxLogSize = 107374182400L; + + // OperationSyncProducer DML cache size + private int operationSyncProducerCacheSize = 1024; + + // OperationSyncProducer DML cache number + private int operationSyncProducerCacheNum = 8; + // customizedProperties, this should be empty by default. private Properties customizedProperties = new Properties(); @@ -1184,6 +1211,7 @@ private void formulateFolders() { mqttDir = addDataHomeDir(mqttDir); extPipeDir = addDataHomeDir(extPipeDir); + operationSyncLogDir = addDataHomeDir(operationSyncLogDir); if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) { String hdfsDir = getHdfsDir(); @@ -3686,4 +3714,100 @@ public boolean isEnableAuditLogForNativeInsertApi() { public void setEnableAuditLogForNativeInsertApi(boolean enableAuditLogForNativeInsertApi) { this.enableAuditLogForNativeInsertApi = enableAuditLogForNativeInsertApi; } + + public boolean isEnableOperationSync() { + return enableOperationSync; + } + + public void setEnableOperationSync(boolean enableOperationSync) { + this.enableOperationSync = enableOperationSync; + } + + public String getSecondaryAddress() { + return secondaryAddress; + } + + public void setSecondaryAddress(String secondaryAddress) { + this.secondaryAddress = secondaryAddress; + } + + public int getSecondaryPort() { + return secondaryPort; + } + + public void setSecondaryPort(int secondaryPort) { + this.secondaryPort = secondaryPort; + } + + public String getSecondaryUser() { + return secondaryUser; + } + + public void setSecondaryUser(String secondaryUser) { + this.secondaryUser = secondaryUser; + } + + public String getSecondaryPassword() { + return secondaryPassword; + } + + public void setSecondaryPassword(String secondaryPassword) { + this.secondaryPassword = secondaryPassword; + } + + public int getSecondarySessionPoolMaxSize() { + return secondarySessionPoolMaxSize; + } + + public void setSecondarySessionPoolMaxSize(int secondarySessionPoolMaxSize) { + this.secondarySessionPoolMaxSize = secondarySessionPoolMaxSize; + } + + public String getOperationSyncLogDir() { + return operationSyncLogDir; + } + + public void setOperationSyncLogDir(String operationSyncLogDir) { + this.operationSyncLogDir = operationSyncLogDir; + } + + public int getOperationSyncLogValiditySecond() { + return operationSyncLogValiditySecond; + } + + public void setOperationSyncLogValiditySecond(int operationSyncLogValiditySecond) { + this.operationSyncLogValiditySecond = operationSyncLogValiditySecond; + } + + public int getOperationSyncLogNum() { + return operationSyncLogNum; + } + + public void setOperationSyncLogNum(int operationSyncLogNum) { + this.operationSyncLogNum = operationSyncLogNum; + } + + public long getOperationSyncMaxLogSize() { + return operationSyncMaxLogSize; + } + + public void setOperationSyncMaxLogSize(long operationSyncMaxLogSize) { + this.operationSyncMaxLogSize = operationSyncMaxLogSize; + } + + public int getOperationSyncProducerCacheSize() { + return operationSyncProducerCacheSize; + } + + public void setOperationSyncProducerCacheSize(int operationSyncProducerCacheSize) { + this.operationSyncProducerCacheSize = operationSyncProducerCacheSize; + } + + public int getOperationSyncProducerCacheNum() { + return operationSyncProducerCacheNum; + } + + public void setOperationSyncProducerCacheNum(int operationSyncProducerCacheNum) { + this.operationSyncProducerCacheNum = operationSyncProducerCacheNum; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 137b1cef49e44..7dbae66278b61 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1009,11 +1009,64 @@ public void loadProperties(Properties properties) { // CEA enable loadCeaProps(properties); + // load sync + loadSync(properties); + conf.setTimePartitionInterval( DateTimeUtils.convertMilliTimeWithPrecision( conf.getTimePartitionInterval(), conf.getTimestampPrecision())); } + private void loadSync(Properties properties) { + conf.setEnableOperationSync( + Boolean.parseBoolean( + properties.getProperty( + "enable_operation_sync", String.valueOf(conf.isEnableOperationSync())))); + + conf.setSecondaryAddress( + properties.getProperty("secondary_address", conf.getSecondaryAddress())); + + conf.setSecondaryPort( + Integer.parseInt( + properties.getProperty("secondary_port", String.valueOf(conf.getSecondaryPort())))); + + conf.setSecondaryUser(properties.getProperty("secondary_user", conf.getSecondaryUser())); + + conf.setSecondaryPassword( + properties.getProperty("secondary_password", conf.getSecondaryPassword())); + + conf.setOperationSyncLogDir( + properties.getProperty("operation_sync_log_dir", conf.getOperationSyncLogDir())); + + conf.setOperationSyncLogValiditySecond( + Integer.parseInt( + properties.getProperty( + "operation_sync_log_file_validity_second", + String.valueOf(conf.getOperationSyncLogValiditySecond())))); + + conf.setOperationSyncLogNum( + Integer.parseInt( + properties.getProperty( + "operation_sync_log_file_num", String.valueOf(conf.getOperationSyncLogNum())))); + + conf.setOperationSyncMaxLogSize( + Long.parseLong( + properties.getProperty( + "operation_sync_max_log_size", String.valueOf(conf.getOperationSyncMaxLogSize())))); + + conf.setOperationSyncProducerCacheSize( + Integer.parseInt( + properties.getProperty( + "operation_sync_producer_cache_size", + String.valueOf(conf.getOperationSyncProducerCacheSize())))); + + conf.setOperationSyncProducerCacheNum( + Integer.parseInt( + properties.getProperty( + "operation_sync_producer_cache_num", + String.valueOf(conf.getOperationSyncProducerCacheNum())))); + } + private void loadCeaProps(Properties properties) { conf.setCeaEnable( Boolean.parseBoolean( diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncConsumer.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncConsumer.java new file mode 100644 index 0000000000000..5ee04d3b2c28e --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncConsumer.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.rpc.BatchExecutionException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.BlockingQueue; + +import static org.apache.iotdb.rpc.TSStatusCode.DATABASE_ALREADY_EXISTS; +import static org.apache.iotdb.rpc.TSStatusCode.DATABASE_NOT_READY; + +public class OperationSyncConsumer implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncConsumer.class); + + private final BlockingQueue> + operationSyncQueue; + private final SessionPool operationSyncSessionPool; + private final OperationSyncLogService dmlLogService; + + public OperationSyncConsumer( + BlockingQueue> + operationSyncQueue, + SessionPool operationSyncSessionPool, + OperationSyncLogService dmlLogService) { + this.operationSyncQueue = operationSyncQueue; + this.operationSyncSessionPool = operationSyncSessionPool; + this.dmlLogService = dmlLogService; + } + + @Override + public void run() { + while (true) { + Pair head; + ByteBuffer headBuffer; + try { + head = operationSyncQueue.take(); + headBuffer = head.left; + } catch (InterruptedException e) { + LOGGER.error("OperationSyncConsumer been interrupted: ", e); + continue; + } + boolean transmitStatus = false; + if (StorageEngineV2.isSecondaryAlive().get()) { + try { + headBuffer.position(0); + transmitStatus = operationSyncSessionPool.operationSyncTransmit(headBuffer); + } catch (IoTDBConnectionException connectionException) { + // warn IoTDBConnectionException and do serialization + LOGGER.warn( + "OperationSyncConsumer can't transmit for connection error", connectionException); + } catch (BatchExecutionException batchExecutionException) { + if (batchExecutionException.getStatusList().stream() + .anyMatch(s -> s.getCode() == DATABASE_NOT_READY.getStatusCode())) { + LOGGER.warn( + "OperationSyncConsumer can't transmit for STORAGE_GROUP_NOT_READY", + batchExecutionException); + } else { + LOGGER.warn( + "OperationSyncConsumer can't transmit for batchExecutionException, discard it", + batchExecutionException); + continue; + } + } catch (StatementExecutionException statementExecutionException) { + if (statementExecutionException.getStatusCode() == DATABASE_NOT_READY.getStatusCode()) { + LOGGER.warn( + "OperationSyncConsumer can't transmit for STORAGE_GROUP_NOT_READY", + statementExecutionException); + } else if (statementExecutionException.getStatusCode() + == DATABASE_ALREADY_EXISTS.getStatusCode()) { + LOGGER.warn(statementExecutionException.getMessage(), statementExecutionException); + } else { + LOGGER.warn( + "OperationSyncConsumer can't transmit for statementExecutionException, discard it", + statementExecutionException); + continue; + } + } catch (Exception e) { + // The statement has internal error, reject transmit + LOGGER.error("OperationSyncConsumer can't transmit, discard it", e); + continue; + } + } + if (!transmitStatus) { + try { + // must set buffer position to limit() before serialization + headBuffer.position(headBuffer.limit()); + dmlLogService.acquireLogWriter(); + dmlLogService.write(headBuffer); + } catch (IOException e) { + LOGGER.error("OperationSyncConsumer can't serialize statement", e); + } finally { + dmlLogService.releaseLogWriter(); + } + } + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDDLProtector.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDDLProtector.java new file mode 100644 index 0000000000000..c01d5d5d7916d --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDDLProtector.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.rpc.BatchExecutionException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.rpc.TSStatusCode.DATABASE_ALREADY_EXISTS; +import static org.apache.iotdb.rpc.TSStatusCode.DATABASE_NOT_READY; + +public class OperationSyncDDLProtector extends OperationSyncProtector { + + private static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncDDLProtector.class);; + + private final SessionPool operationSyncSessionPool; + + public OperationSyncDDLProtector(SessionPool operationSyncSessionPool) { + super(); + this.operationSyncSessionPool = operationSyncSessionPool; + } + + @Override + protected void preCheck() { + // do nothing + } + + @Override + protected void transmitPhysicalPlan(ByteBuffer planBuffer) { + long sleepTimeInSeconds = 1; + while (true) { + // transmit E-Plan until it's been received + boolean transmitStatus = false; + if (StorageEngineV2.isSecondaryAlive().get()) { + try { + // try operation sync + planBuffer.position(0); + transmitStatus = operationSyncSessionPool.operationSyncTransmit(planBuffer); + } catch (IoTDBConnectionException connectionException) { + // warn IoTDBConnectionException and retry + LOGGER.warn( + "OperationSyncDDLProtector can't transmit for connection error, retrying...", + connectionException); + } catch (BatchExecutionException batchExecutionException) { + if (batchExecutionException.getStatusList().stream() + .anyMatch(s -> s.getCode() == DATABASE_NOT_READY.getStatusCode())) { + LOGGER.warn( + "OperationSyncDDLProtector can't transmit for STORAGE_GROUP_NOT_READY", + batchExecutionException); + sleepTimeInSeconds = 10; + } else { + LOGGER.warn( + "OperationSyncDDLProtector can't transmit for batchExecutionException, discard it", + batchExecutionException); + break; + } + } catch (StatementExecutionException statementExecutionException) { + if (statementExecutionException.getStatusCode() + == DATABASE_ALREADY_EXISTS.getStatusCode()) { + sleepTimeInSeconds = 10; + LOGGER.warn( + "OperationSyncDDLProtector can't transmit for STORAGE_GROUP_NOT_READY", + statementExecutionException); + } else { + LOGGER.warn( + "OperationSyncDDLProtector can't transmit for statementExecutionException, discard it", + statementExecutionException); + break; + } + } catch (Exception e) { + // error exception and break + LOGGER.error("OperationSyncDDLProtector can't transmit, discard it", e); + break; + } + } else { + try { + TimeUnit.SECONDS.sleep(30); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + if (transmitStatus) { + break; + } else { + try { + TimeUnit.SECONDS.sleep(sleepTimeInSeconds); + } catch (InterruptedException e) { + LOGGER.warn("OperationSyncDDLProtector is interrupted", e); + } + } + } + } + + public boolean isAtWork() { + return isProtectorAtWork; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDMLProtector.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDMLProtector.java new file mode 100644 index 0000000000000..bb23b5bb4cd62 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncDMLProtector.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.rpc.BatchExecutionException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.rpc.TSStatusCode.DATABASE_NOT_READY; + +public class OperationSyncDMLProtector extends OperationSyncProtector { + + private final OperationSyncDDLProtector ddlProtector; + private final SessionPool operationSyncSessionPool; + + public OperationSyncDMLProtector( + OperationSyncDDLProtector ddlProtector, SessionPool operationSyncSessionPool) { + super(); + this.ddlProtector = ddlProtector; + this.operationSyncSessionPool = operationSyncSessionPool; + } + + @Override + protected void preCheck() { + while (ddlProtector.isAtWork()) { + try { + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException ignore) { + // ignore and retry + } + } + } + + @Override + protected void transmitPhysicalPlan(ByteBuffer planBuffer) { + long sleepTimeInSeconds = 1; + while (true) { + // transmit E-Plan until it's been received + boolean transmitStatus = false; + if (StorageEngineV2.isSecondaryAlive().get()) { + try { + // try operation sync + planBuffer.position(0); + transmitStatus = operationSyncSessionPool.operationSyncTransmit(planBuffer); + } catch (IoTDBConnectionException connectionException) { + // warn IoTDBConnectionException and retry + LOGGER.warn( + "OperationSyncDMLProtector can't transmit for connection error, retrying...", + connectionException); + } catch (BatchExecutionException batchExecutionException) { + if (batchExecutionException.getStatusList().stream() + .anyMatch(s -> s.getCode() == DATABASE_NOT_READY.getStatusCode())) { + sleepTimeInSeconds = 10; + LOGGER.warn( + "OperationSyncDMLProtector can't transmit for STORAGE_GROUP_NOT_READY", + batchExecutionException); + } else { + LOGGER.warn( + "OperationSyncDMLProtector can't transmit for batchExecutionException, discard it", + batchExecutionException); + break; + } + } catch (StatementExecutionException statementExecutionException) { + if (statementExecutionException.getStatusCode() == DATABASE_NOT_READY.getStatusCode()) { + LOGGER.warn( + "OperationSyncDMLProtector can't transmit for STORAGE_GROUP_NOT_READY", + statementExecutionException); + sleepTimeInSeconds = 10; + } else { + LOGGER.warn( + "OperationSyncDMLProtector can't transmit for statementExecutionException, discard it", + statementExecutionException); + break; + } + } catch (Exception e) { + // error exception and break + LOGGER.error("OperationSyncDMLProtector can't transmit, discard it", e); + break; + } + } else { + try { + TimeUnit.SECONDS.sleep(30); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + if (transmitStatus) { + break; + } else { + try { + TimeUnit.SECONDS.sleep(sleepTimeInSeconds); + } catch (InterruptedException e) { + LOGGER.warn("OperationSyncDMLProtector is interrupted", e); + } + } + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncLogService.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncLogService.java new file mode 100644 index 0000000000000..a63398fbebf81 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncLogService.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.utils.writelog.LogWriter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public class OperationSyncLogService implements Runnable { + + private static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncLogService.class); + + private static final String LOG_FILE_DIR = + IoTDBDescriptor.getInstance().getConfig().getOperationSyncLogDir(); + private static final long LOG_FILE_VALIDITY = + IoTDBDescriptor.getInstance().getConfig().getOperationSyncLogValiditySecond() * 1000L; + private static final int MAX_LOG_FILE_NUM = + IoTDBDescriptor.getInstance().getConfig().getOperationSyncLogNum(); + private static final long MAX_LOG_FILE_SIZE = + IoTDBDescriptor.getInstance().getConfig().getOperationSyncMaxLogSize(); + + private static long currentLogFileSize = 0; + + private final OperationSyncProtector protector; + private final Lock logWriterLock; + private final String logFileName; + private int logFileID; + private long logFileCreateTime; + private File logFile; + private LogWriter logWriter; + + public OperationSyncLogService(String logFileName, OperationSyncProtector protector) { + this.logFileName = logFileName; + this.protector = protector; + + this.logWriterLock = new ReentrantLock(); + this.logFile = null; + this.logWriter = null; + + File logDir = new File(LOG_FILE_DIR); + if (!logDir.exists()) { + if (!logDir.mkdirs()) { + LOGGER.error("Can't make OperationSyncLog file dir: {}", logDir.getAbsolutePath()); + } + } + } + + @Override + public void run() { + // Check if there exists remnant logs + List logFileIDList = new ArrayList<>(); + for (int ID = 0; ID < MAX_LOG_FILE_NUM; ID++) { + File file = + SystemFileFactory.INSTANCE.getFile(LOG_FILE_DIR + File.separator + logFileName + ID); + if (file.exists()) { + logFileIDList.add(ID); + } + } + + int firstID = 0; + if (logFileIDList.size() > 0) { + // Re-transmit the remnant logs + for (int i = 0; i < logFileIDList.size() - 1; i++) { + if (logFileIDList.get(i + 1) - logFileIDList.get(i) > 1) { + firstID = i + 1; + break; + } + } + + for (int i = firstID; i < logFileIDList.size(); i++) { + protector.registerLogFile( + LOG_FILE_DIR + File.separator + logFileName + logFileIDList.get(i)); + } + for (int i = 0; i < firstID; i++) { + protector.registerLogFile( + LOG_FILE_DIR + File.separator + logFileName + logFileIDList.get(i)); + } + + int nextID; + if (firstID == 0) { + nextID = logFileIDList.get(logFileIDList.size() - 1) + 1; + } else { + nextID = logFileIDList.get(firstID - 1) + 1; + } + logFileID = nextID % MAX_LOG_FILE_NUM; + } else { + logFileID = 0; + } + + while (true) { + // Check the validity of logFile + logWriterLock.lock(); + try { + if (logWriter != null + && System.currentTimeMillis() - logFileCreateTime > LOG_FILE_VALIDITY) { + // Submit logFile when it's expired + submitLogFile(); + } + } finally { + logWriterLock.unlock(); + } + + try { + // Sleep 10s before next check + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + LOGGER.error("OperationSyncLogService been interrupted", e); + } + } + } + + private void submitLogFile() { + try { + logWriter.force(); + } catch (IOException e) { + LOGGER.error("Can't force logWrite", e); + } + incLogFileSize(logFile.length()); + + for (int retry = 0; retry < 5; retry++) { + try { + logWriter.close(); + } catch (IOException e) { + LOGGER.warn("Can't close OperationSyncLog: {}, retrying...", logFile.getAbsolutePath()); + try { + // Sleep 1s and retry + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException ignored) { + // Ignore and retry + } + continue; + } + + LOGGER.info("OperationSyncLog: {} is expired and closed", logFile.getAbsolutePath()); + break; + } + + protector.registerLogFile( + LOG_FILE_DIR + + File.separator + + logFileName + + (logFileID - 1 + MAX_LOG_FILE_NUM) % MAX_LOG_FILE_NUM); + + logWriter = null; + logFile = null; + } + + private void createLogFile() { + logFile = + SystemFileFactory.INSTANCE.getFile(LOG_FILE_DIR + File.separator + logFileName + logFileID); + while (true) { + try { + if (logFile.createNewFile()) { + logFileCreateTime = System.currentTimeMillis(); + logWriter = new LogWriter(logFile, false); + LOGGER.info("Create OperationSyncLog: {}", logFile.getAbsolutePath()); + break; + } + } catch (IOException e) { + LOGGER.warn("Can't create OperationSyncLog: {}, retrying...", logFile.getAbsolutePath()); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException ignored) { + // Ignore and retry + } + } + } + logFileID = (logFileID + 1) % MAX_LOG_FILE_NUM; + } + + public static synchronized void incLogFileSize(long size) { + currentLogFileSize = currentLogFileSize + size; + } + + public void acquireLogWriter() { + logWriterLock.lock(); + } + + public void write(ByteBuffer buffer) throws IOException { + if (currentLogFileSize < MAX_LOG_FILE_SIZE) { + if (logWriter == null) { + // Create logFile when there are no valid + createLogFile(); + } + logWriter.write(buffer); + } else { + LOGGER.error("The OperationSyncLog is full, new Statements will be discarded."); + } + } + + public void releaseLogWriter() { + logWriterLock.unlock(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncPlanTypeUtils.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncPlanTypeUtils.java new file mode 100644 index 0000000000000..d491973201d07 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncPlanTypeUtils.java @@ -0,0 +1,70 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * */ + +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.db.mpp.plan.statement.Statement; +import org.apache.iotdb.db.mpp.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.AlterTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateMultiTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ActivateTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.CreateSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.DeactivateTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.DropSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.SetSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.UnsetSchemaTemplateStatement; + +public class OperationSyncPlanTypeUtils { + + public static OperationSyncPlanType getOperationSyncPlanType(Statement statement) { + if (statement instanceof InsertStatement + || statement instanceof InsertBaseStatement + || statement instanceof DeleteDataStatement) { + return OperationSyncPlanType.DMLPlan; + } else if (statement instanceof ActivateTemplateStatement + || statement instanceof CreateSchemaTemplateStatement + || statement instanceof DeactivateTemplateStatement + || statement instanceof DropSchemaTemplateStatement + || statement instanceof SetSchemaTemplateStatement + || statement instanceof UnsetSchemaTemplateStatement + || statement instanceof AlterTimeSeriesStatement + || statement instanceof CreateAlignedTimeSeriesStatement + || statement instanceof CreateMultiTimeSeriesStatement + || statement instanceof CreateTimeSeriesStatement + || statement instanceof DeleteStorageGroupStatement + || statement instanceof DeleteTimeSeriesStatement + || statement instanceof SetStorageGroupStatement + || statement instanceof SetTTLStatement) { + return OperationSyncPlanType.DDLPlan; + } + return null; + } + + public enum OperationSyncPlanType { + DDLPlan, // Create, update and delete schema + DMLPlan // insert and delete data + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProducer.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProducer.java new file mode 100644 index 0000000000000..cef956cc00702 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProducer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.tsfile.utils.Pair; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.concurrent.BlockingQueue; + +/** + * OperationSyncProducer using BlockingQueue to cache Statement. And persist some Statement when + * they are too many to transmit + */ +public class OperationSyncProducer { + + private static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncProducer.class); + private final ArrayList< + BlockingQueue>> + operationSyncQueues; + private final OperationSyncLogService dmlLogService; + + public OperationSyncProducer( + ArrayList>> + operationSyncQueue, + OperationSyncLogService operationSyncDMLLogService) { + this.operationSyncQueues = operationSyncQueue; + this.dmlLogService = operationSyncDMLLogService; + } + + public void put( + Pair planPair, + String deviceName) { + + ByteBuffer headBuffer; + headBuffer = planPair.left; + headBuffer.position(0); + int index; + if (deviceName == null) { + index = operationSyncQueues.size() - 1; + } else { + try { + index = Math.abs(deviceName.hashCode()) % (operationSyncQueues.size() - 1); + } catch (Exception e) { + index = 0; + } + } + if (operationSyncQueues.get(index).offer(planPair)) { + return; + } + try { + // must set buffer position to limit() before serialization + headBuffer.position(headBuffer.limit()); + dmlLogService.acquireLogWriter(); + dmlLogService.write(headBuffer); + } catch (IOException e) { + LOGGER.error("OperationSyncConsumer can't serialize Statement", e); + } finally { + dmlLogService.releaseLogWriter(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProtector.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProtector.java new file mode 100644 index 0000000000000..2cc9c631b6364 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncProtector.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.mpp.plan.statement.Statement; +import org.apache.iotdb.db.utils.writelog.SingleFileLogSReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public abstract class OperationSyncProtector implements Runnable { + + protected static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncProtector.class); + protected static final int LOG_FILE_VALIDITY = + IoTDBDescriptor.getInstance().getConfig().getOperationSyncLogValiditySecond(); + + // For transmit log files + protected final Lock logFileListLock; + protected List registeredLogFiles; + protected List processingLogFiles; + + // For serialize PhysicalPlan + private static final int MAX_PHYSICALPLAN_SIZE = 16 * 1024 * 1024; + protected final ByteArrayOutputStream protectorByteStream; + protected final DataOutputStream protectorDeserializeStream; + + // Working state + protected volatile boolean isProtectorAtWork; + + protected OperationSyncProtector() { + logFileListLock = new ReentrantLock(); + registeredLogFiles = new ArrayList<>(); + + protectorByteStream = new ByteArrayOutputStream(MAX_PHYSICALPLAN_SIZE); + protectorDeserializeStream = new DataOutputStream(protectorByteStream); + + isProtectorAtWork = false; + } + + protected void registerLogFile(String logFile) { + logFileListLock.lock(); + try { + registeredLogFiles.add(logFile); + } finally { + logFileListLock.unlock(); + } + } + + protected void wrapLogFiles() { + processingLogFiles = new ArrayList<>(registeredLogFiles); + registeredLogFiles = new ArrayList<>(); + } + + @Override + public void run() { + while (true) { + while (true) { + // Wrap and transmit all OperationSyncLogs + logFileListLock.lock(); + try { + if (registeredLogFiles.size() > 0) { + isProtectorAtWork = true; + wrapLogFiles(); + } else { + isProtectorAtWork = false; + break; + } + } finally { + logFileListLock.unlock(); + } + if (isProtectorAtWork) { + transmitLogFiles(); + } + } + + try { + // Sleep a while before next check + TimeUnit.SECONDS.sleep(LOG_FILE_VALIDITY); + } catch (InterruptedException e) { + LOGGER.warn("OperationSyncProtector been interrupted", e); + } + } + } + + protected void transmitLogFiles() { + preCheck(); + for (String logFileName : processingLogFiles) { + File logFile = SystemFileFactory.INSTANCE.getFile(logFileName); + SingleFileLogSReader logReader; + try { + logReader = new SingleFileLogSReader(logFile); + } catch (FileNotFoundException e) { + LOGGER.error( + "OperationSyncProtector can't open OperationSyncLog: {}, discarded", + logFile.getAbsolutePath(), + e); + continue; + } + LOGGER.info("begin trans {}", logFileName); + while (logReader.hasNext()) { + // read and re-serialize the Statement + Statement s = logReader.next(); + try { + s.serialize(protectorDeserializeStream); + } catch (IOException e) { + LOGGER.error("OperationSyncProtector can't serialize Statement", e); + continue; + } + ByteBuffer nextBuffer = ByteBuffer.wrap(protectorByteStream.toByteArray()); + protectorByteStream.reset(); + transmitPhysicalPlan(nextBuffer); + } + + LOGGER.info("end trans {}", logFileName); + logReader.close(); + try { + // sleep one second then delete OperationSyncLog + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + LOGGER.warn("OperationSyncProtector is interrupted", e); + } + + OperationSyncLogService.incLogFileSize(-logFile.length()); + + boolean deleted = false; + for (int retryCnt = 0; retryCnt < 5; retryCnt++) { + if (logFile.delete()) { + deleted = true; + LOGGER.info("OperationSyncLog: {} is deleted.", logFile.getAbsolutePath()); + break; + } else { + LOGGER.warn("Delete OperationSyncLog: {} failed. Retrying", logFile.getAbsolutePath()); + } + } + if (!deleted) { + OperationSyncLogService.incLogFileSize(logFile.length()); + LOGGER.error("Couldn't delete OperationSyncLog: {}", logFile.getAbsolutePath()); + } + } + } + + protected abstract void preCheck(); + + protected abstract void transmitPhysicalPlan(ByteBuffer planBuffer); +} diff --git a/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncWriteTask.java b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncWriteTask.java new file mode 100644 index 0000000000000..f72b39b59b1da --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/doublelive/OperationSyncWriteTask.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.session.pool.SessionPool; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** OperationSyncWriteTask is used for transmit one E-Plan sending by a client */ +public class OperationSyncWriteTask implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(OperationSyncWriteTask.class); + + private final ByteBuffer StatementBuffer; + private final SessionPool operationSyncSessionPool; + private final OperationSyncDDLProtector ddlProtector; + private final OperationSyncLogService ddlLogService; + + public OperationSyncWriteTask( + ByteBuffer StatementBuffer, + SessionPool operationSyncSessionPool, + OperationSyncDDLProtector ddlProtector, + OperationSyncLogService ddlLogService) { + this.StatementBuffer = StatementBuffer; + this.operationSyncSessionPool = operationSyncSessionPool; + this.ddlProtector = ddlProtector; + this.ddlLogService = ddlLogService; + } + + @Override + public void run() { + if (ddlProtector.isAtWork()) { + serializeEPlan(); + } else { + boolean transmitStatus = false; + try { + StatementBuffer.position(0); + transmitStatus = operationSyncSessionPool.operationSyncTransmit(StatementBuffer); + } catch (IoTDBConnectionException connectionException) { + // warn IoTDBConnectionException and do serialization + LOGGER.warn( + "OperationSyncWriteTask can't transmit because network failure", connectionException); + } catch (Exception e) { + // The Statement has internal error, reject transmit + LOGGER.error("OperationSyncWriteTask can't transmit", e); + return; + } + if (!transmitStatus) { + serializeEPlan(); + } + } + } + + private void serializeEPlan() { + // serialize the E-Plan if necessary + try { + // must set buffer position to limit() before serialization + StatementBuffer.position(StatementBuffer.limit()); + ddlLogService.acquireLogWriter(); + ddlLogService.write(StatementBuffer); + } catch (IOException e) { + LOGGER.error("can't serialize current Statement", e); + } finally { + ddlLogService.releaseLogWriter(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index 498851bd3de41..d6e787c4b2dc8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; @@ -31,11 +32,21 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.service.IService; import org.apache.iotdb.commons.service.ServiceType; +import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.statemachine.visitor.DataExecutionVisitor; +import org.apache.iotdb.db.doublelive.OperationSyncConsumer; +import org.apache.iotdb.db.doublelive.OperationSyncDDLProtector; +import org.apache.iotdb.db.doublelive.OperationSyncDMLProtector; +import org.apache.iotdb.db.doublelive.OperationSyncLogService; +import org.apache.iotdb.db.doublelive.OperationSyncPlanTypeUtils; +import org.apache.iotdb.db.doublelive.OperationSyncProducer; +import org.apache.iotdb.db.doublelive.OperationSyncWriteTask; import org.apache.iotdb.db.engine.flush.CloseFileListener; import org.apache.iotdb.db.engine.flush.FlushListener; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; @@ -52,6 +63,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; +import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.db.sync.SyncService; import org.apache.iotdb.db.utils.ThreadUtils; @@ -59,10 +71,15 @@ import org.apache.iotdb.db.wal.WALManager; import org.apache.iotdb.db.wal.exception.WALException; import org.apache.iotdb.db.wal.recover.WALRecoverManager; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.session.pool.SessionPool; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.utils.FilePathUtils; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.commons.io.FileUtils; @@ -79,6 +96,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -86,6 +105,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -139,7 +159,106 @@ public class StorageEngineV2 implements IService { private LoadTsFileManager loadTsFileManager = new LoadTsFileManager(); - private StorageEngineV2() {} + private static final long HEARTBEAT_CHECK_INTERVAL = 30L; + private static final long SECONDARY_METRIC_INTERVAL = 30L; + + /* OperationSync module */ + private static final boolean isEnableOperationSync = + IoTDBDescriptor.getInstance().getConfig().isEnableOperationSync(); + private static SessionPool operationSyncSessionPool; + private static OperationSyncProducer operationSyncProducer; + private static OperationSyncDDLProtector operationSyncDDLProtector; + private static OperationSyncLogService operationSyncDDLLogService; + private static AtomicBoolean isSecondaryAlive = new AtomicBoolean(false); + + ArrayList>> + arrayListBlockQueue; + + private StorageEngineV2() { + if (isEnableOperationSync) { + // Open OperationSync + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + int cacheNum = config.getOperationSyncProducerCacheNum(); + + // create SessionPool for OperationSync + operationSyncSessionPool = + new SessionPool( + config.getSecondaryAddress(), + config.getSecondaryPort(), + config.getSecondaryUser(), + config.getSecondaryPassword(), + config.getSecondarySessionPoolMaxSize(), + config.isRpcThriftCompressionEnable()); + ThreadPoolExecutor threadPool = + new ThreadPoolExecutor( + cacheNum + 5, + cacheNum + 5, + 3, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(5), + new ThreadPoolExecutor.AbortPolicy()); + + // create operationSyncDDLProtector and operationSyncDDLLogService + ScheduledExecutorService secondaryCheckThread = + IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor("secondary-Check"); + + ScheduledExecutorUtil.safelyScheduleAtFixedRate( + secondaryCheckThread, + this::checkSecondaryIsLife, + 0L, + SECONDARY_METRIC_INTERVAL, + TimeUnit.SECONDS); + + operationSyncDDLProtector = new OperationSyncDDLProtector(operationSyncSessionPool); + threadPool.execute(operationSyncDDLProtector); + operationSyncDDLLogService = + new OperationSyncLogService("OperationSyncDDLLog", operationSyncDDLProtector); + threadPool.execute(operationSyncDDLLogService); + + // create OperationSyncProducer + arrayListBlockQueue = new ArrayList<>(cacheNum); + for (int i = 0; i < cacheNum; i++) { + BlockingQueue> + blockingQueue = new ArrayBlockingQueue<>(config.getOperationSyncProducerCacheSize()); + arrayListBlockQueue.add(blockingQueue); + } + OperationSyncDMLProtector operationSyncDMLProtector = + new OperationSyncDMLProtector(operationSyncDDLProtector, operationSyncSessionPool); + threadPool.execute(operationSyncDMLProtector); + OperationSyncLogService operationSyncDMLLogService = + new OperationSyncLogService("OperationSyncDMLLog", operationSyncDMLProtector); + operationSyncProducer = + new OperationSyncProducer(arrayListBlockQueue, operationSyncDMLLogService); + // create OperationSyncDMLProtector and OperationSyncDMLLogService + threadPool.execute(operationSyncDMLLogService); + // create OperationSyncConsumer + for (int i = 0; i < cacheNum; i++) { + threadPool.execute( + new OperationSyncConsumer( + arrayListBlockQueue.get(i), operationSyncSessionPool, operationSyncDMLLogService)); + } + if (!MetricConfigDescriptor.getInstance() + .getMetricConfig() + .getMetricReporterList() + .isEmpty()) { + ScheduledExecutorService secondaryMetricThread = + IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor("secondary-Metric"); + ScheduledExecutorUtil.safelyScheduleAtFixedRate( + secondaryMetricThread, + this::secondaryMetric, + 0L, + SECONDARY_METRIC_INTERVAL, + TimeUnit.SECONDS); + } + logger.info("Successfully initialize OperationSync!"); + } else { + operationSyncSessionPool = null; + operationSyncProducer = null; + operationSyncDDLProtector = null; + operationSyncDDLLogService = null; + arrayListBlockQueue = null; + } + } public static StorageEngineV2 getInstance() { return InstanceHolder.INSTANCE; @@ -835,6 +954,75 @@ public void getDiskSizeByDataRegion( }); } + public static AtomicBoolean isSecondaryAlive() { + return isSecondaryAlive; + } + + private void checkSecondaryIsLife() { + try { + isSecondaryAlive.set( + operationSyncSessionPool.getSystemStatus().equals(NodeStatus.Running.name())); + } catch (IoTDBConnectionException e) { + isSecondaryAlive.set(false); + } + } + + private void secondaryMetric() { + for (int i = 0; i < arrayListBlockQueue.size(); i++) { + MetricService.getInstance() + .getOrCreateGauge( + Metric.QUEUE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + "OperationSyncProducerQueueSize" + i, + Tag.STATUS.toString(), + "running") + .set(arrayListBlockQueue.get(i).size()); + } + } + + public static void transmitOperationSync(Statement statement) { + + OperationSyncPlanTypeUtils.OperationSyncPlanType planType = + OperationSyncPlanTypeUtils.getOperationSyncPlanType(statement); + if (planType == null) { + // Don't need OperationSync + return; + } + + // serialize statement + ByteBuffer buffer; + try { + buffer = statement.serializeToByteBuffer(); + } catch (IOException e) { + logger.error("OperationSync can't serialize PhysicalPlan", e); + return; + } + + switch (planType) { + case DDLPlan: + // Create OperationSyncWriteTask and wait + OperationSyncWriteTask ddlTask = + new OperationSyncWriteTask( + buffer, + operationSyncSessionPool, + operationSyncDDLProtector, + operationSyncDDLLogService); + ddlTask.run(); + break; + case DMLPlan: + // Put into OperationSyncProducer + operationSyncProducer.put(new Pair<>(buffer, planType), getDeviceNameByPlan(statement)); + } + } + + public static String getDeviceNameByPlan(Statement statement) { + if (!statement.getPaths().isEmpty() && statement.getPaths().size() > 0) { + return statement.getPaths().get(0).getDevice(); + } + return null; + } + static class InstanceHolder { private static final StorageEngineV2 INSTANCE = new StorageEngineV2(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/constant/StatementType.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/constant/StatementType.java index 2867c9729f5bd..597aee6847a6e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/constant/StatementType.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/constant/StatementType.java @@ -157,5 +157,6 @@ public enum StatementType { SHOW_SPACE_QUOTA, SET_THROTTLE_QUOTA, SHOW_THROTTLE_QUOTA, + INSERT_ROW, SHOW_SPACE_RESOURCE } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/Statement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/Statement.java index 6d25e8012e325..e4568b8ac44b6 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/Statement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/Statement.java @@ -19,10 +19,38 @@ package org.apache.iotdb.db.mpp.plan.statement; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.parser.ASTVisitor; +import org.apache.iotdb.db.mpp.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsOfOneDeviceStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.AlterTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateMultiTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ActivateTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.CreateSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.DeactivateTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.DropSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.SetSchemaTemplateStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.template.UnsetSchemaTemplateStatement; +import org.apache.iotdb.tsfile.utils.PublicBAOS; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import io.jsonwebtoken.io.IOException; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; import java.util.List; /** @@ -64,4 +92,94 @@ public boolean isAuthenticationRequired() { } public abstract List getPaths(); + + public ByteBuffer serializeToByteBuffer() throws java.io.IOException { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + serialize(outputStream); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + } + + public void serialize(DataOutputStream dataOutputStream) throws java.io.IOException {} + + public void deserialize(ByteBuffer buffer) throws IllegalPathException {} + + public static class Factory { + private Factory() {} + + public static Statement create(ByteBuffer buffer) throws IllegalPathException { + StatementType type = StatementType.values()[ReadWriteIOUtils.readInt(buffer)]; + Statement s; + switch (type) { + case CREATE_TIMESERIES: + s = new CreateTimeSeriesStatement(); + break; + case SET_STORAGE_GROUP: + s = new SetStorageGroupStatement(); + break; + case DELETE: + s = new DeleteDataStatement(); + break; + case MULTI_BATCH_INSERT: + s = new InsertMultiTabletsStatement(); + break; + case BATCH_INSERT_ONE_DEVICE: + s = new InsertRowsOfOneDeviceStatement(); + break; + case BATCH_INSERT_ROWS: + s = new InsertRowsStatement(); + break; + case INSERT_ROW: + s = new InsertRowStatement(); + break; + case INSERT: + s = new InsertStatement(); + break; + case BATCH_INSERT: + s = new InsertTabletStatement(); + break; + case ACTIVATE_TEMPLATE: + s = new ActivateTemplateStatement(); + break; + case CREATE_TEMPLATE: + s = new CreateSchemaTemplateStatement(); + break; + case DEACTIVATE_TEMPLATE: + s = new DeactivateTemplateStatement(); + break; + case DROP_TEMPLATE: + s = new DropSchemaTemplateStatement(); + break; + case SET_TEMPLATE: + s = new SetSchemaTemplateStatement(); + break; + case UNSET_TEMPLATE: + s = new UnsetSchemaTemplateStatement(); + break; + case ALTER_TIMESERIES: + s = new AlterTimeSeriesStatement(); + break; + case CREATE_ALIGNED_TIMESERIES: + s = new CreateAlignedTimeSeriesStatement(); + break; + case CREATE_MULTI_TIMESERIES: + s = new CreateMultiTimeSeriesStatement(); + break; + case DELETE_STORAGE_GROUP: + s = new DeleteStorageGroupStatement(); + break; + case DELETE_TIMESERIES: + s = new DeleteTimeSeriesStatement(); + break; + case TTL: + s = new SetTTLStatement(); + break; + default: + throw new IOException("unrecognized log type " + type); + } + s.deserialize(buffer); + return s; + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/DeleteDataStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/DeleteDataStatement.java index a7d7af40cd468..b323c3e9dcc93 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/DeleteDataStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/DeleteDataStatement.java @@ -19,12 +19,18 @@ package org.apache.iotdb.db.mpp.plan.statement.crud; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.tsfile.read.common.TimeRange; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; public class DeleteDataStatement extends Statement { @@ -76,4 +82,27 @@ public void setTimeRange(TimeRange timeRange) { public R accept(StatementVisitor visitor, C context) { return visitor.visitDeleteData(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(pathList.size(), dataOutputStream); + for (PartialPath partialPath : pathList) { + ReadWriteIOUtils.write(partialPath.getFullPath(), dataOutputStream); + } + ReadWriteIOUtils.write(deleteStartTime, dataOutputStream); + ReadWriteIOUtils.write(deleteEndTime, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + int size = ReadWriteIOUtils.readInt(buffer); + pathList = new ArrayList<>(); + while (size > 0) { + pathList.add(new PartialPath(ReadWriteIOUtils.readString(buffer))); + size--; + } + deleteStartTime = ReadWriteIOUtils.readLong(buffer); + deleteEndTime = ReadWriteIOUtils.readLong(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertBaseStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertBaseStatement.java index 1de24f70b310e..83bc501da1902 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertBaseStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertBaseStatement.java @@ -19,11 +19,15 @@ package org.apache.iotdb.db.mpp.plan.statement.crud; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -82,4 +86,14 @@ public List getPaths() { } public abstract List collectRedirectInfo(DataPartition dataPartition); + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + // empty body + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + // empty body + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertMultiTabletsStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertMultiTabletsStatement.java index 3134f06d039c8..9409c71775fb8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertMultiTabletsStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertMultiTabletsStatement.java @@ -21,13 +21,18 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -113,4 +118,26 @@ public List collectRedirectInfo(DataPartition dataPartition) { } return result; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(insertTabletStatementList.size(), dataOutputStream); + for (InsertTabletStatement insertTabletStatement : insertTabletStatementList) { + insertTabletStatement.serialize(dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + int size = ReadWriteIOUtils.readInt(buffer); + List insertTabletStatementList = new ArrayList<>(); + InsertTabletStatement insertTabletStatement; + for (int i = 0; i < size; i++) { + insertTabletStatement = new InsertTabletStatement(); + insertTabletStatement.deserialize(buffer); + insertTabletStatementList.add(insertTabletStatement); + } + this.insertTabletStatementList = insertTabletStatementList; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowStatement.java index a57a3ad1c7265..2369f6d970b01 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowStatement.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; @@ -29,8 +30,11 @@ import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -47,7 +51,7 @@ public class InsertRowStatement extends InsertBaseStatement { public InsertRowStatement() { super(); - statementType = StatementType.INSERT; + statementType = StatementType.INSERT_ROW; } @Override @@ -143,4 +147,82 @@ public List collectRedirectInfo(DataPartition dataPartition) { public R accept(StatementVisitor visitor, C context) { return visitor.visitInsertRow(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(devicePath.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(isAligned, dataOutputStream); + // measurements + ReadWriteIOUtils.write(measurements, dataOutputStream); + // dataTypes + ReadWriteIOUtils.write(dataTypes, dataOutputStream); + ReadWriteIOUtils.write(time, dataOutputStream); + + serializeValues(dataOutputStream); + ReadWriteIOUtils.write(isNeedInferType, dataOutputStream); + } + + void serializeValues(DataOutputStream dataOutputStream) throws IOException { + dataOutputStream.writeInt(values.length); + putValues(dataOutputStream); + } + + private void putValues(DataOutputStream outputStream) throws IOException { + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + continue; + } + // types are not determined, the situation mainly occurs when the plan uses string values + // and is forwarded to other nodes + if (dataTypes == null || dataTypes[i] == null) { + ReadWriteIOUtils.write(TYPE_RAW_STRING, outputStream); + ReadWriteIOUtils.write(values[i].toString(), outputStream); + } else { + ReadWriteIOUtils.write(dataTypes[i], outputStream); + switch (dataTypes[i]) { + case BOOLEAN: + ReadWriteIOUtils.write((Boolean) values[i], outputStream); + break; + case INT32: + ReadWriteIOUtils.write((Integer) values[i], outputStream); + break; + case INT64: + ReadWriteIOUtils.write((Long) values[i], outputStream); + break; + case FLOAT: + ReadWriteIOUtils.write((Float) values[i], outputStream); + break; + case DOUBLE: + ReadWriteIOUtils.write((Double) values[i], outputStream); + break; + case TEXT: + ReadWriteIOUtils.write((Binary) values[i], outputStream); + break; + default: + throw new IOException("Unsupported data type:" + dataTypes[i]); + } + } + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + devicePath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + isAligned = ReadWriteIOUtils.readBool(buffer); + measurements = ReadWriteIOUtils.readStrings(buffer); + dataTypes = ReadWriteIOUtils.readTsDataTypes(buffer); + time = ReadWriteIOUtils.readLong(buffer); + deserializeValues(buffer); + isNeedInferType = ReadWriteIOUtils.readBool(buffer); + } + + void deserializeValues(ByteBuffer buffer) { + this.values = new Object[dataTypes.length]; + try { + fillValues(buffer); + } catch (QueryProcessException e) { + e.printStackTrace(); + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index 7b23a107af338..0e83dec98750b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -21,12 +21,17 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.TimePartitionUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -103,4 +108,21 @@ public List getPaths() { } return ret; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(devicePath.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(isAligned, dataOutputStream); + ReadWriteIOUtils.write(measurements, dataOutputStream); + ReadWriteIOUtils.write(dataTypes, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + devicePath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + isAligned = ReadWriteIOUtils.readBool(buffer); + measurements = ReadWriteIOUtils.readStrings(buffer); + dataTypes = ReadWriteIOUtils.readTsDataTypes(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsStatement.java index b24814d8a9bb7..9765ce26854e2 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsStatement.java @@ -21,13 +21,18 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -112,4 +117,26 @@ public List collectRedirectInfo(DataPartition dataPartition) { } return result; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(insertRowStatementList.size(), dataOutputStream); + for (InsertRowStatement insertRowStatement : insertRowStatementList) { + insertRowStatement.serialize(dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + int size = ReadWriteIOUtils.readInt(buffer); + List insertRowStatementList = new ArrayList<>(); + InsertRowStatement insertRowStatement; + for (int i = 0; i < size; i++) { + insertRowStatement = new InsertRowStatement(); + insertRowStatement.deserialize(buffer); + insertRowStatementList.add(insertRowStatement); + } + this.insertRowStatementList = insertRowStatementList; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertStatement.java index f8a11aa278ca3..5a18a74e0d051 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertStatement.java @@ -19,12 +19,17 @@ package org.apache.iotdb.db.mpp.plan.statement.crud; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -126,4 +131,23 @@ public void semanticCheck() { } } } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(device.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(times, dataOutputStream); + ReadWriteIOUtils.write(measurementList, dataOutputStream); + ReadWriteIOUtils.write(valuesList, dataOutputStream); + ReadWriteIOUtils.write(isAligned, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + device = new PartialPath(ReadWriteIOUtils.readString(buffer)); + times = ReadWriteIOUtils.readLongs(buffer); + measurementList = ReadWriteIOUtils.readStrings(buffer); + valuesList = ReadWriteIOUtils.readStringsList(buffer); + isAligned = ReadWriteIOUtils.readBool(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertTabletStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertTabletStatement.java index 3e3767e481e08..84fd0bbd705c2 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertTabletStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertTabletStatement.java @@ -21,13 +21,20 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.TimePartitionUtils; import org.apache.iotdb.tsfile.utils.BitMap; +import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -130,4 +137,56 @@ public List getPaths() { } return ret; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(devicePath.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(isAligned, dataOutputStream); + // measurements + ReadWriteIOUtils.write(measurements, dataOutputStream); + // dataTypes + ReadWriteIOUtils.write(dataTypes, dataOutputStream); + // rowcount + ReadWriteIOUtils.write(rowCount, dataOutputStream); + // times + ReadWriteIOUtils.write(times, dataOutputStream); + // bitMaps + write(bitMaps, dataOutputStream); + // values + ReadWriteIOUtils.serializeValues(dataTypes, columns, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + devicePath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + isAligned = ReadWriteIOUtils.readBool(buffer); + measurements = ReadWriteIOUtils.readStrings(buffer); + dataTypes = ReadWriteIOUtils.readTsDataTypes(buffer); + rowCount = ReadWriteIOUtils.readInt(buffer); + times = ReadWriteIOUtils.readLongs(buffer); + boolean hasBitMaps = BytesUtils.byteToBool(buffer.get()); + if (hasBitMaps) { + bitMaps = QueryDataSetUtils.readBitMapsFromBuffer(buffer, dataTypes.length, rowCount); + } + columns = QueryDataSetUtils.readValuesFromBuffer(buffer, dataTypes, dataTypes.length, rowCount); + } + + public void write(BitMap[] bitMaps, DataOutputStream dataOutputStream) throws IOException { + dataOutputStream.writeBoolean(bitMaps != null); + if (bitMaps != null) { + for (int i = 0; i < bitMaps.length; ++i) { + if (columns[i] == null) { + continue; + } + + if (bitMaps[i] == null) { + dataOutputStream.writeBoolean(false); + } else { + dataOutputStream.writeBoolean(true); + dataOutputStream.write(bitMaps[i].getByteArray()); + } + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/AlterTimeSeriesStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/AlterTimeSeriesStatement.java index d6f4198cba4fc..f5544e0c11526 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/AlterTimeSeriesStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/AlterTimeSeriesStatement.java @@ -19,11 +19,16 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; @@ -124,4 +129,25 @@ public enum AlterType { ADD_ATTRIBUTES, UPSERT } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(alterType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(alterMap, dataOutputStream); + ReadWriteIOUtils.write(alias, dataOutputStream); + ReadWriteIOUtils.write(tagsMap, dataOutputStream); + ReadWriteIOUtils.write(attributesMap, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + path = new PartialPath(ReadWriteIOUtils.readString(buffer)); + alterType = AlterType.values()[ReadWriteIOUtils.readInt(buffer)]; + alterMap = ReadWriteIOUtils.readMap(buffer); + alias = ReadWriteIOUtils.readString(buffer); + tagsMap = ReadWriteIOUtils.readMap(buffer); + attributesMap = ReadWriteIOUtils.readMap(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateAlignedTimeSeriesStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateAlignedTimeSeriesStatement.java index a7b3495f64d90..c624b1b5c6e34 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateAlignedTimeSeriesStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateAlignedTimeSeriesStatement.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; @@ -26,7 +27,11 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -160,4 +165,33 @@ public void addAttributesList(Map attributes) { public R accept(StatementVisitor visitor, C context) { return visitor.visitCreateAlignedTimeseries(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(devicePath.getFullPath(), dataOutputStream); + ReadWriteIOUtils.writeStringList(measurements, dataOutputStream); + ReadWriteIOUtils.writeTSDataTypeList(dataTypes, dataOutputStream); + ReadWriteIOUtils.writeTSEncodingList(encodings, dataOutputStream); + ReadWriteIOUtils.writeCompressionTypeList(compressors, dataOutputStream); + ReadWriteIOUtils.writeStringList(aliasList, dataOutputStream); + ReadWriteIOUtils.write(tagsList.size(), dataOutputStream); + ReadWriteIOUtils.write(tagsList, dataOutputStream); + ReadWriteIOUtils.write(attributesList.size(), dataOutputStream); + ReadWriteIOUtils.write(attributesList, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + devicePath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + measurements = ReadWriteIOUtils.readStringList(buffer); + dataTypes = ReadWriteIOUtils.readTSDataTypeList(buffer); + encodings = ReadWriteIOUtils.readTSEncodingList(buffer); + compressors = ReadWriteIOUtils.readCompressionTypeList(buffer); + aliasList = ReadWriteIOUtils.readStringList(buffer); + int size = ReadWriteIOUtils.readInt(buffer); + tagsList = ReadWriteIOUtils.readMaps(buffer, size); + size = ReadWriteIOUtils.readInt(buffer); + attributesList = ReadWriteIOUtils.readMaps(buffer, size); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateMultiTimeSeriesStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateMultiTimeSeriesStatement.java index b0ad15c08f8f0..350cc1f77fe25 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateMultiTimeSeriesStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateMultiTimeSeriesStatement.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; @@ -26,7 +27,11 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -121,4 +126,61 @@ public void addAttributesList(Map attributes) { public R accept(StatementVisitor visitor, C context) { return visitor.visitCreateMultiTimeseries(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(paths.size(), dataOutputStream); + for (PartialPath path : paths) { + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + } + ReadWriteIOUtils.writeTSDataTypeList(dataTypes, dataOutputStream); + ReadWriteIOUtils.writeTSEncodingList(encodings, dataOutputStream); + ReadWriteIOUtils.writeCompressionTypeList(compressors, dataOutputStream); + ReadWriteIOUtils.write(propsList == null, dataOutputStream); + if (propsList != null) { + ReadWriteIOUtils.write(propsList.size(), dataOutputStream); + ReadWriteIOUtils.write(propsList, dataOutputStream); + } + ReadWriteIOUtils.writeList(aliasList, dataOutputStream); + ReadWriteIOUtils.write(tagsList == null, dataOutputStream); + if (propsList != null) { + ReadWriteIOUtils.write(tagsList.size(), dataOutputStream); + ReadWriteIOUtils.write(tagsList, dataOutputStream); + } + ReadWriteIOUtils.write(attributesList == null, dataOutputStream); + if (propsList != null) { + ReadWriteIOUtils.write(attributesList.size(), dataOutputStream); + ReadWriteIOUtils.write(attributesList, dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + // paths size + int size = ReadWriteIOUtils.readInt(buffer); + paths = new ArrayList<>(); + for (int i = 0; i < size; i++) { + paths.add(new PartialPath(ReadWriteIOUtils.readString(buffer))); + } + dataTypes = ReadWriteIOUtils.readTSDataTypeList(buffer); + encodings = ReadWriteIOUtils.readTSEncodingList(buffer); + compressors = ReadWriteIOUtils.readCompressionTypeList(buffer); + // propsList size + if (!ReadWriteIOUtils.readBool(buffer)) { + size = ReadWriteIOUtils.readInt(buffer); + propsList = ReadWriteIOUtils.readMaps(buffer, size); + } + aliasList = ReadWriteIOUtils.readStringList(buffer); + // tagList size + if (!ReadWriteIOUtils.readBool(buffer)) { + size = ReadWriteIOUtils.readInt(buffer); + tagsList = ReadWriteIOUtils.readMaps(buffer, size); + } + // attributesList size + if (!ReadWriteIOUtils.readBool(buffer)) { + size = ReadWriteIOUtils.readInt(buffer); + attributesList = ReadWriteIOUtils.readMaps(buffer, size); + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateTimeSeriesStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateTimeSeriesStatement.java index f10d7eddffdcb..0ef102721eaa7 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateTimeSeriesStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateTimeSeriesStatement.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; @@ -26,7 +27,11 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; @@ -129,4 +134,29 @@ public void setTags(Map tags) { public R accept(StatementVisitor visitor, C context) { return visitor.visitCreateTimeseries(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(alias, dataOutputStream); + ReadWriteIOUtils.write(dataType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(encoding.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(compressor.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(props, dataOutputStream); + ReadWriteIOUtils.write(attributes, dataOutputStream); + ReadWriteIOUtils.write(tags, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + path = new PartialPath(ReadWriteIOUtils.readString(buffer)); + alias = ReadWriteIOUtils.readString(buffer); + dataType = TSDataType.values()[ReadWriteIOUtils.readInt(buffer)]; + encoding = TSEncoding.values()[ReadWriteIOUtils.readInt(buffer)]; + compressor = CompressionType.values()[ReadWriteIOUtils.readInt(buffer)]; + props = ReadWriteIOUtils.readMap(buffer); + attributes = ReadWriteIOUtils.readMap(buffer); + tags = ReadWriteIOUtils.readMap(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteStorageGroupStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteStorageGroupStatement.java index 25ec719ac7b77..e1e8ebada953e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteStorageGroupStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteStorageGroupStatement.java @@ -26,10 +26,14 @@ import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -74,4 +78,15 @@ public void setPrefixPath(List prefixPathList) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.writeStringList(prefixPathList, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + prefixPathList = ReadWriteIOUtils.readStringList(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteTimeSeriesStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteTimeSeriesStatement.java index c21f590a248a0..3da56f4ca1765 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteTimeSeriesStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DeleteTimeSeriesStatement.java @@ -19,13 +19,19 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; public class DeleteTimeSeriesStatement extends Statement implements IConfigStatement { @@ -64,4 +70,22 @@ public R accept(StatementVisitor visitor, C context) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(pathPatternList.size(), dataOutputStream); + for (PartialPath partialPath : pathPatternList) { + ReadWriteIOUtils.write(partialPath.getFullPath(), dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + int size = ReadWriteIOUtils.readInt(buffer); + pathPatternList = new ArrayList<>(); + for (int i = 0; i < size; i++) { + pathPatternList.add(new PartialPath(ReadWriteIOUtils.readString(buffer))); + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DropContinuousQueryStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DropContinuousQueryStatement.java index 85bc2382e6841..8510b7e6c0b41 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DropContinuousQueryStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/DropContinuousQueryStatement.java @@ -31,7 +31,7 @@ public class DropContinuousQueryStatement extends Statement implements IConfigStatement { - private final String cqId; + private String cqId; public DropContinuousQueryStatement(String cqId) { super(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java index 799dd03c3a7b2..92b4089e1d968 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java @@ -19,13 +19,18 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -98,6 +103,53 @@ public Long getTimePartitionInterval() { return timePartitionInterval; } + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(storageGroupPath.getFullPath(), dataOutputStream); + if (ttl == null) { + ReadWriteIOUtils.write(false, dataOutputStream); + } else { + ReadWriteIOUtils.write(true, dataOutputStream); + ReadWriteIOUtils.write(ttl, dataOutputStream); + } + if (schemaReplicationFactor == null) { + ReadWriteIOUtils.write(false, dataOutputStream); + } else { + ReadWriteIOUtils.write(true, dataOutputStream); + ReadWriteIOUtils.write(schemaReplicationFactor, dataOutputStream); + } + if (dataReplicationFactor == null) { + ReadWriteIOUtils.write(false, dataOutputStream); + } else { + ReadWriteIOUtils.write(true, dataOutputStream); + ReadWriteIOUtils.write(dataReplicationFactor, dataOutputStream); + } + if (timePartitionInterval == null) { + ReadWriteIOUtils.write(false, dataOutputStream); + } else { + ReadWriteIOUtils.write(true, dataOutputStream); + ReadWriteIOUtils.write(timePartitionInterval, dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + storageGroupPath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + if (ReadWriteIOUtils.readBool(buffer)) { + ttl = ReadWriteIOUtils.readLong(buffer); + } + if (ReadWriteIOUtils.readBool(buffer)) { + schemaReplicationFactor = ReadWriteIOUtils.readInt(buffer); + } + if (ReadWriteIOUtils.readBool(buffer)) { + dataReplicationFactor = ReadWriteIOUtils.readInt(buffer); + } + if (ReadWriteIOUtils.readBool(buffer)) { + timePartitionInterval = ReadWriteIOUtils.readLong(buffer); + } + } + @Override public String toString() { return "SetStorageGroupStatement{" diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetTTLStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetTTLStatement.java index 549bc98e47ea0..0d955546ab790 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetTTLStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetTTLStatement.java @@ -19,13 +19,18 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -70,4 +75,17 @@ public List getPaths() { ? Collections.singletonList(storageGroupPath) : Collections.emptyList(); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(storageGroupPath.getFullPath(), dataOutputStream); + ReadWriteIOUtils.write(ttl, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + storageGroupPath = new PartialPath(ReadWriteIOUtils.readString(buffer)); + ttl = ReadWriteIOUtils.readLong(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/ActivateTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/ActivateTemplateStatement.java index 5c424b5d13c2a..cea89fb22b4f7 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/ActivateTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/ActivateTemplateStatement.java @@ -19,11 +19,16 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -59,4 +64,15 @@ public void setPath(PartialPath path) { public R accept(StatementVisitor visitor, C context) { return visitor.visitActivateTemplate(this, context); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + path = new PartialPath(ReadWriteIOUtils.readString(buffer)); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/CreateSchemaTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/CreateSchemaTemplateStatement.java index 047c028d3c762..b563e51b19df0 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/CreateSchemaTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/CreateSchemaTemplateStatement.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; @@ -28,7 +29,11 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -178,4 +183,25 @@ public R accept(StatementVisitor visitor, C context) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(name, dataOutputStream); + ReadWriteIOUtils.write(alignedDeviceId, dataOutputStream); + ReadWriteIOUtils.write(measurements, dataOutputStream); + ReadWriteIOUtils.write(dataTypes, dataOutputStream); + ReadWriteIOUtils.write(encodings, dataOutputStream); + ReadWriteIOUtils.write(compressors, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + name = ReadWriteIOUtils.readString(buffer); + alignedDeviceId = ReadWriteIOUtils.readStringSet(buffer); + measurements = ReadWriteIOUtils.readArrayOfString(buffer); + dataTypes = ReadWriteIOUtils.readArrayOfTSDataType(buffer); + encodings = ReadWriteIOUtils.readArrayOfTSEncoding(buffer); + compressors = ReadWriteIOUtils.readArrayOfCompressionType(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DeactivateTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DeactivateTemplateStatement.java index 8f54876f29806..1cd7b599706b9 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DeactivateTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DeactivateTemplateStatement.java @@ -19,13 +19,19 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; @@ -77,4 +83,25 @@ public R accept(StatementVisitor visitor, C context) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(templateName, dataOutputStream); + ReadWriteIOUtils.write(pathPatternList.size(), dataOutputStream); + for (PartialPath partialPath : pathPatternList) { + ReadWriteIOUtils.write(partialPath.getFullPath(), dataOutputStream); + } + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + templateName = ReadWriteIOUtils.readString(buffer); + int size = ReadWriteIOUtils.readInt(buffer); + List partialPaths = new ArrayList<>(); + for (int i = 0; i < size; i++) { + partialPaths.add(new PartialPath(ReadWriteIOUtils.readString(buffer))); + } + this.pathPatternList = partialPaths; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DropSchemaTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DropSchemaTemplateStatement.java index 51013c94d7904..6d2bf5fbd4a37 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DropSchemaTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/DropSchemaTemplateStatement.java @@ -19,11 +19,17 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStatement; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; public class DropSchemaTemplateStatement extends ShowStatement implements IConfigStatement { @@ -35,6 +41,11 @@ public DropSchemaTemplateStatement(String templateName) { this.statementType = StatementType.DROP_TEMPLATE; } + public DropSchemaTemplateStatement() { + super(); + this.statementType = StatementType.DROP_TEMPLATE; + } + public String getTemplateName() { return templateName; } @@ -52,4 +63,15 @@ public R accept(StatementVisitor visitor, C context) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(templateName, dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + templateName = ReadWriteIOUtils.readString(buffer); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/SetSchemaTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/SetSchemaTemplateStatement.java index 0609d8349c9be..9182a22a427a7 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/SetSchemaTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/SetSchemaTemplateStatement.java @@ -19,13 +19,18 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -41,6 +46,11 @@ public SetSchemaTemplateStatement(String templateName, PartialPath path) { this.path = path; } + public SetSchemaTemplateStatement() { + super(); + statementType = StatementType.SET_TEMPLATE; + } + public String getTemplateName() { return templateName; } @@ -71,4 +81,17 @@ public R accept(StatementVisitor visitor, C context) { public QueryType getQueryType() { return QueryType.WRITE; } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(templateName, dataOutputStream); + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + templateName = ReadWriteIOUtils.readString(buffer); + path = new PartialPath(ReadWriteIOUtils.readString(buffer)); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/UnsetSchemaTemplateStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/UnsetSchemaTemplateStatement.java index de6ca98299d5b..38e64ad7f6ae6 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/UnsetSchemaTemplateStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/template/UnsetSchemaTemplateStatement.java @@ -19,13 +19,18 @@ package org.apache.iotdb.db.mpp.plan.statement.metadata.template; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -41,6 +46,11 @@ public UnsetSchemaTemplateStatement(String templateName, PartialPath path) { this.path = path; } + public UnsetSchemaTemplateStatement() { + super(); + statementType = StatementType.UNSET_TEMPLATE; + } + public String getTemplateName() { return templateName; } @@ -71,4 +81,17 @@ public QueryType getQueryType() { public List getPaths() { return Collections.singletonList(path); } + + @Override + public void serialize(DataOutputStream dataOutputStream) throws IOException { + ReadWriteIOUtils.write(statementType.ordinal(), dataOutputStream); + ReadWriteIOUtils.write(templateName, dataOutputStream); + ReadWriteIOUtils.write(path.getFullPath(), dataOutputStream); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + templateName = ReadWriteIOUtils.readString(buffer); + path = new PartialPath(ReadWriteIOUtils.readString(buffer)); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/quotas/DataNodeThrottleQuotaManager.java b/server/src/main/java/org/apache/iotdb/db/quotas/DataNodeThrottleQuotaManager.java index a9315c332b24f..7b4f5f90475aa 100644 --- a/server/src/main/java/org/apache/iotdb/db/quotas/DataNodeThrottleQuotaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/quotas/DataNodeThrottleQuotaManager.java @@ -81,6 +81,7 @@ public OperationQuota checkQuota(String userName, Statement s) throws RpcThrottl } switch (s.getType()) { case INSERT: + case INSERT_ROW: case BATCH_INSERT: case BATCH_INSERT_ONE_DEVICE: case BATCH_INSERT_ROWS: diff --git a/server/src/main/java/org/apache/iotdb/db/quotas/DefaultOperationQuota.java b/server/src/main/java/org/apache/iotdb/db/quotas/DefaultOperationQuota.java index 652fd74c0c9ea..d513c894f7bd2 100644 --- a/server/src/main/java/org/apache/iotdb/db/quotas/DefaultOperationQuota.java +++ b/server/src/main/java/org/apache/iotdb/db/quotas/DefaultOperationQuota.java @@ -90,6 +90,7 @@ protected void updateEstimateConsumeQuota(int numWrites, int numReads, Statement if (numWrites > 0) { long avgSize = 0; switch (s.getType()) { + case INSERT_ROW: case INSERT: // InsertStatement InsertRowStatement if (s instanceof InsertStatement) { diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java index 5f69695b1094a..12a0eb88a50c6 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java @@ -19,19 +19,28 @@ package org.apache.iotdb.db.service.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.consensus.ConfigNodeRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Operation; import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.db.audit.AuditLogger; import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.client.ConfigNodeClient; +import org.apache.iotdb.db.client.ConfigNodeClientManager; +import org.apache.iotdb.db.client.ConfigNodeInfo; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.OperationType; +import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.metadata.template.TemplateQueryType; import org.apache.iotdb.db.mpp.common.header.DatasetHeader; import org.apache.iotdb.db.mpp.plan.Coordinator; @@ -104,6 +113,7 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSOperationSyncWriteReq; import org.apache.iotdb.service.rpc.thrift.TSPruneSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq; @@ -114,6 +124,7 @@ import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo; import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo; +import org.apache.iotdb.service.rpc.thrift.TSystemStatusResp; import org.apache.iotdb.tsfile.read.common.block.TsBlock; import org.apache.iotdb.tsfile.read.common.block.column.Column; import org.apache.iotdb.tsfile.utils.Pair; @@ -130,6 +141,7 @@ import java.util.Map; import java.util.Optional; +import static org.apache.iotdb.db.client.ConfigNodeClient.MSG_RECONNECTION_FAIL; import static org.apache.iotdb.db.service.basic.ServiceProvider.CURRENT_RPC_VERSION; import static org.apache.iotdb.db.service.basic.ServiceProvider.QUERY_FREQUENCY_RECORDER; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; @@ -216,6 +228,11 @@ private TSExecuteStatementResp executeStatementInternal( DataNodeThrottleQuotaManager.getInstance() .checkQuota(SESSION_MANAGER.getCurrSession().getUsername(), s); + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(s); + } + QUERY_FREQUENCY_RECORDER.incrementAndGet(); if (enableAuditLog) { AuditLogger.log(statement, s); @@ -285,6 +302,11 @@ private TSExecuteStatementResp executeRawDataQueryInternal( return RpcUtils.getTSExecuteStatementResp(status); } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(s); + } + QUERY_FREQUENCY_RECORDER.incrementAndGet(); if (enableAuditLog) { AuditLogger.log(String.format("execute Raw Data Query: %s", req), s); @@ -347,6 +369,12 @@ private TSExecuteStatementResp executeLastDataQueryInternal( if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return RpcUtils.getTSExecuteStatementResp(status); } + + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(s); + } + QUERY_FREQUENCY_RECORDER.incrementAndGet(); if (enableAuditLog) { AuditLogger.log(String.format("Last Data Query: %s", req), s); @@ -584,6 +612,11 @@ public TSStatus setStorageGroup(long sessionId, String storageGroup) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -626,6 +659,11 @@ public TSStatus createTimeseries(TSCreateTimeseriesReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -675,6 +713,11 @@ public TSStatus createAlignedTimeseries(TSCreateAlignedTimeseriesReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -723,6 +766,11 @@ public TSStatus createMultiTimeseries(TSCreateMultiTimeseriesReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -761,6 +809,11 @@ public TSStatus deleteTimeseries(long sessionId, List path) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -803,6 +856,11 @@ public TSStatus deleteStorageGroups(long sessionId, List storageGroups) return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -859,6 +917,11 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(s); + } + QUERY_FREQUENCY_RECORDER.incrementAndGet(); if (enableAuditLog) { @@ -981,6 +1044,11 @@ public TSStatus insertRecords(TSInsertRecordsReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1039,6 +1107,11 @@ public TSStatus insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1096,6 +1169,11 @@ public TSStatus insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceR return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1153,6 +1231,11 @@ public TSStatus insertRecord(TSInsertRecordReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1201,6 +1284,11 @@ public TSStatus insertTablets(TSInsertTabletsReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1248,6 +1336,11 @@ public TSStatus insertTablet(TSInsertTabletReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1304,6 +1397,11 @@ public TSStatus insertStringRecords(TSInsertStringRecordsReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = COORDINATOR.execute( @@ -1383,6 +1481,11 @@ public TSStatus deleteData(TSDeleteDataReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = COORDINATOR.execute( @@ -1437,6 +1540,11 @@ public TSStatus createSchemaTemplate(TSCreateSchemaTemplateReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1521,6 +1629,11 @@ private TSQueryTemplateResp executeTemplateQueryStatement( return resp; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + QUERY_FREQUENCY_RECORDER.incrementAndGet(); if (enableAuditLog) { @@ -1599,6 +1712,11 @@ public TSStatus setSchemaTemplate(TSSetSchemaTemplateReq req) throws TException return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1644,6 +1762,11 @@ public TSStatus unsetSchemaTemplate(TSUnsetSchemaTemplateReq req) throws TExcept return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1686,6 +1809,11 @@ public TSStatus dropSchemaTemplate(TSDropSchemaTemplateReq req) throws TExceptio return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = @@ -1735,6 +1863,79 @@ public TSConnectionInfoResp fetchAllConnectionsInfo() throws TException { return SESSION_MANAGER.getAllConnectionInfo(); } + @Override + public TSStatus executeOperationSync(TSOperationSyncWriteReq req) throws TException { + boolean finished = false; + long queryId = Long.MIN_VALUE; + + long startTime = System.currentTimeMillis(); + try { + Statement s = Statement.Factory.create(req.statement); + + if (s == null) { + return RpcUtils.getStatus( + TSStatusCode.SQL_PARSE_ERROR, "This operation type is not supported"); + } + + QUERY_FREQUENCY_RECORDER.incrementAndGet(); + if (enableAuditLog) { + AuditLogger.log("Synchronous write ", s); + } + + queryId = SESSION_MANAGER.requestQueryId(); + // create and cache dataset + ExecutionResult result = + COORDINATOR.execute( + s, + queryId, + SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()), + "", + PARTITION_FETCHER, + SCHEMA_FETCHER); + + return result.status; + } catch (Exception e) { + finished = true; + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } finally { + addOperationLatency(Operation.EXECUTE_QUERY, startTime); + if (finished) { + COORDINATOR.cleanupQueryExecution(queryId); + } + } + } + + @Override + public TSystemStatusResp getSystemStatus() throws TException { + TSystemStatusResp resp = new TSystemStatusResp(); + IClientManager CONFIG_NODE_CLIENT_MANAGER = + ConfigNodeClientManager.getInstance(); + TShowClusterResp showClusterResp = new TShowClusterResp(); + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.configNodeRegionId)) { + showClusterResp = client.showCluster(); + for (String status : showClusterResp.getNodeStatus().values()) { + if (!status.equals(NodeStatus.Running.name())) { + resp.setSystemStatus(NodeStatus.Unknown.name()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } + } + return new TSystemStatusResp( + NodeStatus.Running.name(), RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + } catch (ClientManagerException | TException e) { + if (showClusterResp.getConfigNodeList() == null) { + resp.setSystemStatus(NodeStatus.Unknown.name()); + resp.setStatus( + RpcUtils.getStatus(TSStatusCode.REDIRECTION_RECOMMEND, MSG_RECONNECTION_FAIL)); + } else { + resp.setSystemStatus(NodeStatus.Unknown.name()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR)); + } + return resp; + } + } + @Override public TSStatus insertStringRecord(TSInsertStringRecordReq req) { long t1 = System.currentTimeMillis(); @@ -1763,6 +1964,11 @@ public TSStatus insertStringRecord(TSInsertStringRecordReq req) { return status; } + if (config.isEnableOperationSync()) { + // OperationSync should transmit before execute + StorageEngineV2.transmitOperationSync(statement); + } + // Step 2: call the coordinator long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index e0575880ea719..6e4ca3591389a 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -438,6 +438,65 @@ public static BitMap[] readBitMapsFromBuffer(ByteBuffer buffer, int columns, int return bitMaps; } + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + public static Object[] readValuesFromBuffer( + ByteBuffer buffer, TSDataType[] types, int columns, int size) { + Object[] values = new Object[columns]; + for (int i = 0; i < columns; i++) { + switch (types[i]) { + case BOOLEAN: + boolean[] boolValues = new boolean[size]; + for (int index = 0; index < size; index++) { + boolValues[index] = BytesUtils.byteToBool(buffer.get()); + } + values[i] = boolValues; + break; + case INT32: + int[] intValues = new int[size]; + for (int index = 0; index < size; index++) { + intValues[index] = buffer.getInt(); + } + values[i] = intValues; + break; + case INT64: + long[] longValues = new long[size]; + for (int index = 0; index < size; index++) { + longValues[index] = buffer.getLong(); + } + values[i] = longValues; + break; + case FLOAT: + float[] floatValues = new float[size]; + for (int index = 0; index < size; index++) { + floatValues[index] = buffer.getFloat(); + } + values[i] = floatValues; + break; + case DOUBLE: + double[] doubleValues = new double[size]; + for (int index = 0; index < size; index++) { + doubleValues[index] = buffer.getDouble(); + } + values[i] = doubleValues; + break; + case TEXT: + Binary[] binaryValues = new Binary[size]; + for (int index = 0; index < size; index++) { + int binarySize = buffer.getInt(); + byte[] binaryValue = new byte[binarySize]; + buffer.get(binaryValue); + binaryValues[index] = new Binary(binaryValue); + } + values[i] = binaryValues; + break; + default: + throw new UnSupportedDataTypeException( + String.format("data type %s is not supported when convert data at client", types[i])); + } + } + return values; + } + public static BitMap[] readBitMapsFromStream(DataInputStream stream, int columns, int size) throws IOException { if (stream.available() <= 0) { diff --git a/server/src/main/java/org/apache/iotdb/db/utils/writelog/BatchLogSReader.java b/server/src/main/java/org/apache/iotdb/db/utils/writelog/BatchLogSReader.java new file mode 100644 index 0000000000000..2c49f8642ed28 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/writelog/BatchLogSReader.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils.writelog; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.db.mpp.plan.statement.Statement; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class BatchLogSReader implements ILogSReader { + private static Logger logger = LoggerFactory.getLogger(BatchLogSReader.class); + + private Iterator planIterator; + + private boolean fileCorrupted = false; + + BatchLogSReader(ByteBuffer buffer) { + List logs = readLogs(buffer); + this.planIterator = logs.iterator(); + } + + private List readLogs(ByteBuffer buffer) { + List plans = new ArrayList<>(); + while (buffer.position() != buffer.limit()) { + try { + plans.add(Statement.Factory.create(buffer)); + } catch (IllegalPathException e) { + logger.error("Cannot deserialize Statements from ByteBuffer, ignore remaining logs", e); + fileCorrupted = true; + break; + } + } + return plans; + } + + @Override + public void close() { + // nothing to be closed + } + + @Override + public boolean hasNext() { + return planIterator.hasNext(); + } + + @Override + public Statement next() { + return planIterator.next(); + } + + public boolean isFileCorrupted() { + return fileCorrupted; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/utils/writelog/ILogSReader.java b/server/src/main/java/org/apache/iotdb/db/utils/writelog/ILogSReader.java new file mode 100644 index 0000000000000..72533c7299758 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/writelog/ILogSReader.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.writelog; + +import org.apache.iotdb.db.mpp.plan.statement.Statement; + +import java.io.FileNotFoundException; +import java.io.IOException; + +public interface ILogSReader { + + /** release resources occupied by this object, like file streams. */ + void close(); + + /** + * return whether there exists next log to be read. + * + * @return whether there exists next log to be read. + * @throws IOException + */ + boolean hasNext() throws FileNotFoundException; + + /** + * return the next log read from media like a WAL file and covert it to a PhysicalPlan. + * + * @return the next log as a PhysicalPlan + * @throws java.util.NoSuchElementException when there are no more logs + */ + Statement next() throws FileNotFoundException; +} diff --git a/server/src/main/java/org/apache/iotdb/db/utils/writelog/SingleFileLogSReader.java b/server/src/main/java/org/apache/iotdb/db/utils/writelog/SingleFileLogSReader.java new file mode 100644 index 0000000000000..f611078757448 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/writelog/SingleFileLogSReader.java @@ -0,0 +1,143 @@ +package org.apache.iotdb.db.utils.writelog; + +import org.apache.iotdb.db.mpp.plan.statement.Statement; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.NoSuchElementException; +import java.util.zip.CRC32; + +public class SingleFileLogSReader implements ILogSReader { + private static final Logger logger = LoggerFactory.getLogger(SingleFileLogReader.class); + public static final int LEAST_LOG_SIZE = 12; // size + checksum + + private DataInputStream logStream; + private String filepath; + + private byte[] buffer; + private CRC32 checkSummer = new CRC32(); + + // used to indicate the position of the broken log + private int idx; + // used to truncate the broken logs + private long unbrokenLogsSize = 0; + + private BatchLogSReader batchLogSReader; + + private boolean fileCorrupted = false; + + public SingleFileLogSReader(File logFile) throws FileNotFoundException { + open(logFile); + } + + @Override + public boolean hasNext() { + try { + if (batchLogSReader != null && batchLogSReader.hasNext()) { + return true; + } + + int logSize; + try { + logSize = logStream.readInt(); + } catch (EOFException e) { + truncateBrokenLogs(); + return false; + } + + if (logSize <= 0) { + truncateBrokenLogs(); + return false; + } + buffer = new byte[logSize]; + + int readLen = logStream.read(buffer, 0, logSize); + if (readLen < logSize) { + throw new IOException("Reach eof"); + } + + final long checkSum = logStream.readLong(); + checkSummer.reset(); + checkSummer.update(buffer, 0, logSize); + if (checkSummer.getValue() != checkSum) { + throw new IOException( + String.format( + "The check sum of the No.%d log batch is incorrect! In " + + "file: " + + "%d Calculated: %d.", + idx, checkSum, checkSummer.getValue())); + } + + batchLogSReader = new BatchLogSReader(ByteBuffer.wrap(buffer)); + if (!batchLogSReader.isFileCorrupted()) { + unbrokenLogsSize = unbrokenLogsSize + logSize + LEAST_LOG_SIZE; + } else { + truncateBrokenLogs(); + } + fileCorrupted = fileCorrupted || batchLogSReader.isFileCorrupted(); + } catch (Exception e) { + logger.error( + "Cannot read more PhysicalPlans from {}, successfully read index is {}. The reason is", + idx, + filepath, + e); + truncateBrokenLogs(); + fileCorrupted = true; + return false; + } + return batchLogSReader != null && batchLogSReader.hasNext(); + } + + @Override + public Statement next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + idx++; + return batchLogSReader.next(); + } + + @Override + public void close() { + if (logStream != null) { + try { + logStream.close(); + } catch (IOException e) { + logger.error("Cannot close log file {}", filepath, e); + } + } + } + + public void open(File logFile) throws FileNotFoundException { + close(); + logStream = new DataInputStream(new BufferedInputStream(new FileInputStream(logFile))); + logger.info("open WAL file: {} size is {}", logFile.getName(), logFile.length()); + this.filepath = logFile.getPath(); + idx = 0; + } + + public boolean isFileCorrupted() { + return fileCorrupted; + } + + private void truncateBrokenLogs() { + try (FileOutputStream outputStream = new FileOutputStream(filepath, true); + FileChannel channel = outputStream.getChannel()) { + channel.truncate(unbrokenLogsSize); + } catch (IOException e) { + logger.error("Fail to truncate log file to size {}", unbrokenLogsSize, e); + } + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/doublelive/OperationSyncManualTestUtils.java b/server/src/test/java/org/apache/iotdb/db/doublelive/OperationSyncManualTestUtils.java new file mode 100644 index 0000000000000..fbcf01467be8e --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/doublelive/OperationSyncManualTestUtils.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.doublelive; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * This is a manual test utils for DoubleLive. First start two IoTDB that enable OperationSync to + * use this. + */ +public class OperationSyncManualTestUtils { + + private static final SessionPool sessionPool = + new SessionPool("127.0.0.1", 6667, "root", "root", 3); + + private static final String sg = "root.sg"; + private static final int sgCnt = 10; + private static final String d = ".d"; + private static final int dCnt = 20; + private static final String s = ".s"; + private static final int sCnt = 100; + private static final int dataCnt = 1000; + + public void setStorageGroups() throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < sgCnt; i++) { + sessionPool.createDatabase(sg + i); + } + } + + public void deleteStorageGroups() throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < sgCnt; i++) { + sessionPool.deleteDatabases(Collections.singletonList(sg + i)); + } + } + + public void createTimeSeries() throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < sgCnt; i++) { + String SG = sg + i; + for (int j = 0; j < dCnt; j++) { + String D = d + j; + for (int k = 0; k < sCnt; k++) { + String S = s + k; + sessionPool.createTimeseries( + SG + D + S, TSDataType.INT32, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED); + } + } + } + } + + public void deleteTimeSeries() throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < sgCnt; i++) { + String SG = sg + i; + for (int j = 0; j < dCnt; j++) { + String D = d + j; + for (int k = 0; k < sCnt; k++) { + String S = s + k; + sessionPool.deleteTimeseries(SG + D + S); + } + } + } + } + + public void insertData() throws IoTDBConnectionException, StatementExecutionException { + long startTime = System.currentTimeMillis(); + for (int i = 0; i < sgCnt; i++) { + String SG = sg + i; + for (int j = 0; j < dCnt; j++) { + String D = d + j; + String device = SG + D; + List measurements = new ArrayList<>(); + List types = new ArrayList<>(); + for (int k = 0; k < sCnt; k++) { + measurements.add("s" + k); + types.add(TSDataType.INT32); + } + for (int l = 0; l < dataCnt; l++) { + List values = new ArrayList<>(); + for (int k = 0; k < sCnt; k++) { + values.add(l); + } + sessionPool.insertRecord(device, l, measurements, types, values); + } + } + } + long endTime = System.currentTimeMillis(); + System.out.println( + "Avg time per insert: " + + ((endTime - startTime) / (double) (sgCnt + dCnt + dataCnt)) + + "ms"); + } + + public void deleteData() throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < sgCnt; i++) { + String SG = sg + i; + for (int j = 0; j < dCnt; j++) { + String D = d + j; + for (int k = 0; k < sCnt; k++) { + String S = s + k; + sessionPool.deleteData(Collections.singletonList(SG + D + S), 0, dataCnt); + } + } + } + } +} diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 9acf641b2d7cb..63925b3858a37 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -76,6 +76,7 @@ public enum TSStatusCode { OVERSIZE_RECORD(522), SCHEMA_FILE_REDO_LOG_BROKEN(523), TEMPLATE_NOT_ACTIVATED(524), + DATABASE_NOT_READY(525), // Storage Engine SYSTEM_READ_ONLY(600), diff --git a/session/src/main/java/org/apache/iotdb/session/Session.java b/session/src/main/java/org/apache/iotdb/session/Session.java index 02987857569b7..8911f3c138e84 100644 --- a/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/session/src/main/java/org/apache/iotdb/session/Session.java @@ -45,6 +45,7 @@ import org.apache.iotdb.service.rpc.thrift.TSInsertStringRecordsReq; import org.apache.iotdb.service.rpc.thrift.TSInsertTabletReq; import org.apache.iotdb.service.rpc.thrift.TSInsertTabletsReq; +import org.apache.iotdb.service.rpc.thrift.TSOperationSyncWriteReq; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; import org.apache.iotdb.service.rpc.thrift.TSPruneSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq; @@ -701,6 +702,11 @@ public long getQueryTimeout() { return queryTimeoutInMs; } + @Override + public String getSystemStatus() throws IoTDBConnectionException { + return defaultSessionConnection.getSystemStatus(); + } + /** * execute query sql * @@ -3440,4 +3446,23 @@ public Session build() { version); } } + + /** Transmit insert record request for operation sync */ + @Override + public void operationSyncTransmit(ByteBuffer buffer) + throws IoTDBConnectionException, StatementExecutionException { + try { + TSOperationSyncWriteReq request = genTSExecuteOperationSyncReq(buffer); + defaultSessionConnection.executeOperationSync(request); + } catch (RedirectException e) { + // ignored + } + } + + private TSOperationSyncWriteReq genTSExecuteOperationSyncReq(ByteBuffer buffer) { + TSOperationSyncWriteReq request = new TSOperationSyncWriteReq(); + request.setOperationSyncType((byte) 0); + request.setStatement(buffer); + return request; + } } diff --git a/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 4fa154dc6a501..c7c026a289b9c 100644 --- a/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -51,6 +51,7 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSOperationSyncWriteReq; import org.apache.iotdb.service.rpc.thrift.TSPruneSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; @@ -58,6 +59,7 @@ import org.apache.iotdb.service.rpc.thrift.TSSetSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq; +import org.apache.iotdb.service.rpc.thrift.TSystemStatusResp; import org.apache.iotdb.session.util.SessionUtils; import org.apache.thrift.TException; @@ -202,6 +204,26 @@ protected IClientRPCService.Iface getClient() { return client; } + protected String getSystemStatus() throws IoTDBConnectionException { + TSystemStatusResp resp; + try { + resp = client.getSystemStatus(); + } catch (TException e) { + logger.warn(e.getMessage()); + if (reconnect()) { + try { + resp = client.getSystemStatus(); + } catch (TException tException) { + logger.warn(tException.getMessage()); + throw new IoTDBConnectionException(tException); + } + } else { + throw new IoTDBConnectionException(logForReconnectionFailure()); + } + } + return resp.getSystemStatus(); + } + protected void setTimeZone(String zoneId) throws StatementExecutionException, IoTDBConnectionException { TSSetTimeZoneReq req = new TSSetTimeZoneReq(sessionId, zoneId); @@ -1001,6 +1023,25 @@ public TSConnectionInfoResp fetchAllConnections() throws IoTDBConnectionExceptio } } + protected void executeOperationSync(TSOperationSyncWriteReq request) + throws IoTDBConnectionException, StatementExecutionException, RedirectException { + request.setSessionId(sessionId); + try { + RpcUtils.verifySuccessWithRedirection(client.executeOperationSync(request)); + } catch (TException e) { + if (reconnect()) { + try { + request.setSessionId(sessionId); + RpcUtils.verifySuccess(client.executeOperationSync(request)); + } catch (TException tException) { + throw new IoTDBConnectionException(tException); + } + } else { + throw new IoTDBConnectionException(MSG_RECONNECTION_FAIL); + } + } + } + public boolean isEnableRedirect() { return enableRedirect; } diff --git a/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index ed89b085bf24f..aacbd6b90bd9f 100644 --- a/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.List; import java.util.Map; @@ -2758,6 +2759,22 @@ public long getQueryTimeout() { return queryTimeoutInMs; } + @Override + public String getSystemStatus() throws IoTDBConnectionException { + for (int i = 0; i < RETRY; i++) { + ISession session = getSession(); + try { + String status = session.getSystemStatus(); + putBack(session); + return status; + } catch (RuntimeException e) { + putBack(session); + throw e; + } + } + return null; + } + public static class Builder { private String host = SessionConfig.DEFAULT_HOST; @@ -2892,4 +2909,26 @@ public SessionPool build() { } } } + + /** Transmit insert record request for OperationSync */ + @Override + public boolean operationSyncTransmit(ByteBuffer buffer) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < RETRY; i++) { + ISession session = getSession(); + try { + buffer.position(0); + session.operationSyncTransmit(buffer); + putBack(session); + return true; + } catch (IoTDBConnectionException e) { + // TException means the connection is broken, remove it and get a new one. + cleanSessionAndMayThrowConnectionException(session, i, e); + } catch (StatementExecutionException | RuntimeException e) { + putBack(session); + throw e; + } + } + return false; + } } diff --git a/thrift/src/main/thrift/client.thrift b/thrift/src/main/thrift/client.thrift index ec216613eadb7..a6ebe7777f674 100644 --- a/thrift/src/main/thrift/client.thrift +++ b/thrift/src/main/thrift/client.thrift @@ -457,6 +457,17 @@ struct TSConnectionInfoResp { 1: required list connectionInfoList } +struct TSOperationSyncWriteReq { + 1: required i64 sessionId + 2: required byte operationSyncType + 3: required binary statement +} + +struct TSystemStatusResp { + 1: required string systemStatus + 2: required common.TSStatus status +} + service IClientRPCService { TSExecuteStatementResp executeQueryStatementV2(1:TSExecuteStatementReq req); @@ -570,4 +581,8 @@ service IClientRPCService { TSBackupConfigurationResp getBackupConfiguration(); TSConnectionInfoResp fetchAllConnectionsInfo(); + + common.TSStatus executeOperationSync(1:TSOperationSyncWriteReq req); + + TSystemStatusResp getSystemStatus(); } \ No newline at end of file diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java index f08a1fdb4ab11..e900515bf2cd7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.utils; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -56,6 +57,8 @@ */ public class ReadWriteIOUtils { + private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported."; + public static final int BOOLEAN_LEN = 1; public static final int SHORT_LEN = 2; public static final int INT_LEN = 4; @@ -863,6 +866,18 @@ public static void writeStringList(List list, OutputStream outputStream) } } + public static void writeList(List list, OutputStream outputStream) throws IOException { + if (list == null) { + write(NO_BYTE_TO_READ, outputStream); + } else { + int size = list.size(); + write(size, outputStream); + for (String s : list) { + write(s, outputStream); + } + } + } + /** read integer set with self define length. */ public static Set readIntegerSet(ByteBuffer buffer) { int size = readInt(buffer); @@ -1050,6 +1065,396 @@ public static Object readObject(ByteBuffer buffer) { } } + public static void write(String[] strings, DataOutputStream dataOutputStream) throws IOException { + if (strings == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(strings.length, dataOutputStream); + for (String s : strings) { + write(s, dataOutputStream); + } + } + } + + public static String[] readStrings(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + String[] strings = new String[length]; + for (int i = 0; i < length; i++) { + strings[i] = readString(buffer); + } + return strings; + } + + public static void write(TSDataType[] tsDataTypes, DataOutputStream dataOutputStream) + throws IOException { + if (tsDataTypes == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(tsDataTypes.length, dataOutputStream); + for (TSDataType tsDataType : tsDataTypes) { + write(tsDataType, dataOutputStream); + } + } + } + + public static TSDataType[] readTsDataTypes(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + TSDataType[] tsDataTypes = new TSDataType[length]; + for (int i = 0; i < length; i++) { + tsDataTypes[i] = readDataType(buffer); + } + return tsDataTypes; + } + + public static void write(TSEncoding[] tsEncodings, DataOutputStream dataOutputStream) + throws IOException { + if (tsEncodings == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(tsEncodings.length, dataOutputStream); + for (TSEncoding tsEncoding : tsEncodings) { + write(tsEncoding, dataOutputStream); + } + } + } + + public static TSEncoding[] readTSEncodings(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + TSEncoding[] tsEncodings = new TSEncoding[length]; + for (int i = 0; i < length; i++) { + tsEncodings[i] = readEncoding(buffer); + } + return tsEncodings; + } + + public static void write(CompressionType[] compressionTypes, DataOutputStream dataOutputStream) + throws IOException { + if (compressionTypes == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(compressionTypes.length, dataOutputStream); + for (CompressionType compressionType : compressionTypes) { + write(compressionType, dataOutputStream); + } + } + } + + public static CompressionType[] readCompressionTypes(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + CompressionType[] compressionTypes = new CompressionType[length]; + for (int i = 0; i < length; i++) { + compressionTypes[i] = readCompressionType(buffer); + } + return compressionTypes; + } + + public static void write(long[] longs, DataOutputStream dataOutputStream) throws IOException { + if (longs == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(longs.length, dataOutputStream); + for (long aLong : longs) { + write(aLong, dataOutputStream); + } + } + } + + public static long[] readLongs(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + long[] longs = new long[length]; + for (int i = 0; i < length; i++) { + longs[i] = readLong(buffer); + } + return longs; + } + + public static void serializeValues( + TSDataType[] dataTypes, Object[] columns, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < columns.length; i++) { + if (columns[i] == null) { + continue; + } + serializeColumn(dataTypes[i], columns[i], outputStream); + } + } + + private static void serializeColumn( + TSDataType dataType, Object column, DataOutputStream outputStream) throws IOException { + switch (dataType) { + case INT32: + int[] intValues = (int[]) column; + for (int intValue : intValues) { + outputStream.writeInt(intValue); + } + break; + case INT64: + long[] longValues = (long[]) column; + for (long longValue : longValues) { + outputStream.writeLong(longValue); + } + break; + case FLOAT: + float[] floatValues = (float[]) column; + for (float floatValue : floatValues) { + outputStream.writeFloat(floatValue); + } + break; + case DOUBLE: + double[] doubleValues = (double[]) column; + for (double doubleValue : doubleValues) { + outputStream.writeDouble(doubleValue); + } + break; + case BOOLEAN: + boolean[] boolValues = (boolean[]) column; + for (boolean boolValue : boolValues) { + outputStream.writeByte(BytesUtils.boolToByte(boolValue)); + } + break; + case TEXT: + Binary[] binaryValues = (Binary[]) column; + for (Binary binaryValue : binaryValues) { + outputStream.writeInt(binaryValue.getLength()); + outputStream.write(binaryValue.getValues()); + } + break; + default: + throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); + } + } + + public static void write(List stringsList, DataOutputStream dataOutputStream) + throws IOException { + if (stringsList == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(stringsList.size(), dataOutputStream); + for (String[] strings : stringsList) { + write(strings, dataOutputStream); + } + } + } + + public static List readStringsList(ByteBuffer buffer) { + int size = readInt(buffer); + if (size < 0) { + return null; + } + List stringsList = new ArrayList<>(); + for (int i = 0; i < size; i++) { + String[] strings = readStrings(buffer); + stringsList.add(strings); + } + return stringsList; + } + + public static void write(Set stringSet, DataOutputStream dataOutputStream) + throws IOException { + if (stringSet == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(stringSet.size(), dataOutputStream); + for (String s : stringSet) { + write(s, dataOutputStream); + } + } + } + + public static Set readStringSet(ByteBuffer buffer) { + int size = readInt(buffer); + if (size < 0) { + return null; + } + Set stringSet = new HashSet<>(); + for (int i = 0; i < size; i++) { + stringSet.add(readString(buffer)); + } + return stringSet; + } + + public static void write(String[][] strings, DataOutputStream dataOutputStream) + throws IOException { + if (strings == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(strings.length, dataOutputStream); + for (String[] string : strings) { + write(string, dataOutputStream); + } + } + } + + public static String[][] readArrayOfString(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + String[][] strings = new String[length][]; + for (int i = 0; i < length; i++) { + strings[i] = readStrings(buffer); + } + return strings; + } + + public static void write(TSDataType[][] tsDataTypes, DataOutputStream dataOutputStream) + throws IOException { + if (tsDataTypes == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(tsDataTypes.length, dataOutputStream); + for (TSDataType[] dataTypes : tsDataTypes) { + write(dataTypes, dataOutputStream); + } + } + } + + public static TSDataType[][] readArrayOfTSDataType(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + TSDataType[][] tsDataTypes = new TSDataType[length][]; + for (int i = 0; i < length; i++) { + tsDataTypes[i] = readTsDataTypes(buffer); + } + return tsDataTypes; + } + + public static void write(TSEncoding[][] tsEncodings, DataOutputStream dataOutputStream) + throws IOException { + if (tsEncodings == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(tsEncodings.length, dataOutputStream); + for (TSEncoding[] encodings : tsEncodings) { + write(encodings, dataOutputStream); + } + } + } + + public static TSEncoding[][] readArrayOfTSEncoding(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + TSEncoding[][] tsEncodings = new TSEncoding[length][]; + for (int i = 0; i < length; i++) { + tsEncodings[i] = readTSEncodings(buffer); + } + return tsEncodings; + } + + public static void write(CompressionType[][] compressionTypes, DataOutputStream dataOutputStream) + throws IOException { + if (compressionTypes == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(compressionTypes.length, dataOutputStream); + for (CompressionType[] compressionType : compressionTypes) { + write(compressionType, dataOutputStream); + } + } + } + + public static CompressionType[][] readArrayOfCompressionType(ByteBuffer buffer) { + int length = readInt(buffer); + if (length < 0) { + return null; + } + CompressionType[][] compressionTypes = new CompressionType[length][]; + for (int i = 0; i < length; i++) { + compressionTypes[i] = readCompressionTypes(buffer); + } + return compressionTypes; + } + + public static List readTSDataTypeList(ByteBuffer buffer) { + int size = readInt(buffer); + if (size <= 0) { + return Collections.emptyList(); + } + List list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add(readDataType(buffer)); + } + return list; + } + + public static void writeTSDataTypeList(List list, DataOutputStream dataOutputStream) + throws IOException { + if (list == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(list.size(), dataOutputStream); + for (TSDataType dataType : list) { + write(dataType, dataOutputStream); + } + } + } + + public static List readTSEncodingList(ByteBuffer buffer) { + int size = readInt(buffer); + if (size <= 0) { + return Collections.emptyList(); + } + List list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add(readEncoding(buffer)); + } + return list; + } + + public static void writeTSEncodingList(List list, DataOutputStream dataOutputStream) + throws IOException { + if (list == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(list.size(), dataOutputStream); + for (TSEncoding encoding : list) { + write(encoding, dataOutputStream); + } + } + } + + public static List readCompressionTypeList(ByteBuffer buffer) { + int size = readInt(buffer); + if (size <= 0) { + return Collections.emptyList(); + } + List list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add(readCompressionType(buffer)); + } + return list; + } + + public static void writeCompressionTypeList( + List list, DataOutputStream dataOutputStream) throws IOException { + if (list == null) { + write(NO_BYTE_TO_READ, dataOutputStream); + } else { + write(list.size(), dataOutputStream); + for (CompressionType compressionType : list) { + write(compressionType, dataOutputStream); + } + } + } + public static ByteBuffer clone(ByteBuffer original) { ByteBuffer clone = ByteBuffer.allocate(original.remaining()); while (original.hasRemaining()) {