fields = dataSet.next().getFields();
+ final int regionId = fields.get(regionIdIndex).getIntV();
+ final String regionStatus = fields.get(regionStatusIndex).toString();
+ if (!"Running".equals(regionStatus)) {
+ result.putIfAbsent(regionId, regionStatus);
+ }
+ }
+ return result;
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
index da6bf2dfdc6f1..23829ae31ee45 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.confignode.it.regionmigration.pass;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState;
import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -31,7 +31,7 @@
@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
-public class IoTDBRegionMigrateClusterCrashIT extends IoTDBRegionMigrateReliabilityITFramework {
+public class IoTDBRegionMigrateClusterCrashIT extends IoTDBRegionOperationReliabilityITFramework {
@Test
public void clusterCrash1() throws Exception {
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
index f6916eff40282..72d8d129cfce6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
@@ -21,7 +21,7 @@
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
import org.apache.iotdb.commons.utils.KillPoint.KillPoint;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState;
import org.apache.iotdb.confignode.procedure.state.RegionTransitionState;
import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState;
@@ -39,7 +39,8 @@
@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
-public class IoTDBRegionMigrateConfigNodeCrashIT extends IoTDBRegionMigrateReliabilityITFramework {
+public class IoTDBRegionMigrateConfigNodeCrashIT
+ extends IoTDBRegionOperationReliabilityITFramework {
@Test
@Ignore
public void cnCrashDuringPreCheckTest() throws Exception {
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
index f56296b9cfaea..e2d6a9db57ba2 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.confignode.it.regionmigration.pass;
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -30,7 +30,7 @@
@Category({ClusterIT.class})
@RunWith(IoTDBTestRunner.class)
-public class IoTDBRegionMigrateNormalIT extends IoTDBRegionMigrateReliabilityITFramework {
+public class IoTDBRegionMigrateNormalIT extends IoTDBRegionOperationReliabilityITFramework {
@Test
public void normal1C2DTest() throws Exception {
successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(), KillNode.ALL_NODES);
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java
index f4ca461edd8b6..b5b2f7759e739 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateOtherIT.java
@@ -21,7 +21,7 @@
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
import org.apache.iotdb.commons.utils.KillPoint.NeverTriggeredKillPoint;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -32,7 +32,7 @@
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
-public class IoTDBRegionMigrateOtherIT extends IoTDBRegionMigrateReliabilityITFramework {
+public class IoTDBRegionMigrateOtherIT extends IoTDBRegionOperationReliabilityITFramework {
@Test
public void badKillPoint() throws Exception {
try {
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java
new file mode 100644
index 0000000000000..3c4aa16a401e4
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.regionmigration.pass.commit;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
+import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
+@Category({ClusterIT.class})
+@RunWith(IoTDBTestRunner.class)
+public class IoTDBRegionGroupExpandAndShrinkForIoTV1IT
+ extends IoTDBRegionOperationReliabilityITFramework {
+ private static final String EXPAND_FORMAT = "extend region %d to %d";
+ private static final String SHRINK_FORMAT = "remove region %d from %d";
+
+ private static Logger LOGGER =
+ LoggerFactory.getLogger(IoTDBRegionGroupExpandAndShrinkForIoTV1IT.class);
+
+ /**
+ * 1. Expand: {a} -> {a,b} -> ... -> {a,b,c,d,e}
+ *
+ * 2. Check
+ *
+ *
3. Shrink: {a,b,c,d,e} -> {a,c,d,e} -> ... -> {d}
+ *
+ *
4. Check
+ */
+ @Test
+ public void normal1C5DTest() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataReplicationFactor(1)
+ .setSchemaReplicationFactor(1);
+
+ EnvFactory.getEnv().initClusterEnvironment(1, 5);
+
+ try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement = makeItCloseQuietly(connection.createStatement());
+ SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ // prepare data
+ statement.execute(INSERTION1);
+ statement.execute(FLUSH_COMMAND);
+
+ // collect necessary information
+ Map> regionMap = getAllRegionMap(statement);
+ Set allDataNodeId = getAllDataNodes(statement);
+
+ // expect one data region, one schema region
+ Assert.assertEquals(2, regionMap.size());
+
+ // expand
+ for (int selectedRegion : regionMap.keySet()) {
+ for (int i = 0; i < 4; i++) {
+ int targetDataNode =
+ selectDataNodeNotContainsRegion(allDataNodeId, regionMap, selectedRegion);
+ regionGroupExpand(statement, client, selectedRegion, targetDataNode);
+ // update regionMap every time
+ regionMap = getAllRegionMap(statement);
+ }
+ }
+
+ // shrink
+ for (int selectedRegion : regionMap.keySet()) {
+ for (int i = 0; i < 4; i++) {
+ int targetDataNode =
+ selectDataNodeContainsRegion(allDataNodeId, regionMap, selectedRegion);
+ regionGroupShrink(statement, client, selectedRegion, targetDataNode);
+ // update regionMap every time
+ regionMap = getAllRegionMap(statement);
+ }
+ }
+ }
+ }
+
+ private void regionGroupExpand(
+ Statement statement,
+ SyncConfigNodeIServiceClient client,
+ int selectedRegion,
+ int targetDataNode)
+ throws Exception {
+ Awaitility.await()
+ .atMost(10, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ statement.execute(String.format(EXPAND_FORMAT, selectedRegion, targetDataNode));
+ return true;
+ });
+
+ Predicate expandRegionPredicate =
+ tShowRegionResp -> {
+ Map> newRegionMap =
+ getRunningRegionMap(tShowRegionResp.getRegionInfoList());
+ Set dataNodes = newRegionMap.get(selectedRegion);
+ return dataNodes.contains(targetDataNode);
+ };
+
+ awaitUntilSuccess(client, expandRegionPredicate, Optional.of(targetDataNode), Optional.empty());
+
+ LOGGER.info("Region {} has expanded to DataNode {}", selectedRegion, targetDataNode);
+ }
+
+ private void regionGroupShrink(
+ Statement statement,
+ SyncConfigNodeIServiceClient client,
+ int selectedRegion,
+ int targetDataNode)
+ throws Exception {
+ Awaitility.await()
+ .atMost(10, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ statement.execute(String.format(SHRINK_FORMAT, selectedRegion, targetDataNode));
+ return true;
+ });
+
+ Predicate shrinkRegionPredicate =
+ tShowRegionResp -> {
+ Map> newRegionMap =
+ getRegionMap(tShowRegionResp.getRegionInfoList());
+ Set dataNodes = newRegionMap.get(selectedRegion);
+ return !dataNodes.contains(targetDataNode);
+ };
+
+ awaitUntilSuccess(client, shrinkRegionPredicate, Optional.empty(), Optional.of(targetDataNode));
+
+ LOGGER.info("Region {} has shrunk from DataNode {}", selectedRegion, targetDataNode);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java
new file mode 100644
index 0000000000000..9c720bfbc2746
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBRegionReconstructForIoTV1IT.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.regionmigration.pass.commit;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.read.common.RowRecord;
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
+@Category({ClusterIT.class})
+@RunWith(IoTDBTestRunner.class)
+public class IoTDBRegionReconstructForIoTV1IT extends IoTDBRegionOperationReliabilityITFramework {
+ private static final String RECONSTRUCT_FORMAT = "reconstruct region %d on %d";
+ private static Logger LOGGER = LoggerFactory.getLogger(IoTDBRegionReconstructForIoTV1IT.class);
+
+ @Test
+ public void normal1C3DTest() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataReplicationFactor(2)
+ .setSchemaReplicationFactor(3);
+
+ EnvFactory.getEnv().initClusterEnvironment(1, 3);
+
+ try (Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ Statement statement = makeItCloseQuietly(connection.createStatement());
+ SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ // prepare data
+ statement.execute(INSERTION1);
+ statement.execute(FLUSH_COMMAND);
+
+ // collect necessary information
+ Map> dataRegionMap = getDataRegionMap(statement);
+ Set allDataNodeId = getAllDataNodes(statement);
+
+ // select datanode
+ final int selectedRegion = 1;
+ Assert.assertTrue(dataRegionMap.containsKey(selectedRegion));
+ Assert.assertEquals(2, dataRegionMap.get(selectedRegion).size());
+ Iterator iterator = dataRegionMap.get(selectedRegion).iterator();
+ final int dataNodeToBeClosed = iterator.next();
+ final int dataNodeToBeReconstructed = iterator.next();
+ final int dataNodeAlwaysGood =
+ allDataNodeId.stream()
+ .filter(x -> x != dataNodeToBeReconstructed && x != dataNodeToBeClosed)
+ .findAny()
+ .get();
+ final DataNodeWrapper dataNodeWrapper =
+ EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeAlwaysGood).get();
+ Session session =
+ new Session.Builder()
+ .host(dataNodeWrapper.getIp())
+ .port(dataNodeWrapper.getPort())
+ .build();
+ session.open();
+
+ // delete one DataNode's data dir, stop another DataNode
+ File dataDirToBeReconstructed =
+ new File(
+ EnvFactory.getEnv()
+ .dataNodeIdToWrapper(dataNodeToBeReconstructed)
+ .get()
+ .getDataPath());
+ FileUtils.deleteDirectory(dataDirToBeReconstructed);
+ EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().stopForcibly();
+
+ // now, the query should throw exception
+ Assert.assertThrows(
+ StatementExecutionException.class,
+ () -> session.executeQueryStatement("select * from root.**"));
+
+ // start DataNode, reconstruct the delete one
+ EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().start();
+ EnvFactory.getAbstractEnv().checkNodeInStatus(dataNodeToBeClosed, NodeStatus.Running);
+ session.executeNonQueryStatement(
+ String.format(RECONSTRUCT_FORMAT, selectedRegion, dataNodeToBeReconstructed));
+ try {
+ Awaitility.await()
+ .pollInterval(1, TimeUnit.SECONDS)
+ .atMost(1, TimeUnit.MINUTES)
+ .until(
+ () ->
+ getRegionStatusWithoutRunning(session).isEmpty()
+ && dataDirToBeReconstructed.getAbsoluteFile().exists());
+ } catch (Exception e) {
+ LOGGER.error(
+ "Two factor: {} && {}",
+ getRegionStatusWithoutRunning(session).isEmpty(),
+ dataDirToBeReconstructed.getAbsoluteFile().exists());
+ Assert.fail();
+ }
+ EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeToBeClosed).get().stopForcibly();
+
+ // now, the query should work fine
+ SessionDataSet resultSet = session.executeQueryStatement("select * from root.**");
+ RowRecord rowRecord = resultSet.next();
+ Assert.assertEquals("2.0", rowRecord.getFields().get(0).getStringValue());
+ Assert.assertEquals("1.0", rowRecord.getFields().get(1).getStringValue());
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
index 6819af9f87b3f..e4c99e22c1fa9 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
@@ -21,7 +21,7 @@
import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints;
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.DailyIT;
@@ -31,7 +31,7 @@
@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
-public class IoTDBRegionMigrateDataNodeCrashIT extends IoTDBRegionMigrateReliabilityITFramework {
+public class IoTDBRegionMigrateDataNodeCrashIT extends IoTDBRegionOperationReliabilityITFramework {
// region Coordinator DataNode crash tests
private final int dataReplicateFactor = 2;
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
index 047037c7d147e..2334bd6938513 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash;
import org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints;
-import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework;
+import org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.DailyIT;
@@ -28,10 +28,12 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import static org.junit.platform.commons.function.Try.success;
+
@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT
- extends IoTDBRegionMigrateDataNodeCrashITFramework {
+ extends IoTDBRegionOperationReliabilityITFramework {
@Test
public void crashBeforeDelete() throws Exception {
success(IoTConsensusDeleteLocalPeerKillPoints.BEFORE_DELETE);
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java
new file mode 100644
index 0000000000000..561e99ec3da39
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.removeconfignode;
+
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.confignode.it.removedatanode.SQLModel;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.itbase.exception.InconsistentDataException;
+import org.apache.iotdb.jdbc.IoTDBSQLException;
+
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMap;
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
+public class IoTDBRemoveConfigNodeITFramework {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(IoTDBRemoveConfigNodeITFramework.class);
+ private static final String TREE_MODEL_INSERTION =
+ "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)";
+
+ private static final String SHOW_CONFIGNODES = "show confignodes";
+
+ private static final String defaultSchemaRegionGroupExtensionPolicy = "CUSTOM";
+ private static final String defaultDataRegionGroupExtensionPolicy = "CUSTOM";
+
+ @Before
+ public void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaRegionGroupExtensionPolicy(defaultSchemaRegionGroupExtensionPolicy)
+ .setDataRegionGroupExtensionPolicy(defaultDataRegionGroupExtensionPolicy);
+ }
+
+ @After
+ public void tearDown() throws InterruptedException {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ public void testRemoveConfigNode(
+ final int dataReplicateFactor,
+ final int schemaReplicationFactor,
+ final int configNodeNum,
+ final int dataNodeNum,
+ final int dataRegionPerDataNode,
+ final SQLModel model)
+ throws Exception {
+
+ // Set up the environment
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaReplicationFactor(schemaReplicationFactor)
+ .setDataReplicationFactor(dataReplicateFactor)
+ .setDefaultDataRegionGroupNumPerDatabase(
+ dataRegionPerDataNode * dataNodeNum / dataReplicateFactor);
+ EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum);
+
+ try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement = makeItCloseQuietly(connection.createStatement());
+ SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+
+ // Insert data in tree model
+ statement.execute(TREE_MODEL_INSERTION);
+
+ Map> regionMap = getDataRegionMap(statement);
+ regionMap.forEach(
+ (key, valueSet) -> {
+ LOGGER.info("Key: {}, Value: {}", key, valueSet);
+ if (valueSet.size() != dataReplicateFactor) {
+ Assert.fail();
+ }
+ });
+
+ // Get all config nodes
+ ResultSet result = statement.executeQuery(SHOW_CONFIGNODES);
+ Set allConfigNodeId = new HashSet<>();
+ while (result.next()) {
+ allConfigNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID));
+ }
+
+ AtomicReference clientRef = new AtomicReference<>(client);
+
+ int removeConfigNodeId = allConfigNodeId.iterator().next();
+ String removeConfigNodeSQL = generateRemoveString(removeConfigNodeId);
+ LOGGER.info("Remove ConfigNodes SQL: {}", removeConfigNodeSQL);
+ try {
+ statement.execute(removeConfigNodeSQL);
+ } catch (IoTDBSQLException e) {
+ LOGGER.error("Remove ConfigNodes SQL execute fail: {}", e.getMessage());
+ Assert.fail();
+ }
+ LOGGER.info("Remove ConfigNodes SQL submit successfully.");
+
+ // Wait until success
+ try {
+ awaitUntilSuccess(statement, removeConfigNodeId);
+ } catch (ConditionTimeoutException e) {
+ LOGGER.error("Remove ConfigNodes timeout in 2 minutes");
+ Assert.fail();
+ }
+
+ LOGGER.info("Remove ConfigNodes success");
+ } catch (InconsistentDataException e) {
+ LOGGER.error("Unexpected error:", e);
+ }
+ }
+
+ private static void awaitUntilSuccess(Statement statement, int removeConfigNodeId) {
+ AtomicReference> lastTimeConfigNodes = new AtomicReference<>();
+ AtomicReference lastException = new AtomicReference<>();
+
+ try {
+ Awaitility.await()
+ .atMost(2, TimeUnit.MINUTES)
+ .pollDelay(2, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try {
+ // Get all config nodes
+ ResultSet result = statement.executeQuery(SHOW_CONFIGNODES);
+ Set allConfigNodeId = new HashSet<>();
+ while (result.next()) {
+ allConfigNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID));
+ }
+ lastTimeConfigNodes.set(allConfigNodeId);
+ return !allConfigNodeId.contains(removeConfigNodeId);
+ } catch (Exception e) {
+ // Any exception can be ignored
+ lastException.set(e);
+ return false;
+ }
+ });
+ } catch (ConditionTimeoutException e) {
+ if (lastTimeConfigNodes.get() == null) {
+ LOGGER.error(
+ "Maybe show confignodes fail, lastTimeConfigNodes is null, last Exception:",
+ lastException.get());
+ throw e;
+ }
+ String actualSetStr = lastTimeConfigNodes.get().toString();
+ lastTimeConfigNodes.get().remove(removeConfigNodeId);
+ String expectedSetStr = lastTimeConfigNodes.get().toString();
+ LOGGER.error(
+ "Remove ConfigNode timeout in 2 minutes, expected set: {}, actual set: {}",
+ expectedSetStr,
+ actualSetStr);
+ if (lastException.get() == null) {
+ LOGGER.info("No exception during awaiting");
+ } else {
+ LOGGER.error("Last exception during awaiting:", lastException.get());
+ }
+ throw e;
+ }
+ }
+
+ public static String generateRemoveString(Integer configNodeId) {
+ return "remove confignode " + configNodeId;
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java
new file mode 100644
index 0000000000000..44a1100672e5e
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeNormalIT.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.removeconfignode;
+
+import org.apache.iotdb.confignode.it.removedatanode.SQLModel;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+@Category({ClusterIT.class})
+@RunWith(IoTDBTestRunner.class)
+public class IoTDBRemoveConfigNodeNormalIT extends IoTDBRemoveConfigNodeITFramework {
+ @Test
+ public void test3C1DUseTreeSQL() throws Exception {
+ testRemoveConfigNode(1, 1, 3, 1, 2, SQLModel.TREE_MODEL_SQL);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java
new file mode 100644
index 0000000000000..73d5fd6fac353
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.removedatanode;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRemoveResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.itbase.exception.InconsistentDataException;
+import org.apache.iotdb.jdbc.IoTDBSQLException;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMap;
+import static org.apache.iotdb.util.MagicUtils.makeItCloseQuietly;
+
+public class IoTDBRemoveDataNodeITFramework {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(IoTDBRemoveDataNodeITFramework.class);
+ private static final String TREE_MODEL_INSERTION =
+ "INSERT INTO root.sg.d1(timestamp,speed,temperature) values(100, 1, 2)";
+
+ private static final String SHOW_REGIONS = "show regions";
+ private static final String SHOW_DATANODES = "show datanodes";
+
+ private static final String defaultSchemaRegionGroupExtensionPolicy = "CUSTOM";
+ private static final String defaultDataRegionGroupExtensionPolicy = "CUSTOM";
+
+ @Before
+ public void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setSchemaRegionGroupExtensionPolicy(defaultSchemaRegionGroupExtensionPolicy)
+ .setDataRegionGroupExtensionPolicy(defaultDataRegionGroupExtensionPolicy);
+ }
+
+ @After
+ public void tearDown() throws InterruptedException {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ public void successTest(
+ final int dataReplicateFactor,
+ final int schemaReplicationFactor,
+ final int configNodeNum,
+ final int dataNodeNum,
+ final int removeDataNodeNum,
+ final int dataRegionPerDataNode,
+ final boolean rejoinRemovedDataNode,
+ final SQLModel model)
+ throws Exception {
+ testRemoveDataNode(
+ dataReplicateFactor,
+ schemaReplicationFactor,
+ configNodeNum,
+ dataNodeNum,
+ removeDataNodeNum,
+ dataRegionPerDataNode,
+ true,
+ rejoinRemovedDataNode,
+ model);
+ }
+
+ public void failTest(
+ final int dataReplicateFactor,
+ final int schemaReplicationFactor,
+ final int configNodeNum,
+ final int dataNodeNum,
+ final int removeDataNodeNum,
+ final int dataRegionPerDataNode,
+ final boolean rejoinRemovedDataNode,
+ final SQLModel model)
+ throws Exception {
+ testRemoveDataNode(
+ dataReplicateFactor,
+ schemaReplicationFactor,
+ configNodeNum,
+ dataNodeNum,
+ removeDataNodeNum,
+ dataRegionPerDataNode,
+ false,
+ rejoinRemovedDataNode,
+ model);
+ }
+
+ public void testRemoveDataNode(
+ final int dataReplicateFactor,
+ final int schemaReplicationFactor,
+ final int configNodeNum,
+ final int dataNodeNum,
+ final int removeDataNodeNum,
+ final int dataRegionPerDataNode,
+ final boolean expectRemoveSuccess,
+ final boolean rejoinRemovedDataNode,
+ final SQLModel model)
+ throws Exception {
+ // Set up the environment
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaReplicationFactor(schemaReplicationFactor)
+ .setDataReplicationFactor(dataReplicateFactor)
+ .setDefaultDataRegionGroupNumPerDatabase(
+ dataRegionPerDataNode * dataNodeNum / dataReplicateFactor);
+ EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum);
+
+ try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement = makeItCloseQuietly(connection.createStatement());
+ SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+
+ // Insert data in tree model
+ statement.execute(TREE_MODEL_INSERTION);
+
+ Map> regionMap = getDataRegionMap(statement);
+ regionMap.forEach(
+ (key, valueSet) -> {
+ LOGGER.info("Key: {}, Value: {}", key, valueSet);
+ if (valueSet.size() != dataReplicateFactor) {
+ Assert.fail();
+ }
+ });
+
+ // Get all data nodes
+ ResultSet result = statement.executeQuery(SHOW_DATANODES);
+ Set allDataNodeId = new HashSet<>();
+ while (result.next()) {
+ allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID));
+ }
+
+ // Select data nodes to remove
+ final Set removeDataNodes =
+ selectRemoveDataNodes(allDataNodeId, regionMap, removeDataNodeNum);
+
+ List removeDataNodeWrappers =
+ removeDataNodes.stream()
+ .map(dataNodeId -> EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeId).get())
+ .collect(Collectors.toList());
+
+ AtomicReference clientRef = new AtomicReference<>(client);
+ List removeDataNodeLocations =
+ clientRef
+ .get()
+ .getDataNodeConfiguration(-1)
+ .getDataNodeConfigurationMap()
+ .values()
+ .stream()
+ .map(TDataNodeConfiguration::getLocation)
+ .filter(location -> removeDataNodes.contains(location.getDataNodeId()))
+ .collect(Collectors.toList());
+ if (SQLModel.NOT_USE_SQL.equals(model)) {
+ TDataNodeRemoveReq removeReq = new TDataNodeRemoveReq(removeDataNodeLocations);
+
+ // Remove data nodes
+ TDataNodeRemoveResp removeResp = clientRef.get().removeDataNode(removeReq);
+ LOGGER.info("Submit Remove DataNodes result {} ", removeResp);
+ if (removeResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ if (expectRemoveSuccess) {
+ LOGGER.error("Submit Remove DataNodes fail");
+ Assert.fail();
+ } else {
+ LOGGER.info("Submit Remove DataNodes fail, as expected.");
+ return;
+ }
+ }
+ LOGGER.info("Submit Remove DataNodes request: {}", removeReq);
+
+ } else {
+ String removeDataNodeSQL = generateRemoveString(removeDataNodes);
+ LOGGER.info("Remove DataNodes SQL: {}", removeDataNodeSQL);
+ try {
+ statement.execute(removeDataNodeSQL);
+ } catch (IoTDBSQLException e) {
+ if (expectRemoveSuccess) {
+ LOGGER.error("Remove DataNodes SQL execute fail: {}", e.getMessage());
+ Assert.fail();
+ } else {
+ LOGGER.info("Submit Remove DataNodes fail, as expected");
+ return;
+ }
+ }
+ LOGGER.info("Remove DataNodes SQL submit successfully.");
+ }
+
+ // Wait until success
+ boolean removeSuccess = false;
+ try {
+ awaitUntilSuccess(clientRef, removeDataNodeLocations);
+ removeSuccess = true;
+ } catch (ConditionTimeoutException e) {
+ if (expectRemoveSuccess) {
+ LOGGER.error("Remove DataNodes timeout in 2 minutes");
+ Assert.fail();
+ }
+ }
+
+ if (!expectRemoveSuccess && removeSuccess) {
+ LOGGER.error("Remove DataNodes success, but expect fail");
+ Assert.fail();
+ }
+
+ LOGGER.info("Remove DataNodes success");
+
+ if (rejoinRemovedDataNode) {
+ try {
+ // Use sleep and restart to ensure that removeDataNodes restarts successfully
+ Thread.sleep(30000);
+ restartDataNodes(removeDataNodeWrappers);
+ LOGGER.info("RemoveDataNodes:{} rejoined successfully.", removeDataNodes);
+ } catch (Exception e) {
+ LOGGER.error("RemoveDataNodes rejoin failed.");
+ Assert.fail();
+ }
+ }
+ } catch (InconsistentDataException e) {
+ LOGGER.error("Unexpected error:", e);
+ }
+
+ try (final Connection connection = makeItCloseQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement = makeItCloseQuietly(connection.createStatement())) {
+
+ // Check the data region distribution after removing data nodes
+ Map> afterRegionMap = getDataRegionMap(statement);
+ afterRegionMap.forEach(
+ (key, valueSet) -> {
+ LOGGER.info("Key: {}, Value: {}", key, valueSet);
+ if (valueSet.size() != dataReplicateFactor) {
+ Assert.fail();
+ }
+ });
+
+ if (rejoinRemovedDataNode) {
+ ResultSet result = statement.executeQuery(SHOW_DATANODES);
+ Set allDataNodeId = new HashSet<>();
+ while (result.next()) {
+ allDataNodeId.add(result.getInt(ColumnHeaderConstant.NODE_ID));
+ }
+ Assert.assertEquals(allDataNodeId.size(), dataNodeNum);
+ }
+ } catch (InconsistentDataException e) {
+ LOGGER.error("Unexpected error:", e);
+ }
+ }
+
+ private static Set selectRemoveDataNodes(
+ Set allDataNodeId, Map> regionMap, int removeDataNodeNum) {
+ Set removeDataNodeIds = new HashSet<>();
+ for (int i = 0; i < removeDataNodeNum; i++) {
+ int removeDataNodeId = allDataNodeId.iterator().next();
+ removeDataNodeIds.add(removeDataNodeId);
+ allDataNodeId.remove(removeDataNodeId);
+ }
+ return removeDataNodeIds;
+ }
+
+ private static void awaitUntilSuccess(
+ AtomicReference clientRef,
+ List removeDataNodeLocations) {
+ AtomicReference> lastTimeDataNodeLocations = new AtomicReference<>();
+ AtomicReference lastException = new AtomicReference<>();
+
+ try {
+ Awaitility.await()
+ .atMost(2, TimeUnit.MINUTES)
+ .pollDelay(2, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try {
+ List remainingDataNodes =
+ clientRef
+ .get()
+ .getDataNodeConfiguration(-1)
+ .getDataNodeConfigurationMap()
+ .values()
+ .stream()
+ .map(TDataNodeConfiguration::getLocation)
+ .collect(Collectors.toList());
+ lastTimeDataNodeLocations.set(remainingDataNodes);
+ for (TDataNodeLocation location : removeDataNodeLocations) {
+ if (remainingDataNodes.contains(location)) {
+ return false;
+ }
+ }
+ return true;
+ } catch (TException e) {
+ clientRef.set(
+ (SyncConfigNodeIServiceClient)
+ EnvFactory.getEnv().getLeaderConfigNodeConnection());
+ lastException.set(e);
+ return false;
+ } catch (Exception e) {
+ // Any exception can be ignored
+ lastException.set(e);
+ return false;
+ }
+ });
+ } catch (ConditionTimeoutException e) {
+ if (lastTimeDataNodeLocations.get() == null) {
+ LOGGER.error(
+ "Maybe getDataNodeConfiguration fail, lastTimeDataNodeLocations is null, last Exception:",
+ lastException.get());
+ throw e;
+ }
+ String actualSetStr = lastTimeDataNodeLocations.get().toString();
+ lastTimeDataNodeLocations.get().removeAll(removeDataNodeLocations);
+ String expectedSetStr = lastTimeDataNodeLocations.get().toString();
+ LOGGER.error(
+ "Remove DataNodes timeout in 2 minutes, expected set: {}, actual set: {}",
+ expectedSetStr,
+ actualSetStr);
+ if (lastException.get() == null) {
+ LOGGER.info("No exception during awaiting");
+ } else {
+ LOGGER.error("Last exception during awaiting:", lastException.get());
+ }
+ throw e;
+ }
+
+ LOGGER.info("DataNodes has been successfully changed to {}", lastTimeDataNodeLocations.get());
+ }
+
+ public void restartDataNodes(List dataNodeWrappers) {
+ dataNodeWrappers.parallelStream()
+ .forEach(
+ nodeWrapper -> {
+ nodeWrapper.stopForcibly();
+ Awaitility.await()
+ .atMost(1, TimeUnit.MINUTES)
+ .pollDelay(2, TimeUnit.SECONDS)
+ .until(() -> !nodeWrapper.isAlive());
+ LOGGER.info("Node {} stopped.", nodeWrapper.getId());
+ nodeWrapper.start();
+ Awaitility.await()
+ .atMost(1, TimeUnit.MINUTES)
+ .pollDelay(2, TimeUnit.SECONDS)
+ .until(nodeWrapper::isAlive);
+ try {
+ TimeUnit.SECONDS.sleep(10);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ LOGGER.info("Node {} restarted.", nodeWrapper.getId());
+ });
+ }
+
+ public static String generateRemoveString(Set dataNodes) {
+ StringBuilder sb = new StringBuilder("remove datanode ");
+
+ for (Integer node : dataNodes) {
+ sb.append(node).append(", ");
+ }
+
+ sb.setLength(sb.length() - 2);
+
+ return sb.toString();
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java
new file mode 100644
index 0000000000000..517d8ee4cad67
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeNormalIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.removedatanode;
+
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+@Category({ClusterIT.class})
+@RunWith(IoTDBTestRunner.class)
+public class IoTDBRemoveDataNodeNormalIT extends IoTDBRemoveDataNodeITFramework {
+
+ @Test
+ public void success1C4DTest() throws Exception {
+ successTest(2, 3, 1, 4, 1, 2, true, SQLModel.NOT_USE_SQL);
+ }
+
+ @Test
+ public void fail1C3DTest() throws Exception {
+ failTest(2, 3, 1, 3, 1, 2, false, SQLModel.NOT_USE_SQL);
+ }
+
+ @Test
+ public void success1C4DTestUseSQL() throws Exception {
+ successTest(2, 3, 1, 4, 1, 2, true, SQLModel.TREE_MODEL_SQL);
+ }
+
+ @Test
+ public void fail1C3DTestUseSQL() throws Exception {
+ failTest(2, 3, 1, 3, 1, 2, false, SQLModel.TREE_MODEL_SQL);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java
new file mode 100644
index 0000000000000..0963072d44cae
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/SQLModel.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.removedatanode;
+
+public enum SQLModel {
+ NOT_USE_SQL,
+
+ TREE_MODEL_SQL,
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
index 76feab5efe32d..5f9da5ac36a83 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
@@ -323,8 +323,8 @@ public static TClusterParameters generateClusterParameters() {
clusterParameters.setTimePartitionInterval(604800000);
clusterParameters.setDataReplicationFactor(1);
clusterParameters.setSchemaReplicationFactor(1);
- clusterParameters.setDataRegionPerDataNode(5.0);
- clusterParameters.setSchemaRegionPerDataNode(1.0);
+ clusterParameters.setDataRegionPerDataNode(0);
+ clusterParameters.setSchemaRegionPerDataNode(1);
clusterParameters.setDiskSpaceWarningThreshold(0.01);
clusterParameters.setReadConsistencyLevel("strong");
clusterParameters.setTimestampPrecision("ms");
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
index bb3a8de7d12f4..620043e6c7b7b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
@@ -128,5 +128,14 @@ public void testBigDateTime() {
e.printStackTrace();
fail();
}
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("insert into root.sg.d1(time,s2) values (16182830055860000000, 8.76);");
+ fail();
+ } catch (SQLException e) {
+ Assert.assertTrue(
+ e.getMessage()
+ .contains("please check whether the timestamp 16182830055860000000 is correct."));
+ }
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java
index 209657e516185..047bb10bfd3ca 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDuplicateTimeIT.java
@@ -43,8 +43,6 @@ public class IoTDBDuplicateTimeIT {
@Before
public void setUp() throws Exception {
- EnvFactory.getEnv().getConfig().getCommonConfig().setAvgSeriesPointNumberThreshold(2);
- // Adjust memstable threshold size to make it flush automatically
EnvFactory.getEnv().initClusterEnvironment();
}
@@ -62,6 +60,7 @@ public void testDuplicateTime() throws SQLException {
// version-1 tsfile
statement.execute("insert into root.db.d1(time,s1) values (2,2)");
statement.execute("insert into root.db.d1(time,s1) values (3,3)");
+ statement.execute("flush");
// version-2 unseq work memtable
statement.execute("insert into root.db.d1(time,s1) values (2,20)");
@@ -69,9 +68,11 @@ public void testDuplicateTime() throws SQLException {
// version-3 tsfile
statement.execute("insert into root.db.d1(time,s1) values (5,5)");
statement.execute("insert into root.db.d1(time,s1) values (6,6)");
+ statement.execute("flush root.db true");
// version-2 unseq work memtable -> unseq tsfile
statement.execute("insert into root.db.d1(time,s1) values (5,50)");
+ statement.execute("flush");
try (ResultSet set = statement.executeQuery("SELECT s1 FROM root.db.d1 where time = 5")) {
int cnt = 0;
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java
index aa7b7024677c0..6bcbc762fc53b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBEncodingIT.java
@@ -19,13 +19,17 @@
package org.apache.iotdb.db.it;
+import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -36,22 +40,37 @@
import java.sql.Statement;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class})
public class IoTDBEncodingIT {
- @Before
- public void setUp() throws Exception {
+ private static final String[] databasesToClear =
+ new String[] {"root.db_0", "root.db1", "root.turbine1"};
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
EnvFactory.getEnv().initClusterEnvironment();
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDownClass() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @After
+ public void tearDown() {
+ for (String database : databasesToClear) {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement("DELETE DATABASE " + database);
+ } catch (Exception ignored) {
+
+ }
+ }
+ }
+
@Test
public void testSetEncodingRegularFailed() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -432,4 +451,75 @@ public void testFloatPrecision2() {
fail();
}
}
+
+ @Test
+ public void testCreateNewTypes() throws Exception {
+ String currDB = "root.db1";
+ int seriesCnt = 0;
+ TSDataType[] dataTypes =
+ new TSDataType[] {
+ TSDataType.STRING, TSDataType.BLOB, TSDataType.TIMESTAMP, TSDataType.DATE
+ };
+
+ // supported encodings
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ for (TSDataType dataType : dataTypes) {
+ for (TSEncoding encoding : TSEncoding.values()) {
+ if (encoding.isSupported(dataType)) {
+ statement.execute(
+ "create timeseries "
+ + currDB
+ + ".d1.s"
+ + seriesCnt
+ + " with datatype="
+ + dataType
+ + ", encoding="
+ + encoding
+ + ", compression=SNAPPY");
+ seriesCnt++;
+ }
+ }
+ }
+
+ ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES");
+
+ while (resultSet.next()) {
+ seriesCnt--;
+ }
+ assertEquals(0, seriesCnt);
+ statement.execute("DROP DATABASE " + currDB);
+ }
+
+ // unsupported encodings
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ for (TSDataType dataType : dataTypes) {
+ for (TSEncoding encoding : TSEncoding.values()) {
+ if (!encoding.isSupported(dataType)) {
+ try {
+ statement.execute(
+ "create timeseries "
+ + currDB
+ + ".d1.s"
+ + seriesCnt
+ + " with datatype="
+ + dataType
+ + ", encoding="
+ + encoding
+ + ", compression=SNAPPY");
+ fail("Should have thrown an exception");
+ } catch (SQLException e) {
+ assertEquals(
+ "507: encoding " + encoding + " does not support " + dataType, e.getMessage());
+ }
+ seriesCnt++;
+ }
+ }
+ }
+
+ ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES");
+ assertFalse(resultSet.next());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java
new file mode 100644
index 0000000000000..a0d0a083701ac
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFileTimeIndexIT.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBFileTimeIndexIT {
+
+ private static final String[] sqls =
+ new String[] {
+ "insert into root.db.d1(time,s1) values(2,2)",
+ "insert into root.db.d1(time,s1) values(3,3)",
+ "flush",
+ "insert into root.db.d2(time,s1) values(5,5)",
+ "flush",
+ "insert into root.db.d1(time,s1) values(4,4)",
+ "flush",
+ "insert into root.db.d2(time,s1) values(1,1)",
+ "insert into root.db.d1(time,s1) values(3,30)",
+ "insert into root.db.d1(time,s1) values(4,40)",
+ "flush",
+ "insert into root.db.d2(time,s1) values(2,2)",
+ "insert into root.db.d1(time,s1) values(4,400)",
+ "flush",
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Locale.setDefault(Locale.ENGLISH);
+
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultDataRegionGroupNumPerDatabase(1)
+ .setEnableSeqSpaceCompaction(false)
+ .setEnableUnseqSpaceCompaction(false)
+ .setEnableCrossSpaceCompaction(false)
+ .setQueryMemoryProportion("1:100:200:50:200:200:0:250");
+ // Adjust memstable threshold size to make it flush automatically
+ EnvFactory.getEnv().initClusterEnvironment();
+ prepareData();
+ }
+
+ private static void prepareData() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : sqls) {
+ statement.addBatch(sql);
+ }
+ statement.executeBatch();
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testQuery() throws SQLException {
+ long[] time = {2L, 3L, 4L};
+ double[] value = {2.0f, 30.0f, 400.0f};
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery("select s1 from root.db.d1")) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ assertEquals(time[cnt], resultSet.getLong(1));
+ assertEquals(value[cnt], resultSet.getDouble(2), 0.00001);
+ cnt++;
+ }
+ assertEquals(time.length, cnt);
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java
index 705f4b8d9ede7..2ddb7128d1a5c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFilterNullIT.java
@@ -46,7 +46,8 @@ public class IoTDBFilterNullIT {
"CREATE DATABASE root.testNullFilter",
"CREATE TIMESERIES root.testNullFilter.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
"CREATE TIMESERIES root.testNullFilter.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
- "CREATE TIMESERIES root.testNullFilter.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN"
+ "CREATE TIMESERIES root.testNullFilter.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+ "CREATE ALIGNED TIMESERIES root.testNullFilter.d2(s1 INT32, s2 BOOLEAN, s3 DOUBLE);"
};
private static final String[] insertSqls =
@@ -54,6 +55,9 @@ public class IoTDBFilterNullIT {
"INSERT INTO root.testNullFilter.d1(timestamp,s2,s3) " + "values(1, false, 11.1)",
"INSERT INTO root.testNullFilter.d1(timestamp,s1,s2) " + "values(2, 22, true)",
"INSERT INTO root.testNullFilter.d1(timestamp,s1,s3) " + "values(3, 23, 33.3)",
+ "INSERT INTO root.testNullFilter.d2(timestamp,s2,s3) " + "values(1, false, 11.1)",
+ "INSERT INTO root.testNullFilter.d2(timestamp,s1,s2) " + "values(2, 22, true)",
+ "INSERT INTO root.testNullFilter.d2(timestamp,s1,s2) " + "values(3, 22, false)",
};
private static void prepareData() {
@@ -128,7 +132,34 @@ public void nullFilterTest() {
assertEquals(retArray.length, count);
}
} catch (Exception e) {
- e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void inPushDownTest() {
+ String[] retArray = new String[] {"2,22,true,null", "3,22,false,null"};
+ try (Connection connectionIsNull = EnvFactory.getEnv().getConnection();
+ Statement statementIsNull = connectionIsNull.createStatement()) {
+ int count = 0;
+ try (ResultSet resultSet =
+ statementIsNull.executeQuery(
+ "select * from root.testNullFilter.d2 where s1 in (22, 23)")) {
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(ColumnHeaderConstant.TIME)
+ + ","
+ + resultSet.getString("root.testNullFilter.d2.s1")
+ + ","
+ + resultSet.getString("root.testNullFilter.d2.s2")
+ + ","
+ + resultSet.getString("root.testNullFilter.d2.s3");
+ assertEquals(retArray[count], ans);
+ count++;
+ }
+ assertEquals(retArray.length, count);
+ }
+ } catch (Exception e) {
fail(e.getMessage());
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java
index c72c7e41b1253..3a73be3adb309 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBFloatPrecisionIT.java
@@ -169,4 +169,49 @@ public void selectAllSQLTest() {
fail(e.getMessage());
}
}
+
+ @Test
+ public void bigFloatNumberTest2() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ float[] floats = new float[] {6.5536403E8F, 3.123456768E20F, Float.NaN};
+ double[] doubles = new double[] {9.223372036854E18, 9.223372036854E100, Double.NaN};
+
+ statement.execute("create timeseries root.sg.d1.s1 with datatype=float, encoding=rle");
+ statement.execute("create timeseries root.sg.d1.s2 with datatype=double, encoding=rle");
+ statement.execute(
+ "insert into root.sg.d1(time, s1, s2) values (1, 6.5536403E8, 9.223372036854E18)");
+ statement.execute(
+ "insert into root.sg.d1(time, s1, s2) values (2, 3.123456768E20, 9.223372036854E100)");
+ statement.execute("insert into root.sg.d1(time, s1, s2) values (3, NaN, NaN)");
+
+ int cnt;
+ try (ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg.d1")) {
+ assertNotNull(resultSet);
+ cnt = 0;
+ while (resultSet.next()) {
+ assertEquals(floats[cnt], resultSet.getFloat("root.sg.d1.s1"), DELTA_FLOAT);
+ assertEquals(doubles[cnt], resultSet.getDouble("root.sg.d1.s2"), DELTA_DOUBLE);
+ cnt++;
+ }
+ assertEquals(3, cnt);
+ }
+
+ statement.execute("flush");
+
+ try (ResultSet resultSet = statement.executeQuery("select s1, s2 from root.sg.d1")) {
+ assertNotNull(resultSet);
+ cnt = 0;
+ while (resultSet.next()) {
+ assertEquals(floats[cnt], resultSet.getFloat("root.sg.d1.s1"), DELTA_FLOAT);
+ assertEquals(doubles[cnt], resultSet.getDouble("root.sg.d1.s2"), DELTA_DOUBLE);
+ cnt++;
+ }
+ assertEquals(3, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
index 7ebbcfc8472b3..6e52368be4ce4 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
@@ -56,7 +56,7 @@ public class IoTDBInsertMultiRowIT {
@BeforeClass
public static void setUp() throws Exception {
- EnvFactory.getEnv().getConfig().getCommonConfig().setMaxInnerCompactionCandidateFileNum(2);
+ EnvFactory.getEnv().getConfig().getCommonConfig().setInnerCompactionCandidateFileNum(2);
EnvFactory.getEnv().initClusterEnvironment();
initCreateSQLStatement();
insertData();
@@ -171,6 +171,13 @@ public void testInsertMultiRowWithWrongTimestampPrecision() {
} catch (SQLException e) {
assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
}
+ try (Statement st1 = connection.createStatement()) {
+ st1.execute(
+ "insert into root.t1.d99.wt01(timestamp, s1, s2) values(-1618283005586000, 1, 1), (-1618283005586001, 1, 2)");
+ fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
+ }
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java
index d237423c52f41..3ebefbba2026d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java
@@ -52,6 +52,7 @@ public class IoTDBInsertWithQueryIT {
@Before
public void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false);
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java
new file mode 100644
index 0000000000000..a005f80d19df2
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadLastCacheIT.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.utils.TsFileGenerator;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.jdbc.IoTDBSQLException;
+
+import com.google.common.util.concurrent.RateLimiter;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@SuppressWarnings({"ResultOfMethodCallIgnored", "UnstableApiUsage"})
+@RunWith(Parameterized.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBLoadLastCacheIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBLoadLastCacheIT.class);
+ private static final long PARTITION_INTERVAL = 10 * 1000L;
+ private static final int connectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(300);
+ private static final long loadTsFileAnalyzeSchemaMemorySizeInBytes = 10 * 1024L;
+
+ private File tmpDir;
+ private final LastCacheLoadStrategy lastCacheLoadStrategy;
+
+ @Parameters(name = "loadLastCacheStrategy={0}")
+ public static Collection data() {
+ return Arrays.asList(
+ new Object[][] {
+ {LastCacheLoadStrategy.CLEAN_ALL},
+ {LastCacheLoadStrategy.UPDATE},
+ {LastCacheLoadStrategy.UPDATE_NO_BLOB},
+ {LastCacheLoadStrategy.CLEAN_DEVICE}
+ });
+ }
+
+ public IoTDBLoadLastCacheIT(LastCacheLoadStrategy lastCacheLoadStrategy) {
+ this.lastCacheLoadStrategy = lastCacheLoadStrategy;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ tmpDir = new File(Files.createTempDirectory("load").toUri());
+ EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(PARTITION_INTERVAL);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setConnectionTimeoutInMS(connectionTimeoutInMS)
+ .setLoadTsFileAnalyzeSchemaMemorySizeInBytes(loadTsFileAnalyzeSchemaMemorySizeInBytes);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setLoadLastCacheStrategy(lastCacheLoadStrategy.name())
+ .setCacheLastValuesForLoad(true);
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ deleteSG();
+ EnvFactory.getEnv().cleanClusterEnvironment();
+
+ if (!deleteDir()) {
+ LOGGER.error("Can not delete tmp dir for loading tsfile.");
+ }
+ }
+
+ private void registerSchema() throws SQLException {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_0);
+ statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_1);
+
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00));
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_01));
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_02));
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_03));
+
+ statement.execute(
+ convert2AlignedSQL(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17)));
+
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_2, SchemaConfig.MEASUREMENT_20));
+
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_3, SchemaConfig.MEASUREMENT_30));
+
+ statement.execute(
+ convert2AlignedSQL(
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)));
+ }
+ }
+
+ private String convert2SQL(final String device, final MeasurementSchema schema) {
+ final String sql =
+ String.format(
+ "create timeseries %s %s",
+ new Path(device, schema.getMeasurementId(), true).getFullPath(),
+ schema.getType().name());
+ LOGGER.info("schema execute: {}", sql);
+ return sql;
+ }
+
+ private String convert2AlignedSQL(final String device, final List schemas) {
+ StringBuilder sql = new StringBuilder(String.format("create aligned timeseries %s(", device));
+ for (int i = 0; i < schemas.size(); i++) {
+ final IMeasurementSchema schema = schemas.get(i);
+ sql.append(String.format("%s %s", schema.getMeasurementId(), schema.getType().name()));
+ sql.append(i == schemas.size() - 1 ? ")" : ",");
+ }
+ LOGGER.info("schema execute: {}.", sql);
+ return sql.toString();
+ }
+
+ private void deleteSG() throws SQLException {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_0));
+ statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_1));
+ } catch (final IoTDBSQLException ignored) {
+ }
+ }
+
+ private boolean deleteDir() {
+ for (final File file : Objects.requireNonNull(tmpDir.listFiles())) {
+ if (!file.delete()) {
+ return false;
+ }
+ }
+ return tmpDir.delete();
+ }
+
+ @Test
+ public void testTreeModelLoadWithLastCache() throws Exception {
+ registerSchema();
+
+ final String device = SchemaConfig.DEVICE_0;
+ final String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId();
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ statement.execute(
+ String.format("insert into %s(timestamp, %s) values(100, 100)", device, measurement));
+
+ try (final ResultSet resultSet =
+ statement.executeQuery(String.format("select last %s from %s", measurement, device))) {
+ if (resultSet.next()) {
+ final String lastValue = resultSet.getString(ColumnHeaderConstant.VALUE);
+ Assert.assertEquals("100", lastValue);
+ } else {
+ Assert.fail("This ResultSet is empty.");
+ }
+ }
+ }
+
+ final File file1 = new File(tmpDir, "1-0-0-0.tsfile");
+ final File file2 = new File(tmpDir, "2-0-0-0.tsfile");
+ // device 0, device 1, sg 0
+ try (final TsFileGenerator generator = new TsFileGenerator(file1)) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_0,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_00,
+ SchemaConfig.MEASUREMENT_01,
+ SchemaConfig.MEASUREMENT_02,
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
+ generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
+ generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
+ }
+
+ // device 2, device 3, device4, sg 1
+ try (final TsFileGenerator generator = new TsFileGenerator(file2)) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
+ generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
+ generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
+ generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
+ }
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
+
+ try (final ResultSet resultSet =
+ statement.executeQuery(String.format("select last %s from %s", measurement, device))) {
+ if (resultSet.next()) {
+ final String lastTime = resultSet.getString(ColumnHeaderConstant.TIME);
+ Assert.assertEquals(String.valueOf(PARTITION_INTERVAL), lastTime);
+ } else {
+ Assert.fail("This ResultSet is empty.");
+ }
+ }
+ }
+ }
+
+ private static class PerformanceSchemas {
+
+ private final String database;
+ private final List measurementSchemas;
+ private final List columnNames;
+ private final List dataTypes;
+
+ public PerformanceSchemas(
+ String database, String tableName, int measurementNum, int blobMeasurementNum) {
+ this.database = database;
+ columnNames = new ArrayList<>(measurementNum + blobMeasurementNum);
+ dataTypes = new ArrayList<>(measurementNum + blobMeasurementNum);
+ measurementSchemas = new ArrayList<>(measurementNum + blobMeasurementNum);
+
+ columnNames.add("device_id");
+ dataTypes.add(TSDataType.STRING);
+ for (int i = 0; i < measurementNum; i++) {
+ columnNames.add("s" + i);
+ dataTypes.add(TSDataType.INT64);
+ measurementSchemas.add(new MeasurementSchema("s" + i, TSDataType.INT64));
+ }
+ for (int i = 0; i < blobMeasurementNum; i++) {
+ columnNames.add("s" + (measurementNum + i));
+ dataTypes.add(TSDataType.BLOB);
+ measurementSchemas.add(new MeasurementSchema("s" + (measurementNum + i), TSDataType.BLOB));
+ }
+ }
+ }
+
+ private void generateAndLoadOne(
+ int deviceCnt,
+ int measurementCnt,
+ int blobMeasurementCnt,
+ int pointCnt,
+ int offset,
+ PerformanceSchemas schemas,
+ int fileNum,
+ Statement statement)
+ throws Exception {
+ File file = new File("target" + File.separator + fileNum + ".tsfile");
+ try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
+ int rowIndex = 0;
+ for (int i = 0; i < deviceCnt; i++) {
+ PartialPath devicePath = new PartialPath(new PlainDeviceID("d" + i));
+ tsFileWriter.registerAlignedTimeseries(devicePath, schemas.measurementSchemas);
+ Tablet tablet =
+ new Tablet("root.db1.d" + i, schemas.measurementSchemas, pointCnt * deviceCnt);
+ for (int j = 0; j < pointCnt; j++) {
+ tablet.addTimestamp(rowIndex, j + offset);
+ for (int k = 0; k < measurementCnt; k++) {
+ tablet.addValue("s" + k, rowIndex, (long) j + offset);
+ }
+ for (int k = 0; k < blobMeasurementCnt; k++) {
+ tablet.addValue("s" + (k + 1 + measurementCnt), rowIndex, String.valueOf(j + offset));
+ }
+ rowIndex++;
+ }
+ tsFileWriter.writeAligned(tablet);
+ }
+ }
+
+ statement.execute(String.format("load '%s'", file.getAbsolutePath(), schemas.database));
+
+ file.delete();
+ }
+
+ private void generateAndLoadAll(
+ int deviceCnt,
+ int measurementCnt,
+ int blobMeasurementCnt,
+ int pointCnt,
+ PerformanceSchemas schemas,
+ int fileNum)
+ throws Exception {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ for (int i = 0; i < fileNum; i++) {
+ generateAndLoadOne(
+ deviceCnt,
+ measurementCnt,
+ blobMeasurementCnt,
+ pointCnt,
+ pointCnt * i,
+ schemas,
+ fileNum,
+ statement);
+ }
+ }
+ }
+
+ private long queryLastOnce(int deviceNum, int measurementNum, Statement statement)
+ throws SQLException {
+ try (final ResultSet resultSet =
+ statement.executeQuery(
+ String.format(
+ "select last %s from root.db1.d%s", "s" + measurementNum, "d" + deviceNum))) {
+ if (resultSet.next()) {
+ return resultSet.getLong("_col0");
+ } else {
+ return -1;
+ }
+ } catch (SQLException e) {
+ if (!e.getMessage().contains("does not exist")) {
+ throw e;
+ }
+ }
+ return -1;
+ }
+
+ @SuppressWarnings("BusyWait")
+ private void queryAll(
+ int deviceCnt,
+ int measurementCnt,
+ int pointCnt,
+ int fileCnt,
+ PerformanceSchemas schemas,
+ RateLimiter rateLimiter)
+ throws SQLException {
+ Random random = new Random();
+ long totalStart = System.currentTimeMillis();
+ List timeConsumptions = new ArrayList<>();
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ while (true) {
+ int deviceNum = random.nextInt(deviceCnt);
+ int measurementNum = random.nextInt(measurementCnt);
+ rateLimiter.acquire();
+ long start = System.nanoTime();
+ long result = queryLastOnce(deviceNum, measurementNum, statement);
+ long timeConsumption = System.nanoTime() - start;
+ if (result == -1) {
+ try {
+ Thread.sleep(1000);
+ continue;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ System.out.printf(
+ "%s: d%d.s%d %s %s%n", new Date(), deviceNum, measurementNum, result, timeConsumption);
+ timeConsumptions.add(timeConsumption);
+ if (result == (long) pointCnt * fileCnt - 1) {
+ break;
+ }
+ }
+ }
+
+ System.out.printf(
+ "Synchronization ends after %dms, query latency avg %fms %n",
+ System.currentTimeMillis() - totalStart,
+ timeConsumptions.stream().mapToLong(i -> i).average().orElse(0.0) / 1000000);
+ }
+
+ @Ignore("Performance")
+ @Test
+ public void testLoadPerformance() throws Exception {
+ int deviceCnt = 100;
+ int measurementCnt = 100;
+ int blobMeasurementCnt = 10;
+ int pointCnt = 100;
+ int fileCnt = 100000;
+ int queryPerSec = 100;
+ int queryThreadsNum = 10;
+
+ PerformanceSchemas schemas =
+ new PerformanceSchemas("test", "test_table", measurementCnt, blobMeasurementCnt);
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE IF NOT EXISTS " + schemas.database);
+ }
+
+ Thread loadThread =
+ new Thread(
+ () -> {
+ try {
+ generateAndLoadAll(
+ deviceCnt, measurementCnt, blobMeasurementCnt, pointCnt, schemas, fileCnt);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ });
+
+ RateLimiter rateLimiter = RateLimiter.create(queryPerSec);
+ List queryThreads = new ArrayList<>(queryThreadsNum);
+ for (int i = 0; i < queryThreadsNum; i++) {
+ Thread queryThread =
+ new Thread(
+ () -> {
+ try {
+ queryAll(
+ deviceCnt,
+ measurementCnt + blobMeasurementCnt,
+ pointCnt,
+ fileCnt,
+ schemas,
+ rateLimiter);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ });
+ queryThreads.add(queryThread);
+ }
+
+ loadThread.start();
+ queryThreads.forEach(Thread::start);
+
+ loadThread.join();
+ for (Thread queryThread : queryThreads) {
+ queryThread.join();
+ }
+ }
+
+ private static class SchemaConfig {
+
+ private static final String STORAGE_GROUP_0 = "root.sg.test_0";
+ private static final String STORAGE_GROUP_1 = "root.sg.test_1";
+
+ // device 0, nonaligned, sg 0
+ private static final String DEVICE_0 = "root.sg.test_0.d_0";
+ private static final MeasurementSchema MEASUREMENT_00 =
+ new MeasurementSchema("sensor_00", TSDataType.INT32, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_01 =
+ new MeasurementSchema("sensor_01", TSDataType.INT64, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_02 =
+ new MeasurementSchema("sensor_02", TSDataType.DOUBLE, TSEncoding.GORILLA);
+ private static final MeasurementSchema MEASUREMENT_03 =
+ new MeasurementSchema("sensor_03", TSDataType.TEXT, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_04 =
+ new MeasurementSchema("sensor_04", TSDataType.TIMESTAMP, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_05 =
+ new MeasurementSchema("sensor_05", TSDataType.DATE, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_06 =
+ new MeasurementSchema("sensor_06", TSDataType.BLOB, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_07 =
+ new MeasurementSchema("sensor_07", TSDataType.STRING, TSEncoding.PLAIN);
+
+ // device 1, aligned, sg 0
+ private static final String DEVICE_1 = "root.sg.test_0.a_1";
+ private static final MeasurementSchema MEASUREMENT_10 =
+ new MeasurementSchema("sensor_10", TSDataType.INT32, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_11 =
+ new MeasurementSchema("sensor_11", TSDataType.INT64, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_12 =
+ new MeasurementSchema("sensor_12", TSDataType.DOUBLE, TSEncoding.GORILLA);
+ private static final MeasurementSchema MEASUREMENT_13 =
+ new MeasurementSchema("sensor_13", TSDataType.TEXT, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_14 =
+ new MeasurementSchema("sensor_14", TSDataType.TIMESTAMP, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_15 =
+ new MeasurementSchema("sensor_15", TSDataType.DATE, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_16 =
+ new MeasurementSchema("sensor_16", TSDataType.BLOB, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_17 =
+ new MeasurementSchema("sensor_17", TSDataType.STRING, TSEncoding.PLAIN);
+
+ // device 2, non aligned, sg 1
+ private static final String DEVICE_2 = "root.sg.test_1.d_2";
+ private static final MeasurementSchema MEASUREMENT_20 =
+ new MeasurementSchema("sensor_20", TSDataType.INT32, TSEncoding.RLE);
+
+ // device 3, non aligned, sg 1
+ private static final String DEVICE_3 = "root.sg.test_1.d_3";
+ private static final MeasurementSchema MEASUREMENT_30 =
+ new MeasurementSchema("sensor_30", TSDataType.INT32, TSEncoding.RLE);
+
+ // device 4, aligned, sg 1
+ private static final String DEVICE_4 = "root.sg.test_1.a_4";
+ private static final MeasurementSchema MEASUREMENT_40 =
+ new MeasurementSchema("sensor_40", TSDataType.INT32, TSEncoding.RLE);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index c1b1fdf91091f..12247e277abdc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.it;
import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -31,10 +32,12 @@
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -47,6 +50,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -55,6 +59,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail;
import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
@@ -542,7 +547,9 @@ public void testLoadWithOnSuccess() throws Exception {
final Statement statement = connection.createStatement()) {
statement.execute(
- String.format("load \"%s\" sglevel=2 onSuccess=none", file1.getAbsolutePath()));
+ String.format(
+ "load \"%s\" with ('database-level'='2', 'on-success'='none')",
+ file1.getAbsolutePath()));
try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
@@ -560,7 +567,9 @@ public void testLoadWithOnSuccess() throws Exception {
final Statement statement = connection.createStatement()) {
statement.execute(
- String.format("load \"%s\" sglevel=2 onSuccess=delete", file2.getAbsolutePath()));
+ String.format(
+ "load \"%s\" with ('database-level'='2', 'on-success'='delete')",
+ file2.getAbsolutePath()));
try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
@@ -760,7 +769,6 @@ public void testLoadWithMods() throws Exception {
generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true);
generator.generateDeletion(SchemaConfig.DEVICE_0, 10);
- generator.generateDeletion(SchemaConfig.DEVICE_1, 10);
writtenPoint1 = generator.getTotalNumber();
}
@@ -774,9 +782,11 @@ public void testLoadWithMods() throws Exception {
generator.registerTimeseries(
SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4,
+ new ArrayList<>(Arrays.asList(SchemaConfig.MEASUREMENT_30, SchemaConfig.MEASUREMENT_40)));
generator.generateData(SchemaConfig.DEVICE_2, 100, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 100, PARTITION_INTERVAL / 10_000, false);
+ generator.generateDeletion(SchemaConfig.DEVICE_3);
generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true);
generator.generateDeletion(SchemaConfig.DEVICE_2, 2);
generator.generateDeletion(SchemaConfig.DEVICE_4, 2);
@@ -784,6 +794,7 @@ public void testLoadWithMods() throws Exception {
generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true);
generator.generateDeletion(SchemaConfig.DEVICE_2, 2);
generator.generateDeletion(SchemaConfig.DEVICE_4, 2);
+ generator.generateDeletion(SchemaConfig.DEVICE_4, SchemaConfig.MEASUREMENT_30);
writtenPoint2 = generator.getTotalNumber();
}
@@ -803,6 +814,10 @@ public void testLoadWithMods() throws Exception {
Assert.fail("This ResultSet is empty.");
}
}
+
+ TestUtils.assertSingleResultSetEqual(
+ TestUtils.executeQueryWithRetry(statement, "count timeSeries"),
+ Collections.singletonMap("count(timeseries)", "18"));
}
}
@@ -894,6 +909,75 @@ public void testLoadLocally() throws Exception {
}
}
+ @Test
+ @Ignore("Load with conversion is currently banned")
+ public void testLoadWithConvertOnTypeMismatch() throws Exception {
+
+ List> measurementSchemas =
+ generateMeasurementSchemasForDataTypeConvertion();
+
+ final File file = new File(tmpDir, "1-0-0-0.tsfile");
+
+ long writtenPoint = 0;
+ List schemaList1 =
+ measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList());
+ List schemaList2 =
+ measurementSchemas.stream().map(pair -> pair.right).collect(Collectors.toList());
+
+ try (final TsFileGenerator generator = new TsFileGenerator(file)) {
+ generator.registerTimeseries(SchemaConfig.DEVICE_0, schemaList2);
+
+ generator.generateData(SchemaConfig.DEVICE_0, 100, PARTITION_INTERVAL / 10_000, false);
+
+ writtenPoint = generator.getTotalNumber();
+ }
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ for (MeasurementSchema schema : schemaList1) {
+ statement.execute(convert2SQL(SchemaConfig.DEVICE_0, schema));
+ }
+
+ statement.execute(String.format("load \"%s\" ", file.getAbsolutePath()));
+
+ try (final ResultSet resultSet =
+ statement.executeQuery("select count(*) from root.** group by level=1,2")) {
+ if (resultSet.next()) {
+ final long sgCount = resultSet.getLong("count(root.sg.test_0.*.*)");
+ Assert.assertEquals(writtenPoint, sgCount);
+ } else {
+ Assert.fail("This ResultSet is empty.");
+ }
+ }
+ }
+ }
+
+ private List>
+ generateMeasurementSchemasForDataTypeConvertion() {
+ TSDataType[] dataTypes = {
+ TSDataType.STRING,
+ TSDataType.TEXT,
+ TSDataType.BLOB,
+ TSDataType.TIMESTAMP,
+ TSDataType.BOOLEAN,
+ TSDataType.DATE,
+ TSDataType.DOUBLE,
+ TSDataType.FLOAT,
+ TSDataType.INT32,
+ TSDataType.INT64
+ };
+ List> pairs = new ArrayList<>();
+
+ for (TSDataType type : dataTypes) {
+ for (TSDataType dataType : dataTypes) {
+ String id = String.format("%s2%s", type.name(), dataType.name());
+ pairs.add(new Pair<>(new MeasurementSchema(id, type), new MeasurementSchema(id, dataType)));
+ }
+ }
+ return pairs;
+ }
+
private static class SchemaConfig {
private static final String STORAGE_GROUP_0 = "root.sg.test_0";
private static final String STORAGE_GROUP_1 = "root.sg.test_1";
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
index 074da6873eea0..182167be41c9f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBPartialInsertionIT.java
@@ -108,7 +108,7 @@ public void testPartialInsertionRestart() throws SQLException {
EnvironmentUtils.restartDaemon();
StorageEngine.getInstance().recover();
// wait for recover
- while (!StorageEngine.getInstance().isAllSgReady()) {
+ while (!StorageEngine.getInstance().isReadyForReadAndWrite()) {
Thread.sleep(500);
time += 500;
if (time > 10000) {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
index 069036e10a534..f423b5928ee3f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestartIT.java
@@ -315,8 +315,6 @@ public void testRecoverWALDeleteSchema() throws Exception {
@Test
public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- int avgSeriesPointNumberThreshold = config.getAvgSeriesPointNumberThreshold();
- config.setAvgSeriesPointNumberThreshold(2);
long tsFileSize = config.getSeqTsFileSize();
long unFsFileSize = config.getSeqTsFileSize();
config.setSeqTsFileSize(10000000);
@@ -327,6 +325,7 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception {
statement.execute("create timeseries root.turbine1.d1.s1 with datatype=INT64");
statement.execute("insert into root.turbine1.d1(timestamp,s1) values(1,1)");
statement.execute("insert into root.turbine1.d1(timestamp,s1) values(2,1)");
+ statement.execute("flush");
statement.execute("create timeseries root.turbine1.d1.s2 with datatype=BOOLEAN");
statement.execute("insert into root.turbine1.d1(timestamp,s2) values(3,true)");
statement.execute("insert into root.turbine1.d1(timestamp,s2) values(4,true)");
@@ -350,7 +349,6 @@ public void testRecoverWALDeleteSchemaCheckResourceTime() throws Exception {
assertEquals(2, cnt);
}
- config.setAvgSeriesPointNumberThreshold(avgSeriesPointNumberThreshold);
config.setSeqTsFileSize(tsFileSize);
config.setUnSeqTsFileSize(unFsFileSize);
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java
index f6e9f98726b5b..09a3a29511276 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.it;
import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.node.AbstractNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -38,10 +39,16 @@
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
+import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class})
public class IoTDBSetConfigurationIT {
@@ -55,6 +62,42 @@ public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Test
+ public void testSetConfigurationWithUndefinedConfigKey() {
+ String expectedExceptionMsg =
+ "301: ignored config items: [a] because they are immutable or undefined.";
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ executeAndExpectException(
+ statement, "set configuration \"a\"=\"false\"", expectedExceptionMsg);
+ int configNodeNum = EnvFactory.getEnv().getConfigNodeWrapperList().size();
+ int dataNodeNum = EnvFactory.getEnv().getDataNodeWrapperList().size();
+
+ for (int i = 0; i < configNodeNum; i++) {
+ executeAndExpectException(
+ statement, "set configuration \"a\"=\"false\" on " + i, expectedExceptionMsg);
+ }
+ for (int i = 0; i < dataNodeNum; i++) {
+ int dnId = configNodeNum + i;
+ executeAndExpectException(
+ statement, "set configuration \"a\"=\"false\" on " + dnId, expectedExceptionMsg);
+ }
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ private void executeAndExpectException(
+ Statement statement, String sql, String expectedContentInExceptionMsg) {
+ try {
+ statement.execute(sql);
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(expectedContentInExceptionMsg));
+ return;
+ }
+ Assert.fail();
+ }
+
@Test
public void testSetConfiguration() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -101,7 +144,8 @@ public void testSetClusterName() throws Exception {
EnvFactory.getEnv().getDataNodeWrapper(0).start();
// set cluster name on datanode
Awaitility.await()
- .atMost(10, TimeUnit.SECONDS)
+ .atMost(30, TimeUnit.SECONDS)
+ .pollDelay(1, TimeUnit.SECONDS)
.until(
() -> {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -118,24 +162,110 @@ public void testSetClusterName() throws Exception {
Awaitility.await()
.atMost(10, TimeUnit.SECONDS)
.until(() -> !EnvFactory.getEnv().getDataNodeWrapper(0).isAlive());
+ AbstractNodeWrapper datanode = EnvFactory.getEnv().getDataNodeWrapper(0);
Assert.assertTrue(
checkConfigFileContains(EnvFactory.getEnv().getDataNodeWrapper(0), "cluster_name=yy"));
+
+ // Modify the config file manually because the datanode can not restart
+ Properties properties = new Properties();
+ properties.put("cluster_name", "xx");
+ ConfigurationFileUtils.updateConfiguration(getConfigFile(datanode), properties, null);
+ EnvFactory.getEnv().getDataNodeWrapper(0).stop();
+ EnvFactory.getEnv().getDataNodeWrapper(0).start();
+ // wait the datanode restart successfully (won't do any meaningful modification)
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .pollDelay(1, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("set configuration \"cluster_name\"=\"xx\" on 1");
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ });
}
private static boolean checkConfigFileContains(
AbstractNodeWrapper nodeWrapper, String... contents) {
try {
- String systemPropertiesPath =
- nodeWrapper.getNodePath()
- + File.separator
- + "conf"
- + File.separator
- + CommonConfig.SYSTEM_CONFIG_NAME;
- File f = new File(systemPropertiesPath);
- String fileContent = new String(Files.readAllBytes(f.toPath()));
+ String fileContent = new String(Files.readAllBytes(getConfigFile(nodeWrapper).toPath()));
return Arrays.stream(contents).allMatch(fileContent::contains);
} catch (IOException ignore) {
return false;
}
}
+
+ private static File getConfigFile(AbstractNodeWrapper nodeWrapper) {
+ String systemPropertiesPath =
+ nodeWrapper.getNodePath()
+ + File.separator
+ + "conf"
+ + File.separator
+ + CommonConfig.SYSTEM_CONFIG_NAME;
+ return new File(systemPropertiesPath);
+ }
+
+ @Test
+ public void testSetDefaultSGLevel() throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ // legal value
+ statement.execute("set configuration \"default_storage_group_level\"=\"3\"");
+ statement.execute("INSERT INTO root.a.b.c.d1(timestamp, s1) VALUES (1, 1)");
+ ResultSet databases = statement.executeQuery("show databases");
+ databases.next();
+ Assert.assertEquals("root.a.b.c", databases.getString(1));
+ assertFalse(databases.next());
+
+ // path too short
+ try {
+ statement.execute("INSERT INTO root.fail(timestamp, s1) VALUES (1, 1)");
+ } catch (SQLException e) {
+ assertEquals(
+ "509: An error occurred when executing getDeviceToDatabase():root.fail is not a legal path, because it is no longer than default sg level: 3",
+ e.getMessage());
+ }
+
+ // illegal value
+ try {
+ statement.execute("set configuration \"default_storage_group_level\"=\"-1\"");
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("Illegal defaultStorageGroupLevel: -1, should >= 1"));
+ }
+
+ // Failed updates will not change the files.
+ assertFalse(
+ checkConfigFileContains(
+ EnvFactory.getEnv().getDataNodeWrapper(0), "default_storage_group_level=-1"));
+ assertTrue(
+ checkConfigFileContains(
+ EnvFactory.getEnv().getDataNodeWrapper(0), "default_storage_group_level=3"));
+ }
+
+ // can start with an illegal value
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultStorageGroupLevel(-1);
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("INSERT INTO root.a.b.c.d1(timestamp, s1) VALUES (1, 1)");
+ ResultSet databases = statement.executeQuery("show databases");
+ databases.next();
+ // the default value should take effect
+ Assert.assertEquals("root.a", databases.getString(1));
+ assertFalse(databases.next());
+
+ // create timeseries with an illegal path
+ try {
+ statement.execute("CREATE TIMESERIES root.db1.s3 WITH datatype=INT32");
+ } catch (SQLException e) {
+ assertEquals(
+ "509: An error occurred when executing getDeviceToDatabase():root.db1 is not a legal path, because it is no longer than default sg level: 3",
+ e.getMessage());
+ }
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
index 19b487193834b..441b3afbbcb05 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
@@ -1188,4 +1188,42 @@ public void testNewDataType() {
fail();
}
}
+
+ @Test
+ public void testIllegalDateType() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("CREATE DATABASE root.sg1");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d1.s4 WITH DATATYPE=DATE, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+ statement.execute(
+ "CREATE TIMESERIES root.sg1.d1.s5 WITH DATATYPE=TIMESTAMP, ENCODING=PLAIN, COMPRESSOR=SNAPPY");
+ try {
+ statement.execute("insert into root.sg1.d1(timestamp, s4) values(1, '2022-04-31')");
+ fail();
+ } catch (Exception e) {
+ assertEquals(
+ TSStatusCode.METADATA_ERROR.getStatusCode()
+ + ": Fail to insert measurements [s4] caused by [data type is not consistent, "
+ + "input '2022-04-31', registered DATE because Invalid date format. "
+ + "Please use YYYY-MM-DD format.]",
+ e.getMessage());
+ }
+ try {
+ statement.execute(
+ "insert into root.sg1.d1(timestamp, s5) values(1999-04-31T00:00:00.000+08:00, 1999-04-31T00:00:00.000+08:00)");
+ fail();
+ } catch (Exception e) {
+ assertEquals(
+ TSStatusCode.SEMANTIC_ERROR.getStatusCode()
+ + ": Input time format 1999-04-31T00:00:00.000+08:00 error. "
+ + "Input like yyyy-MM-dd HH:mm:ss, yyyy-MM-ddTHH:mm:ss "
+ + "or refer to user document for more info.",
+ e.getMessage());
+ }
+ } catch (SQLException e) {
+ fail();
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java
index 95204c783ec21..06ed9dd4397da 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSyntaxConventionStringLiteralIT.java
@@ -255,8 +255,7 @@ public void testIllegalFilePath() {
String errorMsg1 =
TSStatusCode.SQL_PARSE_ERROR.getStatusCode()
- + ": Error occurred while parsing SQL to physical plan: "
- + "line 1:7 mismatched input 'path' expecting STRING_LITERAL";
+ + ": Error occurred while parsing SQL to physical plan: line 1:7 no viable alternative at input 'REMOVE path'";
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("REMOVE path");
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java
index bc82992065bbd..7e1bf625bf82d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBCountIfIT.java
@@ -208,6 +208,19 @@ public void testContIfWithoutTransform() {
resultSetEqualTest("select Count_if(s3, 1) from root.db.d1", expectedHeader, retArray);
}
+ @Test
+ public void testMultiAttributes() {
+ String[] expectedHeader =
+ new String[] {
+ "Count_if(root.db.d1.s3, 1, \"attr1\"=\"1\", \"attr2\"=\"2\", \"attr3\"=\"3\")"
+ };
+ String[] retArray = new String[] {"1,"};
+ resultSetEqualTest(
+ "select Count_if(s3, 1, \"attr1\"=\"1\",\"attr2\"=\"2\",\"attr3\"=\"3\") from root.db.d1",
+ expectedHeader,
+ retArray);
+ }
+
@Test
public void testContIfWithGroupByLevel() {
String[] expectedHeader = new String[] {"Count_if(root.db.*.s1 = 0 & root.db.*.s2 = 0, 3)"};
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
index cc5fdeb5b27fe..1bb3e76061dab 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java
@@ -100,6 +100,7 @@ public class IoTDBAlignByDeviceIT {
"insert into root.vehicle.d1(timestamp,s0) values(1,999)",
"insert into root.vehicle.d1(timestamp,s0) values(1000,888)",
"insert into root.other.d1(timestamp,s0) values(2, 3.14)",
+ "insert into root.other.d2(timestamp,s6) values(6, 6.66)",
};
@BeforeClass
@@ -1184,4 +1185,48 @@ public void removeDeviceWhereMeasurementWhenNoDeviceSelectTest() {
+ e.getMessage());
}
}
+
+ @Test
+ public void nonExistMeasurementInHavingTest() {
+ String[] retArray =
+ new String[] {
+ "1,root.other.d1,3.14,null,", "5,root.other.d2,null,6.66,",
+ };
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "select last_value(s0),last_value(s6) from root.other.** group by ([1,10),2ms) having last_value(s0) is not null or last_value(s6) is not null align by device")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ List actualIndexToExpectedIndexList =
+ checkHeader(
+ resultSetMetaData,
+ "Time,Device,last_value(s0),last_value(s6)",
+ new int[] {
+ Types.TIMESTAMP, Types.VARCHAR, Types.FLOAT, Types.DOUBLE,
+ });
+
+ int cnt = 0;
+ while (resultSet.next()) {
+ String[] expectedStrings = retArray[cnt].split(",");
+ StringBuilder expectedBuilder = new StringBuilder();
+ StringBuilder actualBuilder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ actualBuilder.append(resultSet.getString(i)).append(",");
+ expectedBuilder
+ .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)])
+ .append(",");
+ }
+ Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString());
+ cnt++;
+ }
+ Assert.assertEquals(retArray.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java
index 05e2104b904d2..a71cb77371359 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java
@@ -59,6 +59,9 @@ public class IoTDBAlignByDeviceWithTemplateIT {
"INSERT INTO root.sg2.d4(timestamp,s1,s2,s3) values(1,1111.1,true,1111), (5,5555.5,false,5555);",
};
+ String[] expectedHeader;
+ String[] retArray;
+
@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().initClusterEnvironment();
@@ -70,11 +73,32 @@ public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Test
+ public void singleDeviceTest() {
+ expectedHeader = new String[] {"Time,Device,s3,s1,s2"};
+ retArray =
+ new String[] {
+ "1,root.sg1.d1,1,1.1,false,",
+ };
+ resultSetEqualTest(
+ "SELECT * FROM root.sg1.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;",
+ expectedHeader,
+ retArray);
+ retArray =
+ new String[] {
+ "1,root.sg2.d1,1,1.1,false,",
+ };
+ resultSetEqualTest(
+ "SELECT * FROM root.sg2.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;",
+ expectedHeader,
+ retArray);
+ }
+
@Test
public void selectWildcardNoFilterTest() {
// 1. order by device
- String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"};
- String[] retArray =
+ expectedHeader = new String[] {"Time,Device,s3,s1,s2"};
+ retArray =
new String[] {
"1,root.sg1.d1,1,1.1,false,",
"2,root.sg1.d1,2,2.2,false,",
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java
index a595c5a96cbee..60cf1a5d5f0c1 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBOrderByLimitOffsetAlignByDeviceIT.java
@@ -53,11 +53,30 @@ public static void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ String[] expectedHeader;
+ String[] retArray;
+
+ @Test
+ public void singleDeviceTest() {
+ expectedHeader = new String[] {"Time,Device,precipitation"};
+ retArray = new String[] {"1668960000200,root.weather.London,1667492178318,"};
+ resultSetEqualTest(
+ "select precipitation from root.weather.London where precipitation>1667492178118 order by time offset 1 limit 1 align by device",
+ expectedHeader,
+ retArray);
+
+ retArray = new String[] {"1668960000200,root.weather.London,1667492178318,"};
+ resultSetEqualTest(
+ "select precipitation from root.weather.London where precipitation>1667492178118 order by precipitation offset 1 limit 1 align by device",
+ expectedHeader,
+ retArray);
+ }
+
@Test
public void orderByCanNotPushLimitTest() {
// 1. value filter, can not push down LIMIT
- String[] expectedHeader = new String[] {"Time,Device,s1"};
- String[] retArray = new String[] {"3,root.db.d1,111,"};
+ expectedHeader = new String[] {"Time,Device,s1"};
+ retArray = new String[] {"3,root.db.d1,111,"};
resultSetEqualTest(
"SELECT * FROM root.db.** WHERE s1>40 ORDER BY TIME LIMIT 1 ALIGN BY DEVICE;",
expectedHeader,
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java
index 1a65682fc5b83..f9639d3ea2d2c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java
@@ -74,6 +74,15 @@ public void testAlignedRawDataAlignByTime1() {
resultSetEqualTest(
"select s2, s3 from root.sg1.d1 where s2 - 1 >= 9 and s2 < 30", expectedHeader1, retArray1);
+ resultSetEqualTest(
+ "select s2, s3 from root.sg1.d1 where 9 <= s2 - 1 and 30 > s2", expectedHeader1, retArray1);
+
+ retArray1 = new String[] {"20,20,20,"};
+ resultSetEqualTest(
+ "select s2, s3 from root.sg1.d1 where 9 <= s2 - 1 and 30 > s2 and 19 < time",
+ expectedHeader1,
+ retArray1);
+
String expectedHeader2 = "Time,root.sg1.d1.s3,";
String[] retArray2 =
new String[] {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java
index 93165a16ff6a6..18a2586bde36b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java
@@ -25,7 +25,10 @@
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.jdbc.IoTDBSQLException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import com.google.common.collect.ImmutableList;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -45,6 +48,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.Callable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -1244,4 +1248,83 @@ public void testCreateRoleIdentifierName() throws SQLException {
adminStmt.execute("create role tail");
adminStmt.execute("create user tail 'password'");
}
+
+ @Test
+ public void testClusterManagementSqlOfTreeModel() throws Exception {
+ ImmutableList clusterManagementSQLList =
+ ImmutableList.of(
+ // show cluster, nodes, regions,
+ "show ainodes",
+ "show confignodes",
+ "show datanodes",
+ "show cluster",
+ "show clusterid",
+ "show regions",
+ "show data regionid where database=root.**",
+
+ // remove node
+ "remove datanode 0",
+ "remove confignode 0",
+
+ // region operation
+ "migrate region 0 from 1 to 2",
+ "reconstruct region 0 on 1",
+ "extend region 0 to 1",
+ "remove region 0 from 1",
+
+ // others
+ "show timeslotid where database=root.test",
+ "count timeslotid where database=root.test",
+ "show data seriesslotid where database=root.test",
+ "verify connection");
+
+ try (Connection adminCon = EnvFactory.getEnv().getConnection();
+ Statement adminStmt = adminCon.createStatement()) {
+ adminStmt.execute("CREATE USER Jack 'temppw'");
+
+ try (Connection JackConnection = EnvFactory.getEnv().getConnection("Jack", "temppw");
+ Statement Jack = JackConnection.createStatement()) {
+ testClusterManagementSqlImpl(
+ clusterManagementSQLList,
+ () -> adminStmt.execute("GRANT MAINTAIN ON root.** TO USER Jack"),
+ Jack);
+ }
+ }
+ }
+
+ private void testClusterManagementSqlImpl(
+ List clusterManagementSqlList, Callable giveJackAuthority, Statement Jack)
+ throws Exception {
+ // Jack has no authority to execute these SQLs
+ for (String sql : clusterManagementSqlList) {
+ try {
+ Jack.execute(sql);
+ } catch (IoTDBSQLException e) {
+ if (TSStatusCode.NO_PERMISSION.getStatusCode() != e.getErrorCode()) {
+ fail(
+ String.format(
+ "SQL should fail because of no permission, but the error code is %d: %s",
+ e.getErrorCode(), sql));
+ }
+ continue;
+ }
+ fail(String.format("SQL should fail because of no permission: %s", sql));
+ }
+
+ // Give Jack authority
+ giveJackAuthority.call();
+
+ // Jack is able to execute these SQLs now
+ for (String sql : clusterManagementSqlList) {
+ try {
+ // No exception is fine
+ Jack.execute(sql);
+ } catch (IoTDBSQLException e) {
+ // If there is an exception, error code must not be NO_PERMISSION
+ if (TSStatusCode.NO_PERMISSION.getStatusCode() == e.getErrorCode()) {
+ fail(String.format("SQL should not fail with no permission: %s", sql));
+ }
+ }
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java
index 4b3cebd68f1b3..e8dd4e5f650ff 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java
@@ -247,7 +247,5 @@ public void adminOperationsTest() {
"803: Only the admin user can perform this operation",
"test",
"test123");
- assertTestFail(
- "show regions", "803: Only the admin user can perform this operation", "test", "test123");
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
index e08eb4c5f1d0a..9d56acf518fbf 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthIT.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.it.groupby;
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -49,6 +51,7 @@
import static org.apache.iotdb.db.utils.constant.TestConstant.sum;
import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
import static org.apache.iotdb.itbase.constant.TestConstant.count;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@@ -75,6 +78,8 @@ public class IoTDBGroupByNaturalMonthIT {
calendar.add(Calendar.MONTH, 1), i = calendar.getTimeInMillis()) {
dataSet.add("insert into root.test.d1(timestamp, s1) values (" + i + ", 1)");
}
+
+ dataSet.add("insert into root.testTimeZone.d1(timestamp, s1) values (1, 1)");
}
protected static final DateFormat df = new SimpleDateFormat("MM/dd/yyyy:HH:mm:ss");
@@ -413,4 +418,27 @@ public void groupByNaturalMonthWithMixedUnit2() {
null,
currPrecision);
}
+
+ @Test
+ public void groupByNaturalMonthWithNonSystemDefaultTimeZone() {
+ try (ISession session =
+ EnvFactory.getEnv().getSessionConnection(TimeZone.getTimeZone("UTC+09:00").toZoneId())) {
+
+ SessionDataSet sessionDataSet =
+ session.executeQueryStatement(
+ "select count(s1) from root.testTimeZone.d1 group by([2024-07-01, 2024-08-01), 1mo)");
+
+ int count = 0;
+ while (sessionDataSet.hasNext()) {
+ sessionDataSet.next();
+ count++;
+ }
+ assertEquals(1, count);
+
+ sessionDataSet.closeOperationHandle();
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java
index 111217e56857f..a15991c487683 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java
@@ -110,7 +110,7 @@ public void test2() {
.getConfig()
.getCommonConfig()
.setMaxNumberOfPointsInPage(4)
- .setAvgSeriesPointNumberThreshold(2);
+ .setTargetChunkPointNum(2);
EnvFactory.getEnv().initClusterEnvironment();
String[] expectedHeader = new String[] {TIMESTAMP_STR, count("root.sg2.d1.s1")};
String[] retArray = new String[] {"5,1,", "10,1,", "15,2,", "20,0,", "25,1,"};
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java
index 6d30db0008564..feb710be4fa5c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryLastCacheIT.java
@@ -165,4 +165,18 @@ public void cacheHitTest() {
testLastQueryOrderByTimeDesc();
testLastQuery1();
}
+
+ @Test
+ public void testLastQuerySortWithLimit() {
+ String[] expectedHeader =
+ new String[] {TIMESTAMP_STR, TIMESEIRES_STR, VALUE_STR, DATA_TYPE_STR};
+ String[] retArray =
+ new String[] {
+ "1679477545000,root.ln_1.tb_6141.code_DOUBLE,2.0,DOUBLE,",
+ };
+ resultSetEqualTest(
+ "select last * from root.ln_1.tb_6141 order by time desc, timeseries desc limit 1;",
+ expectedHeader,
+ retArray);
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java
index 062d5a345c3e8..0de811ff271ad 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryWithLimitOffsetIT.java
@@ -190,4 +190,38 @@ public void testWithSLimitOrSOffset() {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testWithSortLimit() {
+ String[] retArray =
+ new String[] {
+ "2,root.sg.d2.s2,1.0,DOUBLE",
+ };
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "select last * from root.sg.** order by time desc, timeseries desc limit 1")) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(ColumnHeaderConstant.TIME)
+ + ","
+ + resultSet.getString(ColumnHeaderConstant.TIMESERIES)
+ + ","
+ + resultSet.getString(ColumnHeaderConstant.VALUE)
+ + ","
+ + resultSet.getString(ColumnHeaderConstant.DATATYPE);
+ assertEquals(retArray[cnt++], ans);
+ }
+ assertEquals(retArray.length, cnt);
+ }
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java
index 76e6ddce8170b..3e6f660d4f2e9 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/metric/IoTDBMetricIT.java
@@ -23,16 +23,19 @@
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.metrics.reporter.prometheus.PrometheusReporter;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
@@ -65,7 +68,13 @@ public class IoTDBMetricIT {
private static final String VALID_LOG_STRING =
"This line {} is invalid in prometheus line protocol";
- public static boolean isValidPrometheusTextFormat(String metrics) {
+ private static final String TEST_USERNAME = "good";
+ private static final String TEST_PASSWORD = "??";
+
+ private static final String WRONG_USERNAME = "bad";
+ private static final String WRONG_PASSWORD = "!!";
+
+ private static boolean isValidPrometheusTextFormat(String metrics) {
String[] lines = metrics.split("\\n");
boolean valid = true;
@@ -107,8 +116,8 @@ private static boolean isValidTypeLine(String line) {
return Pattern.matches(TYPE_REGEX, line.trim());
}
- @BeforeClass
- public static void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
// Start ConfigNode with Prometheus reporter up
EnvFactory.getEnv()
.getConfig()
@@ -119,21 +128,86 @@ public static void setUp() throws Exception {
.getConfig()
.getDataNodeConfig()
.setMetricReporterType(Collections.singletonList("PROMETHEUS"));
- EnvFactory.getEnv().initClusterEnvironment();
}
- @AfterClass
- public static void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
EnvFactory.getEnv().cleanClusterEnvironment();
}
+ @Test
+ public void testPrometheusReporterWithoutAuth() {
+ EnvFactory.getEnv().initClusterEnvironment();
+
+ List metricContents = EnvFactory.getEnv().getMetricPrometheusReporterContents(null);
+ for (String metricContent : metricContents) {
+ Assert.assertNotNull(metricContent);
+ Assert.assertNotEquals(0, metricContent.length());
+ Assert.assertTrue(isValidPrometheusTextFormat(metricContent));
+ }
+ }
+
@Test
public void testPrometheusReporter() {
- List metricContents = EnvFactory.getEnv().getMetricPrometheusReporterContents();
+ EnvFactory.getEnv()
+ .getConfig()
+ .getConfigNodeConfig()
+ .setMetricPrometheusReporterUsername(base64Encode(TEST_USERNAME))
+ .setMetricPrometheusReporterPassword(base64Encode(TEST_PASSWORD));
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricPrometheusReporterUsername(base64Encode(TEST_USERNAME))
+ .setMetricPrometheusReporterPassword(base64Encode(TEST_PASSWORD));
+ EnvFactory.getEnv().initClusterEnvironment();
+
+ wrongUsernameTest();
+ wrongPasswordTest();
+ correctUsernameAndPasswordTest();
+ }
+
+ private void wrongUsernameTest() {
+ List metricContents =
+ EnvFactory.getEnv()
+ .getMetricPrometheusReporterContents(
+ buildPrometheusReporterAuthHeader(WRONG_USERNAME, TEST_PASSWORD));
+ for (String metricContent : metricContents) {
+ Assert.assertNull(metricContent);
+ }
+ }
+
+ private void wrongPasswordTest() {
+ List metricContents =
+ EnvFactory.getEnv()
+ .getMetricPrometheusReporterContents(
+ buildPrometheusReporterAuthHeader(TEST_USERNAME, WRONG_PASSWORD));
+ for (String metricContent : metricContents) {
+ Assert.assertNull(metricContent);
+ }
+ }
+
+ private void correctUsernameAndPasswordTest() {
+ List metricContents =
+ EnvFactory.getEnv()
+ .getMetricPrometheusReporterContents(
+ buildPrometheusReporterAuthHeader(TEST_USERNAME, TEST_PASSWORD));
for (String metricContent : metricContents) {
Assert.assertNotNull(metricContent);
Assert.assertNotEquals(0, metricContent.length());
Assert.assertTrue(isValidPrometheusTextFormat(metricContent));
}
}
+
+ private String buildPrometheusReporterAuthHeader(String username, String password) {
+ if (username == null || username.isEmpty()) {
+ return null;
+ }
+ String raw = username + PrometheusReporter.DIVIDER_BETWEEN_USERNAME_AND_DIVIDER + password;
+ String base64 = Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
+ return PrometheusReporter.BASIC_AUTH_PREFIX + base64;
+ }
+
+ private static String base64Encode(String raw) {
+ return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderBy2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderBy2IT.java
index 131ea8bf3b5c4..51bfdf3dd2a44 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderBy2IT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderBy2IT.java
@@ -37,6 +37,10 @@ public class IoTDBOrderBy2IT extends IoTDBOrderByIT {
public static void setUp() throws Exception {
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(2048);
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setMaxTsBlockSizeInByte(200);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeCommonConfig()
+ .setQueryMemoryProportion("1:100:200:50:400:200:200:50");
EnvFactory.getEnv().initClusterEnvironment();
insertData();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java
index c6f502bd4b54f..2f5807388d130 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByForDebugIT.java
@@ -104,6 +104,10 @@ public class IoTDBOrderByForDebugIT {
@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeCommonConfig()
+ .setQueryMemoryProportion("1:100:200:50:400:200:200:50");
EnvFactory.getEnv().initClusterEnvironment();
insertData();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java
index 6a65ab1ed7ee2..d1f9d5a2c25a9 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java
@@ -38,6 +38,7 @@
import java.sql.Statement;
import java.util.Objects;
+import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
@@ -97,9 +98,19 @@ public class IoTDBOrderByIT {
"insert into root.sg.d2(timestamp,num,bigNum,floatNum,str,bool) values(51536000000,15,3147483648,235.213,\"watermelon\",TRUE)"
};
+ private static final String[] sql3 =
+ new String[] {
+ "create aligned timeseries root.test.dev (v_timestamp TIMESTAMP, v_string STRING, v_date DATE, v_blob BLOB encoding=PLAIN, v_int32 INT32);",
+ "insert into root.test.dev(timestamp, v_timestamp, v_string, v_date, v_blob, v_int32) aligned values(1, 2024-09-20T06:15:35.000+00:00, 'e1', '2012-12-12', X'108DCD62', 1);"
+ };
+
@BeforeClass
public static void setUp() throws Exception {
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeCommonConfig()
+ .setQueryMemoryProportion("1:100:200:50:400:200:200:50");
EnvFactory.getEnv().initClusterEnvironment();
insertData();
}
@@ -112,12 +123,13 @@ public static void tearDown() throws Exception {
protected static void insertData() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- for (String sql : sql) {
- statement.execute(sql);
- }
- for (String sql : sql2) {
- statement.execute(sql);
+
+ for (String[] sqlList : java.util.Arrays.asList(sql, sql2, sql3)) {
+ for (String sql : sqlList) {
+ statement.execute(sql);
+ }
}
+
} catch (Exception e) {
e.printStackTrace();
}
@@ -192,6 +204,23 @@ private void testNormalOrderBy(String sql, int[] ans) {
}
}
+ @Test
+ public void newDataTypeTest() {
+ String[] expectedHeader =
+ new String[] {"Time,Device,v_int32,v_blob,v_date,v_timestamp,v_string"};
+ String[] retArray =
+ new String[] {
+ "1,root.test.dev,1,0x108dcd62,2012-12-12,1726812935000,e1,",
+ };
+
+ for (String key : new String[] {"time", "v_int32", "v_date", "v_timestamp", "v_string"}) {
+ resultSetEqualTest(
+ String.format("SELECT * FROM root.test.dev ORDER BY %s LIMIT 1 ALIGN BY DEVICE", key),
+ expectedHeader,
+ retArray);
+ }
+ }
+
// 1. One-level order by test
@Test
public void orderByTest1() {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java
index cb074e324a9a0..b9e3ca7b6dbea 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java
@@ -18,11 +18,13 @@
*/
package org.apache.iotdb.db.it.query;
+import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import com.google.common.collect.ImmutableList;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -41,6 +43,7 @@
import java.util.List;
import java.util.Map;
+import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@@ -661,4 +664,26 @@ public void RegexpNonExistTest() {
fail(e.getMessage());
}
}
+
+ @Test
+ public void selectWithTimeTest() {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeRawDataQuery(
+ ImmutableList.of("root.ln.wf01.wt01.time", "root.ln.wf01.wt01.temperature"), 0, 100);
+
+ fail();
+ } catch (Exception e) {
+ e.getMessage().contains("509: root.ln.wf01.wt01.time is not a legal path");
+ }
+
+ String expectedErrMsg =
+ "701: Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible";
+ assertTestFail("select time from root.ln.wf01.wt01", expectedErrMsg);
+ assertTestFail("select time, temperature from root.ln.wf01.wt01", expectedErrMsg);
+ assertTestFail("select time from root.ln.wf01.wt01 where temperature > 1", expectedErrMsg);
+ // parse error when process 'wt01.time'
+ assertTestFail(
+ "select wt01.time, wt01.temperature from root.ln.wf01",
+ "700: Error occurred while parsing SQL to physical plan");
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java
index 9bb22ea599a42..f1af45ffee883 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java
@@ -72,24 +72,24 @@ public void tearDown() throws Exception {
/** Test if creating a time series will cause the database with same name to disappear */
@Test
public void testCreateTimeseries() throws Exception {
- String storageGroup = "root.sg1.a.b.c";
+ String database = "root.sg1.a.b.c";
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE %s", storageGroup));
+ statement.execute(String.format("CREATE DATABASE %s", database));
statement.execute(
String.format(
"create timeseries %s with datatype=INT64, encoding=PLAIN, compression=SNAPPY",
- storageGroup));
+ database));
} catch (Exception ignored) {
}
// ensure that current database in cache is right.
- createTimeSeriesTool(storageGroup);
+ createTimeSeriesTool(database);
}
- private void createTimeSeriesTool(String storageGroup) throws SQLException {
+ private void createTimeSeriesTool(String database) throws SQLException {
Set resultList = new HashSet<>();
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement();
@@ -99,7 +99,7 @@ private void createTimeSeriesTool(String storageGroup) throws SQLException {
resultList.add(str);
}
}
- Assert.assertFalse(resultList.contains(storageGroup));
+ Assert.assertFalse(resultList.contains(database));
resultList.clear();
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement();
@@ -109,11 +109,16 @@ private void createTimeSeriesTool(String storageGroup) throws SQLException {
resultList.add(res);
}
}
- Assert.assertTrue(resultList.contains(storageGroup));
+ Assert.assertTrue(resultList.contains(database));
}
@Test
public void testCreateTimeseriesWithSpecialCharacter() throws Exception {
+ // Currently this test may fail in PBTree
+ // Will solve this in the future
+ if (schemaTestMode == SchemaTestMode.PBTree) {
+ return;
+ }
try (Connection connection = EnvFactory.getEnv().getConnection()) {
try (Statement statement = connection.createStatement()) {
statement.execute(
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java
index 3defa886f8dbe..79198e97429ad 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java
@@ -246,4 +246,46 @@ public void deleteTimeSeriesMultiIntervalTest() {
fail(e.getMessage());
}
}
+
+ @Test
+ public void deleteTimeseriesAndCreateSameTypeTest2() throws Exception {
+ String[] retArray = new String[] {"1,4.0,", "2,8.0,"};
+ int cnt = 0;
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "create aligned timeseries root.turbine1.d1(s1 FLOAT encoding=PLAIN compression=SNAPPY, "
+ + "s2 INT64 encoding=PLAIN compression=SNAPPY, s4 DOUBLE encoding=PLAIN compression=SNAPPY)");
+ statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1,s2,s4) ALIGNED VALUES(1,1,2,4)");
+
+ try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ while (resultSet.next()) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ builder.append(resultSet.getString(i)).append(",");
+ }
+ Assert.assertEquals(retArray[cnt], builder.toString());
+ cnt++;
+ }
+ }
+ // delete series in the middle
+ statement.execute("DELETE timeseries root.turbine1.d1.s4");
+ statement.execute(
+ "INSERT INTO root.turbine1.d1(timestamp,s3,s4) ALIGNED VALUES(2,false,8.0)");
+ statement.execute("FLUSH");
+
+ try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ while (resultSet.next()) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ builder.append(resultSet.getString(i)).append(",");
+ }
+ Assert.assertEquals(retArray[cnt], builder.toString());
+ cnt++;
+ }
+ }
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
index 9945843b09185..f3d53c3a46187 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
@@ -142,10 +142,15 @@ public void testAutoExtendTemplate() throws SQLException {
statement.execute("SET DEVICE TEMPLATE t1 to root.db");
+ // single-row insertion
statement.execute("INSERT INTO root.db.d1(time, s1, s3) values(1, 1, 1)");
statement.execute("INSERT INTO root.db.d2(time, s4, s5) values(1, 1, 1)");
statement.execute("INSERT INTO root.db1.d1(time, s2, s3) values(1, 1, 1)");
+ // multi-row insertion with null
+ statement.execute(
+ "INSERT INTO root.db.d1(time, s1, s6) values(1, 1, 1), (2, 2, null), (3, 3, 3)");
+
String[] sqls =
new String[] {
"show timeseries",
@@ -159,11 +164,13 @@ public void testAutoExtendTemplate() throws SQLException {
"root.db.d1.s3,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db.d1.s4,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db.d1.s5,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
+ "root.db.d1.s6,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db.d2.s1,null,root.db,INT64,PLAIN,LZ4,null,null,null,null,BASE,",
"root.db.d2.s2,null,root.db,DOUBLE,RLE,LZ4,null,null,null,null,BASE,",
"root.db.d2.s3,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db.d2.s4,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db.d2.s5,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
+ "root.db.d2.s6,null,root.db,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db1.d1.s2,null,root.db1,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
"root.db1.d1.s3,null,root.db1,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,"))
};
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java
index 90c5ea657254c..3e013c898ba83 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java
@@ -785,4 +785,53 @@ public void testNewDataType() {
String expectedQueryHeader = "Time,root.db.d2.s7,root.db.d2.s8,root.db.d2.s9,root.db.d2.s10,";
resultSetEqualTest("select s7,s8,s9,s10 from root.db.d2;", expectedQueryHeader, resultSet);
}
+
+ // -------------------------------------- OTHER TEST -------------------------------------
+ @Test
+ public void testRemoveBackQuote() {
+ String[] intoRetArray =
+ new String[] {
+ "count(root.sg.d1.s1),root.sg_agg1.d1.count_s1,1,",
+ "last_value(root.sg.d1.s2),root.sg_agg1.d1.last_value_s2,1,",
+ "count(root.sg.d2.s1),root.sg_agg1.d2.count_s1,1,",
+ "last_value(root.sg.d2.s2),root.sg_agg1.d2.last_value_s2,1,"
+ };
+ resultSetEqualTest(
+ "select count(d1.s1), last_value(d1.s2), count(d2.s1), last_value(d2.s2) "
+ + "into root.sg_agg1.`d1`(`count_s1`, last_value_s2), aligned root.sg_agg1.d2(count_s1, last_value_s2) "
+ + "from root.sg;",
+ selectIntoHeader,
+ intoRetArray);
+
+ String expectedQueryHeader =
+ "Time,root.sg_agg1.d1.count_s1,root.sg_agg1.d2.count_s1,root.sg_agg1.d1.last_value_s2,root.sg_agg1.d2.last_value_s2,";
+ String[] queryRetArray = new String[] {"0,10,7,12.0,11.0,"};
+ resultSetEqualTest(
+ "select count_s1, last_value_s2 from root.sg_agg1.d1, root.sg_agg1.d2;",
+ expectedQueryHeader,
+ queryRetArray);
+ }
+
+ @Test
+ public void testRemoveBackQuoteAlignByDevice() {
+ String[] intoRetArray =
+ new String[] {
+ "root.sg.d1,count(s1),root.sg_abd_agg1.d1.count_s1,1,",
+ "root.sg.d1,last_value(s2),root.sg_abd_agg1.d1.last_value_s2,1,"
+ };
+ resultSetEqualTest(
+ "select count(s1), last_value(s2) "
+ + "into root.sg_abd_agg1.`d1`(`count_s1`, last_value_s2) "
+ + "from root.sg.d1 align by device;",
+ selectIntoAlignByDeviceHeader,
+ intoRetArray);
+
+ String expectedQueryHeader =
+ "Time,root.sg_abd_agg1.d1.count_s1," + "root.sg_abd_agg1.d1.last_value_s2,";
+ String[] queryRetArray = new String[] {"0,10,12.0,"};
+ resultSetEqualTest(
+ "select count_s1, last_value_s2 from root.sg_abd_agg1.d1, root.sg_abd_agg1.d2;",
+ expectedQueryHeader,
+ queryRetArray);
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
index cb21faa628d3f..a36a9c048aabf 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/trigger/IoTDBTriggerManagementIT.java
@@ -109,6 +109,7 @@ private static void createTimeSeries() {
"CREATE TIMESERIES root.test.stateful.b with datatype=INT32,encoding=PLAIN");
statement.execute(
"CREATE TIMESERIES root.test.stateful.c with datatype=INT32,encoding=PLAIN");
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
} catch (SQLException throwable) {
fail(throwable.getMessage());
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
index 75dc0c1f5f770..eff19645f63f8 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udaf/IoTDBUDAFManagementIT.java
@@ -235,6 +235,7 @@ public void createFunctionWithURITest() throws SQLException {
public void createFunctionWithInvalidURITest() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
try {
statement.execute(
String.format(
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
index 5de421ebd2a6d..71555bce0e3cd 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
@@ -152,4 +152,16 @@ public void testUDFSingleRowQuery() {
fail(throwable.getMessage());
}
}
+
+ @Test
+ public void testUntrustedUri() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE FUNCTION two_sum AS 'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'");
+ fail("should fail");
+ } catch (SQLException throwable) {
+ assertTrue(throwable.getMessage().contains("701: Untrusted uri "));
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
index 798256e5a16d6..261675a3c53dc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFManagementIT.java
@@ -262,6 +262,7 @@ public void testCreateFunctionWithURI() throws SQLException {
public void testCreateFunctionWithInvalidURI() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
+ statement.execute("set configuration \"trusted_uri_pattern\"='.*'");
try {
statement.execute(
String.format(
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
index 372c9200c5e6c..168914651eda5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
@@ -48,6 +48,7 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import static org.apache.iotdb.itbase.constant.TestConstant.DELTA;
import static org.apache.iotdb.itbase.constant.TestConstant.NULL;
@@ -694,6 +695,15 @@ public static void assertDataEventuallyOnEnv(
assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600);
}
+ public static void assertDataEventuallyOnEnv(
+ final BaseEnv env,
+ final String sql,
+ final String expectedHeader,
+ final Set expectedResSet,
+ final Consumer handleFailure) {
+ assertDataEventuallyOnEnv(env, sql, expectedHeader, expectedResSet, 600, handleFailure);
+ }
+
public static void assertDataEventuallyOnEnv(
BaseEnv env,
String sql,
@@ -723,6 +733,86 @@ public static void assertDataEventuallyOnEnv(
}
}
+ public static void assertDataEventuallyOnEnv(
+ final BaseEnv env,
+ final String sql,
+ final String expectedHeader,
+ final Set expectedResSet,
+ final long timeoutSeconds,
+ final Consumer handleFailure) {
+ try (Connection connection = env.getConnection();
+ Statement statement = connection.createStatement()) {
+ // Keep retrying if there are execution failures
+ await()
+ .pollInSameThread()
+ .pollDelay(1L, TimeUnit.SECONDS)
+ .pollInterval(1L, TimeUnit.SECONDS)
+ .atMost(timeoutSeconds, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ try {
+ TestUtils.assertResultSetEqual(
+ executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet);
+ } catch (Exception e) {
+ if (handleFailure != null) {
+ handleFailure.accept(e.getMessage());
+ }
+ Assert.fail();
+ } catch (Error e) {
+ if (handleFailure != null) {
+ handleFailure.accept(e.getMessage());
+ }
+ throw e;
+ }
+ });
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ public static void assertDataEventuallyOnEnv(
+ final BaseEnv env,
+ final DataNodeWrapper dataNodeWrapper,
+ final String sql,
+ final String expectedHeader,
+ final Set expectedResSet) {
+ assertDataEventuallyOnEnv(env, dataNodeWrapper, sql, expectedHeader, expectedResSet, 600);
+ }
+
+ public static void assertDataEventuallyOnEnv(
+ final BaseEnv env,
+ final DataNodeWrapper dataNodeWrapper,
+ final String sql,
+ final String expectedHeader,
+ final Set expectedResSet,
+ final long timeoutSeconds) {
+ try (Connection connection =
+ env.getConnection(
+ dataNodeWrapper, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD);
+ Statement statement = connection.createStatement()) {
+ // Keep retrying if there are execution failures
+ await()
+ .pollInSameThread()
+ .pollDelay(1L, TimeUnit.SECONDS)
+ .pollInterval(1L, TimeUnit.SECONDS)
+ .atMost(timeoutSeconds, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ try {
+ if (sql != null && !sql.isEmpty()) {
+ TestUtils.assertResultSetEqual(
+ executeQueryWithRetry(statement, sql), expectedHeader, expectedResSet);
+ }
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ });
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
public static void assertDataEventuallyOnEnv(
BaseEnv env, String sql, Map expectedHeaderWithResult) {
assertDataEventuallyOnEnv(env, sql, expectedHeaderWithResult, 600);
@@ -786,4 +876,27 @@ public static void assertDataAlwaysOnEnv(
fail();
}
}
+
+ public static void stopForciblyAndRestartDataNodes() {
+ EnvFactory.getEnv().shutdownForciblyAllDataNodes();
+ EnvFactory.getEnv().startAllDataNodes();
+ long waitStartMS = System.currentTimeMillis();
+ long maxWaitMS = 60_000L;
+ long retryIntervalMS = 1000;
+ while (true) {
+ try (Connection connection = EnvFactory.getEnv().getConnection()) {
+ break;
+ } catch (Exception e) {
+ try {
+ Thread.sleep(retryIntervalMS);
+ } catch (InterruptedException ex) {
+ break;
+ }
+ }
+ long waited = System.currentTimeMillis() - waitStartMS;
+ if (waited > maxWaitMS) {
+ fail("Timeout while waiting for datanodes restart");
+ }
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java b/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java
index 07e0451468836..17e20d91e9ac0 100644
--- a/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java
+++ b/integration-test/src/test/java/org/apache/iotdb/it/framework/IoTDBTestRunner.java
@@ -37,33 +37,30 @@ public class IoTDBTestRunner extends BlockJUnit4ClassRunner {
private static final Logger logger = IoTDBTestLogger.logger;
private IoTDBTestListener listener;
- public IoTDBTestRunner(Class testClass) throws InitializationError {
+ public IoTDBTestRunner(final Class testClass) throws InitializationError {
super(testClass);
}
@Override
- public void run(RunNotifier notifier) {
- TimeZone.setDefault(TimeZone.getTimeZone("Bejing"));
+ public void run(final RunNotifier notifier) {
+ TimeZone.setDefault(TimeZone.getTimeZone("UTC+08:00"));
listener = new IoTDBTestListener(this.getName());
notifier.addListener(listener);
super.run(notifier);
}
@Override
- protected void runChild(final FrameworkMethod method, RunNotifier notifier) {
- Description description = describeChild(method);
+ protected void runChild(final FrameworkMethod method, final RunNotifier notifier) {
+ final Description description = describeChild(method);
logger.info("Run {}", description.getMethodName());
- long currentTime = System.currentTimeMillis();
+ final long currentTime = System.currentTimeMillis();
if (EnvType.getSystemEnvType() != EnvType.MultiCluster) {
EnvFactory.getEnv().setTestMethodName(description.getMethodName());
- } else {
- // TestMethodName must be set globally in MultiEnvFactory, since the
- // cluster environments are not created now
- MultiEnvFactory.setTestMethodName(description.getMethodName());
}
+ MultiEnvFactory.setTestMethodName(description.getMethodName());
super.runChild(method, notifier);
- double timeCost = (System.currentTimeMillis() - currentTime) / 1000.0;
- String testName = description.getClassName() + "." + description.getMethodName();
+ final double timeCost = (System.currentTimeMillis() - currentTime) / 1000.0;
+ final String testName = description.getClassName() + "." + description.getMethodName();
logger.info("Done {}. Cost: {}s", description.getMethodName(), timeCost);
listener.addTestStat(new IoTDBTestStat(testName, timeCost));
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
index a0a27791a318a..6d3f2e85d8bd5 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
@@ -36,27 +36,33 @@ public void setUp() {
MultiEnvFactory.createEnv(2);
senderEnv = MultiEnvFactory.getEnv(0);
receiverEnv = MultiEnvFactory.getEnv(1);
+ setupConfig();
+ senderEnv.initClusterEnvironment();
+ receiverEnv.initClusterEnvironment();
+ }
+ protected void setupConfig() {
// TODO: delete ratis configurations
senderEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setPipeMemoryManagementEnabled(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setPipeMemoryManagementEnabled(false)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
- senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000);
- receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000);
-
- senderEnv.initClusterEnvironment();
- receiverEnv.initClusterEnvironment();
+ senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
}
@After
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java
index fb541e7dcdb7b..364c5d475f0c0 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java
@@ -36,6 +36,7 @@
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -470,7 +471,7 @@ public void testAlterPipeSourceAndProcessor() {
try (final Connection connection = senderEnv.getConnection();
final Statement statement = connection.createStatement()) {
statement.execute(sql);
- } catch (SQLException e) {
+ } catch (final SQLException e) {
fail(e.getMessage());
}
@@ -491,12 +492,13 @@ public void testAlterPipeSourceAndProcessor() {
TestUtils.assertDataEventuallyOnEnv(
receiverEnv, "select * from root.db.**", "Time,root.db.d1.at1,", expectedResSet);
- // Alter pipe (modify 'source.path' and 'processor.tumbling-time.interval-seconds')
+ // Alter pipe (modify 'source.path', 'source.inclusion' and
+ // 'processor.tumbling-time.interval-seconds')
try (final Connection connection = senderEnv.getConnection();
final Statement statement = connection.createStatement()) {
statement.execute(
- "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**') modify processor ('processor.tumbling-time.interval-seconds'='2')");
- } catch (SQLException e) {
+ "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**', 'source.inclusion'='all') modify processor ('processor.tumbling-time.interval-seconds'='2')");
+ } catch (final SQLException e) {
fail(e.getMessage());
}
@@ -527,5 +529,15 @@ public void testAlterPipeSourceAndProcessor() {
"select * from root.db.** where time > 10000",
"Time,root.db.d1.at1,root.db.d2.at1,",
expectedResSet);
+
+ // Create database on sender
+ if (!TestUtils.tryExecuteNonQueryWithRetry(
+ senderEnv, "create timeSeries root.db.d2.at2 int32")) {
+ fail();
+ }
+
+ // Check database on receiver
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv, "count timeSeries", "count(timeseries),", Collections.singleton("3,"));
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
index c00814cab0d4d..b89483840cb58 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
@@ -37,6 +37,7 @@
import org.junit.runner.RunWith;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -58,18 +59,22 @@ public void setUp() {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setPipeMemoryManagementEnabled(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setPipeMemoryManagementEnabled(false)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
- senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000);
- receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000);
+ senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
senderEnv.initClusterEnvironment();
receiverEnv.initClusterEnvironment();
@@ -328,4 +333,105 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception {
TestUtils.assertDataEventuallyOnEnv(
receiverEnv, "select s1 from root.db.d1", "Time,root.db.d1.s1,", expectedResSet);
}
+
+ @Test
+ public void testAutoManualCreateRace() throws Exception {
+ final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
+
+ final String receiverIp = receiverDataNode.getIp();
+ final int receiverPort = receiverDataNode.getPort();
+
+ try (final SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
+ final Map