diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 9f029eefcff6..3f8821714ed4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -87,7 +87,7 @@ public class ConfigNodeConfig { * SchemaRegionGroups for each Database. When set schema_region_group_extension_policy=AUTO, this * parameter is the default minimal number of SchemaRegionGroups for each Database. */ - private int defaultSchemaRegionGroupNumPerDatabase = 1; + private int defaultSchemaRegionGroupNumPerDatabase = 2; /** The maximum number of SchemaRegions expected to be managed by each DataNode. */ private double schemaRegionPerDataNode = schemaReplicationFactor; @@ -101,7 +101,7 @@ public class ConfigNodeConfig { * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 2; + private int defaultDataRegionGroupNumPerDatabase = 1; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 1ddd33aca483..fc36d0b99e48 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -115,9 +115,13 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan; import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.sync.ShowPipePlanV1; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -408,6 +412,18 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case CommitCreateTable: plan = new CommitCreateTablePlan(); break; + case PreDropTable: + plan = new PreDropTablePlan(); + break; + case RollbackPreDropTable: + plan = new RollbackDropTablePlan(); + break; + case CommitDropTable: + plan = new CommitDropTablePlan(); + break; + case AddTableColumn: + plan = new AddTableColumnPlan(); + break; case GetNodePathsPartition: plan = new GetNodePathsPartitionPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 9b476e12d295..b3b54c34446e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -151,6 +151,10 @@ public enum ConfigPhysicalPlanType { PreCreateTable((short) 850), RollbackCreateTable((short) 851), CommitCreateTable((short) 852), + PreDropTable((short) 853), + RollbackPreDropTable((short) 854), + CommitDropTable((short) 855), + AddTableColumn((short) 856), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java new file mode 100644 index 000000000000..eda7a26af74e --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class AddTableColumnPlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + private List columnSchemaList; + + private boolean isRollback; + + public AddTableColumnPlan() { + super(ConfigPhysicalPlanType.AddTableColumn); + } + + public AddTableColumnPlan( + String database, + String tableName, + List columnSchemaList, + boolean isRollback) { + super(ConfigPhysicalPlanType.AddTableColumn); + this.database = database; + this.tableName = tableName; + this.columnSchemaList = columnSchemaList; + this.isRollback = isRollback; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getColumnSchemaList() { + return columnSchemaList; + } + + public boolean isRollback() { + return isRollback; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + TsTableColumnSchemaUtil.serialize(columnSchemaList, stream); + ReadWriteIOUtils.write(isRollback, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + this.columnSchemaList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(buffer); + this.isRollback = ReadWriteIOUtils.readBool(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java new file mode 100644 index 000000000000..c8e6690718af --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class CommitDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public CommitDropTablePlan() { + super(ConfigPhysicalPlanType.CommitDropTable); + } + + public CommitDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.CommitDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java new file mode 100644 index 000000000000..e8ed113ece5b --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class PreDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public PreDropTablePlan() { + super(ConfigPhysicalPlanType.PreDropTable); + } + + public PreDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.PreDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java new file mode 100644 index 000000000000..9e8e150880e4 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class RollbackDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public RollbackDropTablePlan() { + super(ConfigPhysicalPlanType.RollbackPreDropTable); + } + + public RollbackDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.RollbackPreDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index da2d97cd4d35..d840690031de 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -44,6 +44,7 @@ import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.AlterTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.service.metric.MetricService; @@ -116,6 +117,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; @@ -141,6 +143,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -2285,4 +2288,27 @@ public TSStatus createTable(ByteBuffer tableInfo) { return status; } } + + public TSStatus dropTable(TDropTableReq req) { + TSStatus status = confirmLeader(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return procedureManager.dropTable(req.getDatabase(), req.getTableName(), req.getQueryId()); + } else { + return status; + } + } + + public TSStatus alterTable(TAlterTableReq req) { + TSStatus status = confirmLeader(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + switch (AlterTableOperationType.getType(req.operationType)) { + case ADD_COLUMN: + return procedureManager.alterTableAddColumn(req); + default: + throw new IllegalArgumentException(); + } + } else { + return status; + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 1e2e675eda10..37c4e4ca4697 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -34,6 +34,8 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.trigger.TriggerInformation; @@ -76,7 +78,9 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure; @@ -98,6 +102,7 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -1259,6 +1264,99 @@ public TSStatus createTable(String database, TsTable table) { } } + public TSStatus dropTable(String database, String tableName, String queryId) { + long procedureId = -1; + synchronized (this) { + boolean hasOverlappedTask = false; + ProcedureType type; + DropTableProcedure dropTableProcedure; + for (Procedure procedure : executor.getProcedures().values()) { + type = ProcedureFactory.getProcedureType(procedure); + if (type == null || !type.equals(ProcedureType.DROP_TABLE_PROCEDURE)) { + continue; + } + dropTableProcedure = (DropTableProcedure) procedure; + if (queryId.equals(dropTableProcedure.getQueryId())) { + procedureId = dropTableProcedure.getProcId(); + break; + } + if (database.equals(dropTableProcedure.getDatabase()) + && tableName.equals(dropTableProcedure.getTableName())) { + hasOverlappedTask = true; + break; + } + } + + if (procedureId == -1) { + if (hasOverlappedTask) { + return RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task dropping table with same name."); + } + procedureId = + this.executor.submitProcedure(new DropTableProcedure(database, tableName, queryId)); + } + } + List procedureStatus = new ArrayList<>(); + boolean isSucceed = + waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus); + if (isSucceed) { + return StatusUtils.OK; + } else { + return procedureStatus.get(0); + } + } + + public TSStatus alterTableAddColumn(TAlterTableReq req) { + String database = req.database; + String tableName = req.tableName; + String queryId = req.queryId; + List columnSchemaList = + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.updateInfo); + + long procedureId = -1; + synchronized (this) { + boolean hasOverlappedTask = false; + ProcedureType type; + AddTableColumnProcedure addTableColumnProcedure; + for (Procedure procedure : executor.getProcedures().values()) { + type = ProcedureFactory.getProcedureType(procedure); + if (type == null || !type.equals(ProcedureType.ADD_TABLE_COLUMN_PROCEDURE)) { + continue; + } + addTableColumnProcedure = (AddTableColumnProcedure) procedure; + if (queryId.equals(addTableColumnProcedure.getQueryId())) { + procedureId = addTableColumnProcedure.getProcId(); + break; + } + if (database.equals(addTableColumnProcedure.getDatabase()) + && tableName.equals(addTableColumnProcedure.getTableName())) { + hasOverlappedTask = true; + break; + } + } + + if (procedureId == -1) { + if (hasOverlappedTask) { + return RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task dropping table with same name."); + } + procedureId = + this.executor.submitProcedure( + new AddTableColumnProcedure(database, tableName, queryId, columnSchemaList)); + } + } + List procedureStatus = new ArrayList<>(); + boolean isSucceed = + waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus); + if (isSucceed) { + return StatusUtils.OK; + } else { + return procedureStatus.get(0); + } + } + // ====================================================== /* GET-SET Region diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 15ea10a16358..23f232cb4dfc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -28,7 +28,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -52,6 +54,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; @@ -1170,6 +1173,59 @@ public void updateSchemaQuotaConfiguration(long seriesThreshold, long deviceThre schemaQuotaStatistics.setSeriesThreshold(seriesThreshold); } + public TsTable getTable(String database, String tableName) { + return clusterSchemaInfo.getTsTable(database, tableName); + } + + public synchronized Pair> addTableColumn( + String database, String tableName, List columnSchemaList) { + Map> currentUsingTable = clusterSchemaInfo.getAllUsingTables(); + TsTable targetTable = null; + for (TsTable table : currentUsingTable.get(database)) { + if (table.getTableName().equals(tableName)) { + targetTable = table; + break; + } + } + + if (targetTable == null) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.TABLE_NOT_EXISTS, + String.format("Table %s.%s not exist", database, tableName)), + null); + } + + List copiedList = new ArrayList<>(); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (targetTable.getColumnSchema(columnSchema.getColumnName()) == null) { + copiedList.add(columnSchema); + } + } + + AddTableColumnPlan addTableColumnPlan = + new AddTableColumnPlan(database, tableName, copiedList, false); + try { + return new Pair<>(getConsensusManager().write(addTableColumnPlan), copiedList); + } catch (ConsensusException e) { + LOGGER.warn(e.getMessage(), e); + return new Pair<>( + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()), null); + } + } + + public synchronized TSStatus rollbackAddTableColumn( + String database, String tableName, List columnSchemaList) { + AddTableColumnPlan addTableColumnPlan = + new AddTableColumnPlan(database, tableName, columnSchemaList, true); + try { + return getConsensusManager().write(addTableColumnPlan); + } catch (ConsensusException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + public void clearSchemaQuotaCache() { schemaQuotaStatistics.clear(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index f09e7c7c6dc8..a226f1c9bf2e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -97,9 +97,13 @@ import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.runtime.TopicHandleMetaChangePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -453,6 +457,14 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.rollbackCreateTable((RollbackCreateTablePlan) physicalPlan); case CommitCreateTable: return clusterSchemaInfo.commitCreateTable((CommitCreateTablePlan) physicalPlan); + case PreDropTable: + return clusterSchemaInfo.preDropTable((PreDropTablePlan) physicalPlan); + case RollbackPreDropTable: + return clusterSchemaInfo.rollbackDropTable((RollbackDropTablePlan) physicalPlan); + case CommitDropTable: + return clusterSchemaInfo.dropTable((CommitDropTablePlan) physicalPlan); + case AddTableColumn: + return clusterSchemaInfo.addTableColumn((AddTableColumnPlan) physicalPlan); case CreatePipeV2: return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 8a92c73fb14b..eb0fa53e02d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -45,9 +45,13 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -1112,6 +1116,83 @@ public Map> getAllPreCreateTables() { } } + public TsTable getTsTable(String database, String tableName) { + databaseReadWriteLock.readLock().lock(); + try { + return mTree.getTable(new PartialPath(new String[] {ROOT, database}), tableName); + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + throw new RuntimeException(e); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + + public TSStatus preDropTable(PreDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.preDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus rollbackDropTable(RollbackDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.rollbackDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus dropTable(CommitDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.commitDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus addTableColumn(AddTableColumnPlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + if (plan.isRollback()) { + mTree.rollbackAddTableColumn( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), + plan.getTableName(), + plan.getColumnSchemaList()); + } else { + mTree.addTableColumn( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), + plan.getTableName(), + plan.getColumnSchemaList()); + } + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + // endregion @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index d031aa984209..27b8e263e3a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.factory.ConfigMNodeFactory; @@ -705,6 +706,81 @@ public Map> getAllPreCreateTables() throws MetadataExcepti return result; } + public TsTable getTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + return tableNode.getTable(); + } + + public void preDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.USING)) { + throw new IllegalStateException(); + } + tableNode.setStatus(TableNodeStatus.PRE_DELETE); + } + + public void rollbackDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.PRE_DELETE)) { + throw new IllegalStateException(); + } + tableNode.setStatus(TableNodeStatus.USING); + } + + public void commitDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.PRE_DELETE)) { + throw new IllegalStateException(); + } + databaseNode.deleteChild(tableName); + } + + public void addTableColumn( + PartialPath database, String tableName, List columnSchemaList) + throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + TsTable table = tableNode.getTable(); + columnSchemaList.forEach(table::addColumnSchema); + } + + public void rollbackAddTableColumn( + PartialPath database, String tableName, List columnSchemaList) + throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + TsTable table = tableNode.getTable(); + columnSchemaList.forEach(o -> table.removeColumnSchema(o.getColumnName())); + } + // endregion // region Serialization and Deserialization @@ -728,7 +804,7 @@ private void serializeChildren(IConfigMNode node, OutputStream outputStream) thr if (child.isDatabase()) { serializeDatabaseNode(child.getAsDatabaseMNode(), outputStream); } else if (child instanceof ConfigTableNode) { - serializeTableNode((ConfigTableNode) node, outputStream); + serializeTableNode((ConfigTableNode) child, outputStream); } else { serializeConfigBasicMNode(child, outputStream); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java new file mode 100644 index 000000000000..201e8307dfd8 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.client.DataNodeRequestType; +import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; +import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class AddTableColumnProcedure + extends StateMachineProcedure { + + private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); + + private String database; + + private String tableName; + + private String queryId; + + private List inputColumnList; + + private List actualAddedColumnList; + + public AddTableColumnProcedure() {} + + public AddTableColumnProcedure( + String database, + String tableName, + String queryId, + List inputColumnList) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + this.inputColumnList = inputColumnList; + } + + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, AddTableColumnState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case ADD_COLUMN: + LOGGER.info("Add column to table {}.{}", database, tableName); + addColumn(env); + break; + case UPDATE_CACHE: + LOGGER.info("Update cache of table {}.{} when adding column", database, tableName); + updateCache(env); + return Flow.NO_MORE_STATE; + default: + setFailure(new ProcedureException("Unrecognized AddTableColumnState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "AddTableColumn-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + private void addColumn(ConfigNodeProcedureEnv env) { + Pair> result = + env.getConfigManager() + .getClusterSchemaManager() + .addTableColumn(database, tableName, inputColumnList); + TSStatus status = result.getLeft(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + return; + } + actualAddedColumnList = result.getRight(); + setNextState(AddTableColumnState.UPDATE_CACHE); + } + + private void updateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = + new TUpdateTableReq( + TsTableInternalRPCType.ADD_COLUMN.getOperationType(), getCacheRequestInfo()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to update cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Update table cache failed"))); + return; + } + } + } + + @Override + protected void rollbackState(ConfigNodeProcedureEnv env, AddTableColumnState state) + throws IOException, InterruptedException, ProcedureException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case ADD_COLUMN: + rollbackAddColumn(env); + break; + case UPDATE_CACHE: + rollbackUpdateCache(env); + break; + } + } finally { + LOGGER.info( + "Rollback DropTable-{} costs {}ms.", state, (System.currentTimeMillis() - startTime)); + } + } + + private void rollbackAddColumn(ConfigNodeProcedureEnv env) { + if (actualAddedColumnList == null) { + return; + } + TSStatus status = + env.getConfigManager() + .getClusterSchemaManager() + .rollbackAddTableColumn(database, tableName, actualAddedColumnList); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + private void rollbackUpdateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = + new TUpdateTableReq( + TsTableInternalRPCType.ROLLBACK_ADD_COLUMN.getOperationType(), getCacheRequestInfo()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to rollback cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Rollback table cache failed"))); + return; + } + } + } + + private ByteBuffer getCacheRequestInfo() { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + TsTableColumnSchemaUtil.serialize(actualAddedColumnList, stream); + } catch (IOException ignored) { + // won't happen + } + return ByteBuffer.wrap(stream.toByteArray()); + } + + @Override + protected AddTableColumnState getState(int stateId) { + return AddTableColumnState.values()[stateId]; + } + + @Override + protected int getStateId(AddTableColumnState state) { + return state.ordinal(); + } + + @Override + protected AddTableColumnState getInitialState() { + return AddTableColumnState.ADD_COLUMN; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getQueryId() { + return queryId; + } + + public List getInputColumnList() { + return inputColumnList; + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.ADD_TABLE_COLUMN_PROCEDURE.getTypeCode()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + + TsTableColumnSchemaUtil.serialize(inputColumnList, stream); + TsTableColumnSchemaUtil.serialize(actualAddedColumnList, stream); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + + this.inputColumnList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(byteBuffer); + this.actualAddedColumnList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(byteBuffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AddTableColumnProcedure)) return false; + AddTableColumnProcedure that = (AddTableColumnProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java new file mode 100644 index 000000000000..1d3ddb81c30a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; +import org.apache.iotdb.confignode.client.DataNodeRequestType; +import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; +import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; +import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; + +public class DropTableProcedure + extends StateMachineProcedure { + + private static final Logger LOGGER = LoggerFactory.getLogger(DropTableProcedure.class); + + private String database; + + private String tableName; + + private String queryId; + + public DropTableProcedure() {} + + public DropTableProcedure(String database, String tableName, String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DropTableState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case PRE_DROP: + LOGGER.info("Pre drop table {}.{}", database, tableName); + preDropTable(env); + break; + case INVALIDATE_CACHE: + LOGGER.info("Invalidate cache of table {}.{}", database, tableName); + invalidateCache(env); + break; + case DELETE_DATA: + LOGGER.info("Delete data of table {}.{}", database, tableName); + deleteTableData(env); + break; + case DROP_TABLE: + LOGGER.info("Commit drop table {}.{}", database, tableName); + dropTable(env); + return Flow.NO_MORE_STATE; + default: + setFailure(new ProcedureException("Unrecognized DropTableState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "DropTable-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + private void preDropTable(ConfigNodeProcedureEnv env) { + PreDropTablePlan plan = new PreDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setNextState(DropTableState.INVALIDATE_CACHE); + } else { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + private void invalidateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = new TUpdateTableReq(); + req.setType(TsTableInternalRPCType.INVALIDATE_CACHE.getOperationType()); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } catch (IOException ignored) { + // won't happen + } + req.setTableInfo(stream.toByteArray()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to invalidate cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Invalidate table cache failed"))); + return; + } + } + setNextState(DropTableState.DELETE_DATA); + } + + private void deleteTableData(ConfigNodeProcedureEnv env) { + PartialPath tableTsPath = new PartialPath(new String[] {"root", database, tableName, "**"}); + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendFullPath(tableTsPath); + patternTree.constructTree(); + + deleteDataInDataRegions(env, patternTree); + deleteSchemaInSchemaRegions(env, patternTree); + setNextState(DropTableState.DROP_TABLE); + } + + private void deleteDataInDataRegions(ConfigNodeProcedureEnv env, PathPatternTree patternTree) { + Map targetDataRegionGroup = + env.getConfigManager().getRelatedDataRegionGroup(patternTree); + if (targetDataRegionGroup.isEmpty()) { + return; + } + DropTableRegionTaskExecutor deleteDataTask = + new DropTableRegionTaskExecutor<>( + "delete data", + env, + targetDataRegionGroup, + true, + DataNodeRequestType.UPDATE_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TUpdateTableReq( + TsTableInternalRPCType.DELETE_DATA_IN_DATA_REGION.getOperationType(), + serializeDeleteDataReq(database, tableName, targetDataRegionGroup)))); + deleteDataTask.execute(); + } + + private void deleteSchemaInSchemaRegions( + ConfigNodeProcedureEnv env, PathPatternTree patternTree) { + Map targetSchemaRegionGroup = + env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); + if (targetSchemaRegionGroup.isEmpty()) { + return; + } + DropTableRegionTaskExecutor deleteDataTask = + new DropTableRegionTaskExecutor<>( + "delete schema", + env, + targetSchemaRegionGroup, + false, + DataNodeRequestType.UPDATE_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TUpdateTableReq( + TsTableInternalRPCType.DELETE_SCHEMA_IN_SCHEMA_REGION.getOperationType(), + serializeDeleteDataReq(database, tableName, targetSchemaRegionGroup)))); + deleteDataTask.execute(); + } + + private ByteBuffer serializeDeleteDataReq( + String database, + String tableName, + Map targetRegionMap) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(targetRegionMap.size(), stream); + for (TConsensusGroupId id : targetRegionMap.keySet()) { + ReadWriteIOUtils.write(id.getId(), stream); + } + } catch (IOException ignored) { + // won't happen + } + return ByteBuffer.wrap(stream.toByteArray()); + } + + private void dropTable(ConfigNodeProcedureEnv env) { + CommitDropTablePlan plan = new CommitDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + @Override + protected void rollbackState(ConfigNodeProcedureEnv env, DropTableState state) + throws IOException, InterruptedException, ProcedureException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case PRE_DROP: + rollbackPreDrop(env); + break; + case INVALIDATE_CACHE: + rollbackInvalidateCache(env); + break; + } + } finally { + LOGGER.info( + "Rollback DropTable-{} costs {}ms.", state, (System.currentTimeMillis() - startTime)); + } + } + + private void rollbackInvalidateCache(ConfigNodeProcedureEnv env) { + TsTable table = env.getConfigManager().getClusterSchemaManager().getTable(database, tableName); + TSStatus status = env.getConfigManager().getProcedureManager().createTable(database, table); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode()) { + return; + } + LOGGER.warn( + "Error occurred during rollback table cache for DropTable {}.{}: {}", + database, + tableName, + status); + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + + private void rollbackPreDrop(ConfigNodeProcedureEnv env) { + RollbackDropTablePlan plan = new RollbackDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + @Override + protected DropTableState getState(int stateId) { + return DropTableState.values()[stateId]; + } + + @Override + protected int getStateId(DropTableState dropTableState) { + return dropTableState.ordinal(); + } + + @Override + protected DropTableState getInitialState() { + return DropTableState.PRE_DROP; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getQueryId() { + return queryId; + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.DROP_TABLE_PROCEDURE.getTypeCode()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DropTableProcedure)) return false; + DropTableProcedure that = (DropTableProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } + + private class DropTableRegionTaskExecutor extends DataNodeRegionTaskExecutor { + + private final String taskName; + + DropTableRegionTaskExecutor( + String taskName, + ConfigNodeProcedureEnv env, + Map targetSchemaRegionGroup, + boolean executeOnAllReplicaset, + DataNodeRequestType dataNodeRequestType, + BiFunction, Q> dataNodeRequestGenerator) { + super( + env, + targetSchemaRegionGroup, + executeOnAllReplicaset, + dataNodeRequestType, + dataNodeRequestGenerator); + this.taskName = taskName; + } + + @Override + protected List processResponseOfOneDataNode( + TDataNodeLocation dataNodeLocation, + List consensusGroupIdList, + TSStatus response) { + List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return failedRegionList; + } + + if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + List subStatus = response.getSubStatus(); + for (int i = 0; i < subStatus.size(); i++) { + if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } + } else { + failedRegionList.addAll(consensusGroupIdList); + } + return failedRegionList; + } + + @Override + protected void onAllReplicasetFailure( + TConsensusGroupId consensusGroupId, Set dataNodeLocationSet) { + setFailure( + new ProcedureException( + new MetadataException( + String.format( + "Drop table %s.%s failed when [%s] because all replicaset of schemaRegion %s failed. %s", + database, tableName, taskName, consensusGroupId.id, dataNodeLocationSet)))); + interruptTask(); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java new file mode 100644 index 000000000000..d819e73aa5d0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum AddTableColumnState { + ADD_COLUMN, + UPDATE_CACHE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java new file mode 100644 index 000000000000..941139746367 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum DropTableState { + PRE_DROP, + INVALIDATE_CACHE, + DELETE_DATA, + DROP_TABLE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index 394a76a802d7..99dd350d05a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -45,7 +45,9 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure; @@ -181,6 +183,12 @@ public Procedure create(ByteBuffer buffer) throws IOException { case CREATE_TABLE_PROCEDURE: procedure = new CreateTableProcedure(); break; + case DROP_TABLE_PROCEDURE: + procedure = new DropTableProcedure(); + break; + case ADD_TABLE_COLUMN_PROCEDURE: + procedure = new AddTableColumnProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; @@ -308,6 +316,10 @@ public static ProcedureType getProcedureType(Procedure procedure) { return ProcedureType.UNSET_TEMPLATE_PROCEDURE; } else if (procedure instanceof CreateTableProcedure) { return ProcedureType.CREATE_TABLE_PROCEDURE; + } else if (procedure instanceof DropTableProcedure) { + return ProcedureType.DROP_TABLE_PROCEDURE; + } else if (procedure instanceof AddTableColumnProcedure) { + return ProcedureType.ADD_TABLE_COLUMN_PROCEDURE; } else if (procedure instanceof CreatePipePluginProcedure) { return ProcedureType.CREATE_PIPE_PLUGIN_PROCEDURE; } else if (procedure instanceof DropPipePluginProcedure) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 39a92287cdf1..f2c4cea6089a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -63,6 +63,8 @@ public enum ProcedureType { SET_TEMPLATE_PROCEDURE((short) 702), CREATE_TABLE_PROCEDURE((short) 750), + DROP_TABLE_PROCEDURE((short) 751), + ADD_TABLE_COLUMN_PROCEDURE((short) 752), // ProcedureId 800-899 is used by IoTDB-Ml diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index e88ab55d144d..deeda0835579 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -69,6 +69,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -108,6 +109,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -1114,4 +1116,14 @@ public TThrottleQuotaResp getThrottleQuota() { public TSStatus createTable(ByteBuffer tableInfo) throws TException { return configManager.createTable(tableInfo); } + + @Override + public TSStatus dropTable(TDropTableReq req) throws TException { + return configManager.dropTable(req); + } + + @Override + public TSStatus alterTable(TAlterTableReq req) throws TException { + return configManager.alterTable(req); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index d8b4ff691de3..c67cef061019 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -38,8 +38,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -525,6 +527,29 @@ public TSStatus visitDeleteLogicalView(DeleteLogicalViewNode node, ISchemaRegion } } + @Override + public TSStatus visitCreateTableDevice(CreateTableDeviceNode node, ISchemaRegion schemaRegion) { + try { + schemaRegion.createTableDevice( + node.getDevicePathList(), node.getAttributeNameList(), node.getAttributeValueList()); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + + @Override + public TSStatus visitDeleteTableDevice(DeleteTableDeviceNode node, ISchemaRegion schemaRegion) { + try { + schemaRegion.deleteTableDevice(node.getTable()); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( PipeEnrichedWritePlanNode node, ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index f31728aa1b86..e20b26f615ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -38,6 +38,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -77,6 +78,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -1073,6 +1075,18 @@ public TSStatus createTable(ByteBuffer tableInfo) throws TException { () -> client.createTable(tableInfo), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus dropTable(TDropTableReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.dropTable(req), status -> !updateConfigNodeLeader(status)); + } + + @Override + public TSStatus alterTable(TAlterTableReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.alterTable(req), status -> !updateConfigNodeLeader(status)); + } + public static class Factory extends ThriftClientFactory { public Factory( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index aaf2fa1a514e..acdff0628eae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -91,6 +91,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateAlignedTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateMultiTimeSeriesStatement; @@ -104,6 +105,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.SetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.relational.sql.tree.Insert; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -336,17 +338,29 @@ private TSExecuteStatementResp executeStatementInternal( queryId = SESSION_MANAGER.requestQueryId(clientSession, req.statementId); - // TODO audit log, quota, StatementType - result = - COORDINATOR.executeForTableModel( - s, - relationSqlParser, - clientSession, - queryId, - SESSION_MANAGER.getSessionInfo(clientSession), - statement, - metadata, - req.getTimeout()); + if (s instanceof Insert) { + result = + COORDINATOR.executeForTreeModel( + new InsertTableStatement(clientSession, (Insert) s), + queryId, + SESSION_MANAGER.getSessionInfo(clientSession), + statement, + partitionFetcher, + schemaFetcher, + req.getTimeout()); + } else { + // TODO audit log, quota, StatementType + result = + COORDINATOR.executeForTableModel( + s, + relationSqlParser, + clientSession, + queryId, + SESSION_MANAGER.getSessionInfo(clientSession), + statement, + metadata, + req.getTimeout()); + } } if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index b74a7def880e..383ad26d28b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.protocol.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; @@ -48,6 +49,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -107,6 +109,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.PreDeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.RollbackPreDeactivateTemplateNode; @@ -118,6 +121,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -267,6 +271,7 @@ import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -493,7 +498,15 @@ public TSStatus invalidateSchemaCache(TInvalidateCacheReq req) { // req.getFullPath() is a database path DataNodeSchemaCache.getInstance().invalidate(req.getFullPath()); ClusterTemplateManager.getInstance().invalid(req.getFullPath()); + if (req.isStorageGroup()) { + PartialPath path = new PartialPath(req.getFullPath()); + String[] nodes = path.getNodes(); + DataNodeTableCache.getInstance().invalidateTable(nodes[1]); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } catch (IllegalPathException e) { + throw new RuntimeException(e); } finally { DataNodeSchemaCache.getInstance().releaseWriteLock(); } @@ -1416,6 +1429,9 @@ public TFetchFragmentInstanceStatisticsResp fetchFragmentInstanceStatistics( @Override public TSStatus updateTable(TUpdateTableReq req) throws TException { + String database; + String tableName; + List regionIdList; switch (TsTableInternalRPCType.getType(req.type)) { case PRE_CREATE: DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.TIMESERIES_VS_TABLE); @@ -1440,6 +1456,62 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo)); break; + case INVALIDATE_CACHE: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); + DataNodeTableCache.getInstance().invalidateTable(database, tableName); + TableModelSchemaFetcher.getInstance().invalidateDeviceCache(database, tableName); + break; + case DELETE_DATA_IN_DATA_REGION: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); + regionIdList = deserializeRegionList(req.tableInfo, true); + return executeInternalSchemaTask( + regionIdList, + consensusGroupId -> { + RegionWriteExecutor executor = new RegionWriteExecutor(); + return executor + .execute( + new DataRegionId(consensusGroupId.getId()), + new DeleteDataNode( + new PlanNodeId(""), + Collections.singletonList( + new PartialPath( + new String[] { + PATH_ROOT, database, tableName, MULTI_LEVEL_PATH_WILDCARD + })), + Long.MIN_VALUE, + Long.MAX_VALUE)) + .getStatus(); + }); + case DELETE_SCHEMA_IN_SCHEMA_REGION: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); + regionIdList = deserializeRegionList(req.tableInfo, false); + return executeInternalSchemaTask( + regionIdList, + consensusGroupId -> { + RegionWriteExecutor executor = new RegionWriteExecutor(); + return executor + .execute( + new SchemaRegionId(consensusGroupId.getId()), + new DeleteTableDeviceNode(new PlanNodeId(""), tableName)) + .getStatus(); + }); + case ADD_COLUMN: + DataNodeTableCache.getInstance() + .addTableColumn( + ReadWriteIOUtils.readString(req.tableInfo), + ReadWriteIOUtils.readString(req.tableInfo), + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.tableInfo)); + break; + case ROLLBACK_ADD_COLUMN: + DataNodeTableCache.getInstance() + .rollbackAddColumn( + ReadWriteIOUtils.readString(req.tableInfo), + ReadWriteIOUtils.readString(req.tableInfo), + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.tableInfo)); + break; default: LOGGER.warn("Unsupported type {} when updating table", req.type); return RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER); @@ -1447,6 +1519,23 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + private List deserializeRegionList(ByteBuffer byteBuffer, boolean isData) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + List regionIdList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + if (isData) { + regionIdList.add( + new TConsensusGroupId( + TConsensusGroupType.DataRegion, ReadWriteIOUtils.readInt(byteBuffer))); + } else { + regionIdList.add( + new TConsensusGroupId( + TConsensusGroupType.SchemaRegion, ReadWriteIOUtils.readInt(byteBuffer))); + } + } + return regionIdList; + } + private PathPatternTree filterPathPatternTree(PathPatternTree patternTree, String storageGroup) { PathPatternTree filteredPatternTree = new PathPatternTree(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 1cc2f8663dad..3307329accf4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -75,6 +75,8 @@ public class MPPQueryContext { QueryPlanStatistics queryPlanStatistics = null; + private boolean skipSchemaValidate = false; + public MPPQueryContext(QueryId queryId) { this.queryId = queryId; this.endPointBlackList = new LinkedList<>(); @@ -290,4 +292,12 @@ public void setLogicalOptimizationCost(long logicalOptimizeCost) { } queryPlanStatistics.setLogicalOptimizationCost(logicalOptimizeCost); } + + public boolean isSkipSchemaValidate() { + return skipSchemaValidate; + } + + public void setSkipSchemaValidate(boolean skipSchemaValidate) { + this.skipSchemaValidate = skipSchemaValidate; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java index a2589c6bb19f..dff3becd51b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java @@ -22,6 +22,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; @@ -69,7 +71,15 @@ public void serialize(ByteBuffer byteBuffer) { if (hasAlias()) { ReadWriteIOUtils.write(alias, byteBuffer); } - dataType.serializeTo(byteBuffer); + } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(columnName, stream); + ReadWriteIOUtils.write(dataType.ordinal(), stream); + ReadWriteIOUtils.write(hasAlias(), stream); + if (hasAlias()) { + ReadWriteIOUtils.write(alias, stream); + } } public static ColumnHeader deserialize(ByteBuffer byteBuffer) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 7fe108425795..d3e596c46011 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -98,4 +99,22 @@ public static ISchemaSource getLogicalViewSchemaSource( PathPatternTree scope) { return new LogicalViewSchemaSource(pathPattern, limit, offset, schemaFilter, scope); } + + public static ISchemaSource getTableDeviceSchemaSource( + String database, + String tableName, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, + List columnHeaderList) { + return new TableDeviceSchemaSource( + database, tableName, idDeterminedFilterList, idFuzzyFilter, columnHeaderList); + } + + public static ISchemaSource getTableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + return new TableDeviceFetchSource(database, tableName, deviceIdList, columnHeaderList); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java new file mode 100644 index 000000000000..654d3e28f8f4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.List; + +public class TableDeviceFetchSource implements ISchemaSource { + + private String database; + + private String tableName; + + private List deviceIdList; + + private List columnHeaderList; + + public TableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + this.columnHeaderList = columnHeaderList; + } + + @Override + public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { + try { + return schemaRegion.getDeviceReader( + DeviceFilterUtil.convertToDevicePath(database, tableName, deviceIdList)); + } catch (MetadataException e) { + throw new SchemaExecutionException(e); + } + } + + @Override + public List getInfoQueryColumnHeaders() { + return columnHeaderList; + } + + @Override + public void transformToTsBlockColumns( + IDeviceSchemaInfo schemaInfo, TsBlockBuilder builder, String database) { + builder.getTimeColumnBuilder().writeLong(0L); + int resultIndex = 0; + int idIndex = 0; + String[] pathNodes = schemaInfo.getRawNodes(); + TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); + TsTableColumnSchema columnSchema; + for (ColumnHeader columnHeader : columnHeaderList) { + columnSchema = table.getColumnSchema(columnHeader.getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + if (pathNodes[idIndex + 3] == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary(new Binary(pathNodes[idIndex + 3], TSFileConfig.STRING_CHARSET)); + } + idIndex++; + } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + String attributeValue = schemaInfo.getAttributeValue(columnHeader.getColumnName()); + if (attributeValue == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } + } + resultIndex++; + } + builder.declarePosition(); + } + + @Override + public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { + return false; + } + + @Override + public long getSchemaStatistic(ISchemaRegion schemaRegion) { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java new file mode 100644 index 000000000000..b5275b444c7e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.List; +import java.util.NoSuchElementException; + +public class TableDeviceSchemaSource implements ISchemaSource { + + private String database; + + private String tableName; + + private List> idDeterminedFilterList; + + private SchemaFilter idFuzzyFilter; + + private List columnHeaderList; + + public TableDeviceSchemaSource( + String database, + String tableName, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, + List columnHeaderList) { + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilter = idFuzzyFilter; + this.columnHeaderList = columnHeaderList; + } + + @Override + public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { + List devicePatternList = getDevicePatternList(); + return new ISchemaReader() { + + private ISchemaReader deviceReader; + private Throwable throwable; + private int index = 0; + + @Override + public boolean isSuccess() { + return throwable == null && (deviceReader == null || deviceReader.isSuccess()); + } + + @Override + public Throwable getFailure() { + if (throwable != null) { + return throwable; + } else if (deviceReader != null) { + return deviceReader.getFailure(); + } + return null; + } + + @Override + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + @Override + public boolean hasNext() { + try { + if (throwable != null) { + return false; + } + if (deviceReader != null) { + if (deviceReader.hasNext()) { + return true; + } else { + deviceReader.close(); + if (!deviceReader.isSuccess()) { + throwable = deviceReader.getFailure(); + return false; + } + } + } + + while (index < devicePatternList.size()) { + deviceReader = + schemaRegion.getTableDeviceReader( + new ShowTableDevicesPlan(devicePatternList.get(index), idFuzzyFilter)); + index++; + if (deviceReader.hasNext()) { + return true; + } else { + deviceReader.close(); + } + } + return false; + } catch (Exception e) { + throw new SchemaExecutionException(e.getMessage(), e); + } + } + + @Override + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return deviceReader.next(); + } + + @Override + public void close() throws Exception { + if (deviceReader != null) { + deviceReader.close(); + } + } + }; + } + + private List getDevicePatternList() { + return DeviceFilterUtil.convertToDevicePattern( + database, + DataNodeTableCache.getInstance().getTable(database, tableName), + idDeterminedFilterList); + } + + @Override + public List getInfoQueryColumnHeaders() { + return columnHeaderList; + } + + @Override + public void transformToTsBlockColumns( + IDeviceSchemaInfo schemaInfo, TsBlockBuilder builder, String database) { + builder.getTimeColumnBuilder().writeLong(0L); + int resultIndex = 0; + int idIndex = 0; + String[] pathNodes = schemaInfo.getRawNodes(); + TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); + TsTableColumnSchema columnSchema; + for (ColumnHeader columnHeader : columnHeaderList) { + columnSchema = table.getColumnSchema(columnHeader.getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + if (pathNodes[idIndex + 3] == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary(new Binary(pathNodes[idIndex + 3], TSFileConfig.STRING_CHARSET)); + } + idIndex++; + } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + String attributeValue = schemaInfo.getAttributeValue(columnHeader.getColumnName()); + if (attributeValue == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } + } + resultIndex++; + } + builder.declarePosition(); + } + + @Override + public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { + return false; + } + + @Override + public long getSchemaStatistic(ISchemaRegion schemaRegion) { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index e136ae05110a..911f5a4ded54 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -79,6 +80,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByVariationParameter; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; @@ -101,6 +103,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -140,6 +143,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; @@ -2719,6 +2725,9 @@ public Analysis visitPipeEnrichedStatement( private void validateSchema( Analysis analysis, InsertBaseStatement insertStatement, MPPQueryContext context) { + if (context.isSkipSchemaValidate()) { + return; + } final long startTime = System.nanoTime(); try { SchemaValidator.validate(schemaFetcher, insertStatement, context); @@ -3644,4 +3653,123 @@ public Analysis visitShowCurrentTimestamp( analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowCurrentTimestampHeader()); return analysis; } + + @Override + public Analysis visitInsertTable( + InsertTableStatement insertTableStatement, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + Analysis analysis = new Analysis(); + + try { + insertTableStatement.setTableSchema( + TableModelSchemaFetcher.getInstance() + .validateTableHeaderSchema( + insertTableStatement.getDatabase(), + insertTableStatement.getTableSchema(), + context)); + } catch (SemanticException e) { + analysis.setFinishQueryAfterAnalyze(true); + if (e.getCause() instanceof IoTDBException) { + IoTDBException exception = (IoTDBException) e.getCause(); + analysis.setFailStatus( + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); + } else { + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); + } + } + + InsertRowStatement insertRowStatement = insertTableStatement.getInsertRowStatement(); + + try { + TableModelSchemaFetcher.getInstance().validateDeviceSchema(insertTableStatement, context); + } catch (SemanticException e) { + analysis.setFinishQueryAfterAnalyze(true); + if (e.getCause() instanceof IoTDBException) { + IoTDBException exception = (IoTDBException) e.getCause(); + analysis.setFailStatus( + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); + } else { + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); + } + } + + if (analysis.isFinishQueryAfterAnalyze()) { + return analysis; + } + context.setSkipSchemaValidate(true); + return insertRowStatement.accept(this, context); + } + + @Override + public Analysis visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + Analysis analysis = new Analysis(); + analysis.setStatement(createTableDeviceStatement); + + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath devicePath : createTableDeviceStatement.getPaths()) { + patternTree.appendFullPath(devicePath.concatNode(ONE_LEVEL_PATH_WILDCARD)); + } + SchemaPartition partition = + partitionFetcher.getOrCreateSchemaPartition( + patternTree, context.getSession().getUserName()); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; + } + + @Override + public Analysis visitShowTableDevices( + ShowTableDevicesStatement statement, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + Analysis analysis = new Analysis(); + analysis.setStatement(statement); + + String database = statement.getDatabase(); + String tableName = statement.getTableName(); + + List devicePatternList = + DeviceFilterUtil.convertToDevicePattern( + database, + DataNodeTableCache.getInstance().getTable(database, tableName), + statement.getIdDeterminedFilterList()); + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath devicePattern : devicePatternList) { + patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); + } + + SchemaPartition partition = partitionFetcher.getSchemaPartition(patternTree); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; + } + + @Override + public Analysis visitFetchTableDevices( + FetchTableDevicesStatement statement, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + Analysis analysis = new Analysis(); + analysis.setStatement(statement); + + String database = statement.getDatabase(); + String tableName = statement.getTableName(); + + List devicePatternList = + DeviceFilterUtil.convertToDevicePath(database, tableName, statement.getDeviceIdList()); + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath devicePattern : devicePatternList) { + patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); + } + + SchemaPartition partition = + partitionFetcher.getOrCreateSchemaPartition( + patternTree, context.getSession().getUserName()); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index b005c08c5c69..9cd116f13342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; @@ -151,7 +152,16 @@ private TSDataType getDataType(DataType dataType) { @Override protected IConfigTask visitDropTable(DropTable node, MPPQueryContext context) { - return super.visitDropTable(node, context); + context.setQueryType(QueryType.WRITE); + String database = clientSession.getDatabaseName(); + if (node.getTableName().getPrefix().isPresent()) { + database = node.getTableName().getPrefix().get().toString(); + } + if (database == null) { + throw new SemanticException("unknown database"); + } + return new DropTableTask( + database, node.getTableName().getSuffix(), context.getQueryId().toString()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 534eb794ed87..732ee3c8f92d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -42,8 +42,11 @@ import org.apache.iotdb.commons.pipe.plugin.service.PipePluginClassLoader; import org.apache.iotdb.commons.pipe.plugin.service.PipePluginExecutableManager; import org.apache.iotdb.commons.pipe.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.schema.table.AlterTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.trigger.service.TriggerExecutableManager; @@ -53,6 +56,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -71,6 +75,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp; @@ -2856,6 +2861,86 @@ public SettableFuture showTables(String database) { return future; } + @Override + public SettableFuture dropTable( + String database, String tableName, String queryId) { + final SettableFuture future = SettableFuture.create(); + try (ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + do { + try { + tsStatus = client.dropTable(new TDropTableReq(database, tableName, queryId)); + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + LOGGER.warn( + "Failed to execute drop table {}.{}, status is {}.", database, tableName, tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + + @Override + public SettableFuture alterTableAddColumn( + String database, + String tableName, + List columnSchemaList, + String queryId) { + final SettableFuture future = SettableFuture.create(); + try (ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + TAlterTableReq req = new TAlterTableReq(); + req.setDatabase(database); + req.setTableName(tableName); + req.setQueryId(queryId); + req.setOperationType(AlterTableOperationType.ADD_COLUMN.getTypeValue()); + req.setUpdateInfo(TsTableColumnSchemaUtil.serialize(columnSchemaList)); + + do { + try { + tsStatus = client.alterTable(req); + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + LOGGER.warn( + "Failed to add column to table {}.{}, status is {}.", database, tableName, tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + public void handlePipeConfigClientExit(String clientId) { try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index b5f8ce9c54f7..7406331d2d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.db.protocol.session.IClientSession; @@ -84,6 +85,8 @@ import com.google.common.util.concurrent.SettableFuture; +import java.util.List; + public interface IConfigTaskExecutor { SettableFuture setDatabase(DatabaseSchemaStatement databaseSchemaStatement); @@ -261,4 +264,12 @@ SettableFuture showThrottleQuota( SettableFuture describeTable(String database, String tableName); SettableFuture showTables(String database); + + SettableFuture dropTable(String database, String tableName, String queryId); + + SettableFuture alterTableAddColumn( + String database, + String tableName, + List columnSchemaList, + String queryId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java new file mode 100644 index 000000000000..febd54cc8a7a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.IdColumnSchema; +import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.BinaryType; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class AlterTableAddColumnTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final List columnList; + + private final String queryId; + + public AlterTableAddColumnTask( + String database, String tableName, List columnList, String queryId) { + this.database = database; + this.tableName = tableName; + this.columnList = parseInputColumnSchema(columnList); + this.queryId = queryId; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterTableAddColumn(database, tableName, columnList, queryId); + } + + private List parseInputColumnSchema(List inputColumnList) { + List columnSchemaList = new ArrayList<>(inputColumnList.size()); + for (ColumnSchema inputColumn : inputColumnList) { + switch (inputColumn.getColumnCategory()) { + case ID: + if (!inputColumn.getType().equals(BinaryType.TEXT)) { + throw new SemanticException("Id column only support data type TEXT."); + } + columnSchemaList.add(new IdColumnSchema(inputColumn.getName(), TSDataType.TEXT)); + break; + case ATTRIBUTE: + if (!inputColumn.getType().equals(BinaryType.TEXT)) { + throw new SemanticException("Attribute column only support data type TEXT."); + } + columnSchemaList.add(new AttributeColumnSchema(inputColumn.getName(), TSDataType.TEXT)); + break; + case MEASUREMENT: + TSDataType dataType = InternalTypeManager.getTSDataType(inputColumn.getType()); + columnSchemaList.add( + new MeasurementColumnSchema( + inputColumn.getName(), + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor())); + break; + } + } + return columnSchemaList; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java new file mode 100644 index 000000000000..80dc871180d6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class DropTableTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final String queryId; + + public DropTableTask(String database, String tableName, String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.dropTable(database, tableName, queryId); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 05e0a90b4c43..6c7e80c260f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; import org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil; @@ -56,6 +57,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.DeviceViewIntoNode; @@ -1644,4 +1647,38 @@ public LogicalPlanBuilder planEndTimeColumnInject( new SlidingTimeColumnGeneratorParameter(groupByTimeParameter, ascending)); return this; } + + public LogicalPlanBuilder planTableDeviceSource( + String database, + String tableName, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, + List columnHeaderList) { + this.root = + new TableDeviceScanNode( + context.getQueryId().genPlanNodeId(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilter, + columnHeaderList, + null); + return this; + } + + public LogicalPlanBuilder planTableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + this.root = + new TableDeviceFetchNode( + context.getQueryId().genPlanNodeId(), + database, + tableName, + deviceIdList, + columnHeaderList, + null); + return this; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index a9e348031498..f1aa9c07725e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -21,10 +21,14 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer; import org.apache.iotdb.db.queryengine.plan.expression.Expression; @@ -40,6 +44,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -91,6 +96,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.commons.lang3.StringUtils; @@ -1012,4 +1021,74 @@ public PlanNode visitShowLogicalView( .planLimit(showLogicalViewStatement.getLimit()) .getRoot(); } + + @Override + public PlanNode visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, MPPQueryContext context) { + return new CreateTableDeviceNode( + context.getQueryId().genPlanNodeId(), + createTableDeviceStatement.getPaths(), + createTableDeviceStatement.getAttributeNameList(), + createTableDeviceStatement.getAttributeValueList()); + } + + @Override + public PlanNode visitShowTableDevices( + ShowTableDevicesStatement statement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + + List columnHeaderList = + getColumnHeaderList(statement.getDatabase(), statement.getTableName()); + + analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true)); + + planBuilder = + planBuilder + .planTableDeviceSource( + statement.getDatabase(), + statement.getTableName(), + statement.getIdDeterminedFilterList(), + statement.getIdFuzzyFilter(), + columnHeaderList) + .planSchemaQueryMerge(false); + + return planBuilder.getRoot(); + } + + private List getColumnHeaderList(String database, String tableName) { + List columnSchemaList = + DataNodeTableCache.getInstance().getTable(database, tableName).getColumnList(); + + List columnHeaderList = new ArrayList<>(columnSchemaList.size()); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID) + || columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + columnHeaderList.add( + new ColumnHeader(columnSchema.getColumnName(), columnSchema.getDataType())); + } + } + return columnHeaderList; + } + + @Override + public PlanNode visitFetchTableDevices( + FetchTableDevicesStatement statement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + + List columnHeaderList = + getColumnHeaderList(statement.getDatabase(), statement.getTableName()); + + analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true)); + + planBuilder = + planBuilder + .planTableDeviceFetchSource( + statement.getDatabase(), + statement.getTableName(), + statement.getDeviceIdList(), + columnHeaderList) + .planSchemaQueryMerge(false); + + return planBuilder.getRoot(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index acd45e357579..8f0af6b1f39e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -175,6 +175,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; @@ -3266,4 +3268,45 @@ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanC return new ExplainAnalyzeOperator( operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); } + + @Override + public Operator visitTableDeviceScan( + TableDeviceScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaQueryScanOperator.class.getSimpleName()); + return new SchemaQueryScanOperator<>( + node.getPlanNodeId(), + operatorContext, + SchemaSourceFactory.getTableDeviceSchemaSource( + node.getDatabase(), + node.getTableName(), + node.getIdDeterminedFilterList(), + node.getIdFuzzyFilter(), + node.getColumnHeaderList())); + } + + @Override + public Operator visitTableDeviceFetch( + TableDeviceFetchNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaQueryScanOperator.class.getSimpleName()); + return new SchemaQueryScanOperator<>( + node.getPlanNodeId(), + operatorContext, + SchemaSourceFactory.getTableDeviceFetchSource( + node.getDatabase(), + node.getTableName(), + node.getDeviceIdList(), + node.getColumnHeaderList())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index 3ae7a3f126f5..3c7f24a19da3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -33,6 +33,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.DeviceMergeNode; @@ -154,6 +156,16 @@ public PlanNode visitSchemaQueryScan(SchemaQueryScanNode node, NodeGroupContext return processNoChildSourceNode(node, context); } + @Override + public PlanNode visitTableDeviceScan(TableDeviceScanNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + + @Override + public PlanNode visitTableDeviceFetch(TableDeviceFetchNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + @Override public PlanNode visitSchemaFetchScan(SchemaFetchScanNode node, NodeGroupContext context) { return processNoChildSourceNode(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 0e2d77bc2b3c..0e52538126bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -36,14 +36,18 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -455,6 +459,14 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return ExplainAnalyzeNode.deserialize(buffer); case 91: return PipeOperateSchemaQueueNode.deserialize(buffer); + case 92: + return CreateTableDeviceNode.deserialize(buffer); + case 93: + return TableDeviceScanNode.deserialize(buffer); + case 94: + return TableDeviceFetchNode.deserialize(buffer); + case 95: + return DeleteTableDeviceNode.deserialize(buffer); case 1000: return org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode .deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 4e15290f47b4..34fd19bd7b3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -33,14 +33,18 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -554,4 +558,20 @@ public R visitTopK( org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode node, C context) { return visitPlan(node, context); } + + public R visitCreateTableDevice(CreateTableDeviceNode node, C context) { + return visitPlan(node, context); + } + + public R visitTableDeviceScan(TableDeviceScanNode node, C context) { + return visitPlan(node, context); + } + + public R visitTableDeviceFetch(TableDeviceFetchNode node, C context) { + return visitPlan(node, context); + } + + public R visitDeleteTableDevice(DeleteTableDeviceNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java new file mode 100644 index 000000000000..1a517c95ebae --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class TableDeviceFetchNode extends SchemaQueryScanNode { + + private String database; + + private String tableName; + + private List deviceIdList; + + private List columnHeaderList; + + private TRegionReplicaSet schemaRegionReplicaSet; + + public TableDeviceFetchNode(PlanNodeId id) { + super(id); + } + + public TableDeviceFetchNode( + PlanNodeId id, + String database, + String tableName, + List deviceIdList, + List columnHeaderList, + TRegionReplicaSet regionReplicaSet) { + super(id); + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + this.columnHeaderList = columnHeaderList; + this.schemaRegionReplicaSet = regionReplicaSet; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + public List getColumnHeaderList() { + return columnHeaderList; + } + + @Override + public void open() throws Exception {} + + @Override + public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { + this.schemaRegionReplicaSet = regionReplicaSet; + } + + @Override + public void close() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public PlanNode clone() { + return new TableDeviceFetchNode( + getPlanNodeId(), + database, + tableName, + deviceIdList, + columnHeaderList, + schemaRegionReplicaSet); + } + + @Override + public List getOutputColumnNames() { + return columnHeaderList.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_DEVICE_FETCH.serialize(byteBuffer); + ReadWriteIOUtils.write(database, byteBuffer); + ReadWriteIOUtils.write(tableName, byteBuffer); + + ReadWriteIOUtils.write(deviceIdList.size(), byteBuffer); + for (String[] deviceId : deviceIdList) { + ReadWriteIOUtils.write(deviceId.length, byteBuffer); + for (String idValue : deviceId) { + ReadWriteIOUtils.write(idValue, byteBuffer); + } + } + + ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(byteBuffer); + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_DEVICE_FETCH.serialize(stream); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + ReadWriteIOUtils.write(deviceIdList.size(), stream); + for (String[] deviceId : deviceIdList) { + ReadWriteIOUtils.write(deviceId.length, stream); + for (String idValue : deviceId) { + ReadWriteIOUtils.write(idValue, stream); + } + } + + ReadWriteIOUtils.write(columnHeaderList.size(), stream); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(stream); + } + } + + public static TableDeviceFetchNode deserialize(ByteBuffer buffer) { + String database = ReadWriteIOUtils.readString(buffer); + String tableName = ReadWriteIOUtils.readString(buffer); + + int size = ReadWriteIOUtils.readInt(buffer); + List deviceIdList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + int length = ReadWriteIOUtils.readInt(buffer); + String[] nodes = new String[length]; + for (int j = 0; j < length; j++) { + nodes[j] = ReadWriteIOUtils.readString(buffer); + } + deviceIdList.add(nodes); + } + + size = ReadWriteIOUtils.readInt(buffer); + List columnHeaderList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnHeaderList.add(ColumnHeader.deserialize(buffer)); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableDeviceFetchNode( + planNodeId, database, tableName, deviceIdList, columnHeaderList, null); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitTableDeviceFetch(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceFetchNode)) return false; + if (!super.equals(o)) return false; + TableDeviceFetchNode that = (TableDeviceFetchNode) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(deviceIdList, that.deviceIdList) + && Objects.equals(columnHeaderList, that.columnHeaderList) + && Objects.equals(schemaRegionReplicaSet, that.schemaRegionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + database, + tableName, + deviceIdList, + columnHeaderList, + schemaRegionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java new file mode 100644 index 000000000000..64400d3dd1ef --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class TableDeviceScanNode extends SchemaQueryScanNode { + + private String database; + + private String tableName; + + private List> idDeterminedFilterList; + + private SchemaFilter idFuzzyFilter; + + private List columnHeaderList; + + private TRegionReplicaSet schemaRegionReplicaSet; + + public TableDeviceScanNode(PlanNodeId id) { + super(id); + } + + public TableDeviceScanNode( + PlanNodeId id, + String database, + String tableName, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, + List columnHeaderList, + TRegionReplicaSet regionReplicaSet) { + super(id); + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilter = idFuzzyFilter; + this.columnHeaderList = columnHeaderList; + this.schemaRegionReplicaSet = regionReplicaSet; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List> getIdDeterminedFilterList() { + return idDeterminedFilterList; + } + + public SchemaFilter getIdFuzzyFilter() { + return idFuzzyFilter; + } + + public List getColumnHeaderList() { + return columnHeaderList; + } + + @Override + public List getPathPatternList() { + return DeviceFilterUtil.convertToDevicePattern( + database, + DataNodeTableCache.getInstance().getTable(database, tableName), + idDeterminedFilterList); + } + + @Override + public void open() throws Exception {} + + @Override + public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { + this.schemaRegionReplicaSet = regionReplicaSet; + } + + @Override + public void close() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public PlanNode clone() { + return new TableDeviceScanNode( + getPlanNodeId(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilter, + columnHeaderList, + schemaRegionReplicaSet); + } + + @Override + public List getOutputColumnNames() { + return columnHeaderList.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_DEVICE_SCAN.serialize(byteBuffer); + ReadWriteIOUtils.write(database, byteBuffer); + ReadWriteIOUtils.write(tableName, byteBuffer); + + ReadWriteIOUtils.write(idDeterminedFilterList.size(), byteBuffer); + for (List filterList : idDeterminedFilterList) { + ReadWriteIOUtils.write(filterList.size(), byteBuffer); + for (SchemaFilter schemaFilter : filterList) { + SchemaFilter.serialize(schemaFilter, byteBuffer); + } + } + + SchemaFilter.serialize(idFuzzyFilter, byteBuffer); + + ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(byteBuffer); + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_DEVICE_SCAN.serialize(stream); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + ReadWriteIOUtils.write(idDeterminedFilterList.size(), stream); + for (List filterList : idDeterminedFilterList) { + ReadWriteIOUtils.write(filterList.size(), stream); + for (SchemaFilter schemaFilter : filterList) { + SchemaFilter.serialize(schemaFilter, stream); + } + } + + SchemaFilter.serialize(idFuzzyFilter, stream); + + ReadWriteIOUtils.write(columnHeaderList.size(), stream); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(stream); + } + } + + public static TableDeviceScanNode deserialize(ByteBuffer buffer) { + String database = ReadWriteIOUtils.readString(buffer); + String tableName = ReadWriteIOUtils.readString(buffer); + + int size = ReadWriteIOUtils.readInt(buffer); + List> idDeterminedFilterList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + int singleSize = ReadWriteIOUtils.readInt(buffer); + idDeterminedFilterList.add(new ArrayList<>(singleSize)); + for (int k = 0; k < singleSize; k++) { + idDeterminedFilterList.get(i).add(SchemaFilter.deserialize(buffer)); + } + } + + SchemaFilter idFuzzyFilter = SchemaFilter.deserialize(buffer); + + size = ReadWriteIOUtils.readInt(buffer); + List columnHeaderList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnHeaderList.add(ColumnHeader.deserialize(buffer)); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableDeviceScanNode( + planNodeId, + database, + tableName, + idDeterminedFilterList, + idFuzzyFilter, + columnHeaderList, + null); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitTableDeviceScan(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceScanNode)) return false; + if (!super.equals(o)) return false; + TableDeviceScanNode that = (TableDeviceScanNode) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(idDeterminedFilterList, that.idDeterminedFilterList) + && Objects.equals(idFuzzyFilter, that.idFuzzyFilter) + && Objects.equals(columnHeaderList, that.columnHeaderList) + && Objects.equals(schemaRegionReplicaSet, that.schemaRegionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilter, + columnHeaderList, + schemaRegionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java new file mode 100644 index 000000000000..229191129b4d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java @@ -0,0 +1,217 @@ +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class CreateTableDeviceNode extends WritePlanNode { + + private final List devicePathList; + + private final List attributeNameList; + + private final List> attributeValueList; + + private TRegionReplicaSet regionReplicaSet; + + public CreateTableDeviceNode( + PlanNodeId id, + List devicePathList, + List attributeNameList, + List> attributeValueList) { + super(id); + this.devicePathList = devicePathList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + } + + public CreateTableDeviceNode( + PlanNodeId id, + TRegionReplicaSet regionReplicaSet, + List devicePathList, + List attributeNameList, + List> attributeValueList) { + super(id); + this.devicePathList = devicePathList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + this.regionReplicaSet = regionReplicaSet; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.CREATE_TABLE_DEVICE; + } + + public List getDevicePathList() { + return devicePathList; + } + + public List getAttributeNameList() { + return attributeNameList; + } + + public List> getAttributeValueList() { + return attributeValueList; + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return regionReplicaSet; + } + + @Override + public List getChildren() { + return new ArrayList<>(); + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return new CreateTableDeviceNode( + getPlanNodeId(), regionReplicaSet, devicePathList, attributeNameList, attributeValueList); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.CREATE_TABLE_DEVICE.serialize(byteBuffer); + ReadWriteIOUtils.write(devicePathList.size(), byteBuffer); + for (PartialPath deviceId : devicePathList) { + deviceId.serialize(byteBuffer); + } + ReadWriteIOUtils.write(attributeNameList.size(), byteBuffer); + for (String attributeName : attributeNameList) { + ReadWriteIOUtils.write(attributeName, byteBuffer); + } + ReadWriteIOUtils.write(attributeValueList.size(), byteBuffer); + for (List deviceValueList : attributeValueList) { + ReadWriteIOUtils.write(deviceValueList.size(), byteBuffer); + for (String value : deviceValueList) { + ReadWriteIOUtils.write(value, byteBuffer); + } + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.CREATE_TABLE_DEVICE.serialize(stream); + ReadWriteIOUtils.write(devicePathList.size(), stream); + for (PartialPath deviceId : devicePathList) { + deviceId.serialize(stream); + } + ReadWriteIOUtils.write(attributeNameList.size(), stream); + for (String attributeName : attributeNameList) { + ReadWriteIOUtils.write(attributeName, stream); + } + for (List deviceValueList : attributeValueList) { + for (String value : deviceValueList) { + ReadWriteIOUtils.write(value, stream); + } + } + } + + public static CreateTableDeviceNode deserialize(ByteBuffer buffer) { + int deviceNum = ReadWriteIOUtils.readInt(buffer); + List devicePathList = new ArrayList<>(deviceNum); + for (int i = 0; i < deviceNum; i++) { + devicePathList.add((PartialPath) PathDeserializeUtil.deserialize(buffer)); + } + int attributeNameNum = ReadWriteIOUtils.readInt(buffer); + List attributeNameList = new ArrayList<>(attributeNameNum); + for (int i = 0; i < attributeNameNum; i++) { + attributeNameList.add(ReadWriteIOUtils.readString(buffer)); + } + List> attributeValueList = new ArrayList<>(deviceNum); + for (int i = 0; i < deviceNum; i++) { + List deviceValueList = new ArrayList<>(attributeNameNum); + for (int j = 0; j < attributeNameNum; j++) { + deviceValueList.add(ReadWriteIOUtils.readString(buffer)); + } + attributeValueList.add(deviceValueList); + } + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new CreateTableDeviceNode( + planNodeId, devicePathList, attributeNameList, attributeValueList); + } + + @Override + public List splitByPartition(Analysis analysis) { + Map> splitMap = new HashMap<>(); + for (int i = 0; i < devicePathList.size(); i++) { + TRegionReplicaSet regionReplicaSet = + analysis + .getSchemaPartitionInfo() + .getSchemaRegionReplicaSet(devicePathList.get(i).getFullPath()); + splitMap.computeIfAbsent(regionReplicaSet, k -> new ArrayList<>()).add(i); + } + List result = new ArrayList<>(splitMap.size()); + for (Map.Entry> entry : splitMap.entrySet()) { + List subDevicePathList = new ArrayList<>(entry.getValue().size()); + List> subAttributeValueList = new ArrayList<>(entry.getValue().size()); + for (Integer index : entry.getValue()) { + subDevicePathList.add(devicePathList.get(index)); + subAttributeValueList.add(attributeValueList.get(index)); + } + result.add( + new CreateTableDeviceNode( + getPlanNodeId(), + entry.getKey(), + subDevicePathList, + attributeNameList, + subAttributeValueList)); + } + return result; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateTableDevice(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof CreateTableDeviceNode)) return false; + if (!super.equals(o)) return false; + CreateTableDeviceNode that = (CreateTableDeviceNode) o; + return Objects.equals(devicePathList, that.devicePathList) + && Objects.equals(attributeNameList, that.attributeNameList) + && Objects.equals(attributeValueList, that.attributeValueList) + && Objects.equals(regionReplicaSet, that.regionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), devicePathList, attributeNameList, attributeValueList, regionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java new file mode 100644 index 000000000000..0c54b9bb0a1b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +public class DeleteTableDeviceNode extends PlanNode { + + private final String table; + + public DeleteTableDeviceNode(PlanNodeId id, String table) { + super(id); + this.table = table; + } + + public String getTable() { + return table; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitDeleteTableDevice(this, context); + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return new DeleteTableDeviceNode(getPlanNodeId(), table); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.DELETE_TABLE_DEVICE.serialize(byteBuffer); + ReadWriteIOUtils.write(table, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.DELETE_TABLE_DEVICE.serialize(stream); + ReadWriteIOUtils.write(table, stream); + } + + public static DeleteTableDeviceNode deserialize(ByteBuffer buffer) { + String table = ReadWriteIOUtils.readString(buffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new DeleteTableDeviceNode(planNodeId, table); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeleteTableDeviceNode)) return false; + if (!super.equals(o)) return false; + DeleteTableDeviceNode that = (DeleteTableDeviceNode) o; + return Objects.equals(table, that.table); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), table); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java new file mode 100644 index 000000000000..d4811ae4ad8b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.relational.sql.tree.BetweenPredicate; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.IfExpression; +import org.apache.iotdb.db.relational.sql.tree.InPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNotNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.LikePredicate; +import org.apache.iotdb.db.relational.sql.tree.Literal; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.NotExpression; +import org.apache.iotdb.db.relational.sql.tree.NullIfExpression; +import org.apache.iotdb.db.relational.sql.tree.SearchedCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.StringLiteral; +import org.apache.iotdb.db.relational.sql.tree.SymbolReference; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ConvertSchemaPredicateToFilterVisitor + extends PredicateVisitor { + + @Override + protected SchemaFilter visitInPredicate(InPredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitIsNullPredicate(IsNullPredicate node, Context context) { + String columnName = ((SymbolReference) node.getValue()).getName(); + if (context + .table + .getColumnSchema(columnName) + .getColumnCategory() + .equals(TsTableColumnCategory.ID)) { + return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), null); + } else { + context.hasAttribute = true; + return new DeviceAttributeFilter(columnName, null); + } + } + + @Override + protected SchemaFilter visitIsNotNullPredicate(IsNotNullPredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitLikePredicate(LikePredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitLogicalExpression(LogicalExpression node, Context context) { + // the operator of the logical expression shall be OR + return new OrFilter( + node.getTerms().get(0).accept(this, context), node.getTerms().get(1).accept(this, context)); + } + + @Override + protected SchemaFilter visitNotExpression(NotExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitComparisonExpression(ComparisonExpression node, Context context) { + String columnName; + String value; + if (node.getLeft() instanceof Literal) { + value = ((StringLiteral) (node.getLeft())).getValue(); + if (node.getRight() instanceof Identifier) { + columnName = ((Identifier) (node.getRight())).getValue(); + } else { + columnName = ((SymbolReference) (node.getRight())).getName(); + } + } else { + value = ((StringLiteral) (node.getRight())).getValue(); + if (node.getLeft() instanceof Identifier) { + columnName = ((Identifier) (node.getLeft())).getValue(); + } else { + columnName = ((SymbolReference) (node.getLeft())).getName(); + } + } + if (context + .table + .getColumnSchema(columnName) + .getColumnCategory() + .equals(TsTableColumnCategory.ID)) { + return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), value); + } else { + context.hasAttribute = true; + return new DeviceAttributeFilter(columnName, value); + } + } + + @Override + protected SchemaFilter visitSimpleCaseExpression(SimpleCaseExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitSearchedCaseExpression(SearchedCaseExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitIfExpression(IfExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitNullIfExpression(NullIfExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitBetweenPredicate(BetweenPredicate node, Context context) { + return visitExpression(node, context); + } + + public static class Context { + + private final TsTable table; + private final Map idColumeIndexMap; + + private boolean hasAttribute = false; + + public Context(TsTable table) { + this.table = table; + this.idColumeIndexMap = getIdColumnIndex(table); + } + + private Map getIdColumnIndex(TsTable table) { + Map map = new HashMap<>(); + List columnSchemaList = table.getColumnList(); + int idIndex = 0; + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + map.put(columnSchema.getColumnName(), idIndex); + idIndex++; + } + } + return map; + } + + public boolean hasAttribute() { + return hasAttribute; + } + + public void reset() { + hasAttribute = false; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java index 3f7488205df5..cd2d90974473 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java @@ -60,7 +60,7 @@ protected Boolean visitInPredicate(InPredicate node, Void context) { @Override protected Boolean visitIsNullPredicate(IsNullPredicate node, Void context) { - return Boolean.FALSE; + return isIdOrAttributeColumn(node.getValue()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java new file mode 100644 index 000000000000..56b9b040c6e9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DeviceInCacheFilterVisitor extends SchemaFilterVisitor { + + private final Map attributeIndexMap = new HashMap<>(); + + DeviceInCacheFilterVisitor(List attributeColumns) { + for (int i = 0; i < attributeColumns.size(); i++) { + attributeIndexMap.put(attributeColumns.get(i), i); + } + } + + @Override + protected boolean visitNode(SchemaFilter filter, DeviceEntry deviceEntry) { + return false; + } + + @Override + public boolean visitDeviceIdFilter(DeviceIdFilter filter, DeviceEntry deviceEntry) { + IDeviceID deviceID = deviceEntry.getDeviceID(); + // the first segment is "db.table", skip it + if (deviceID.segmentNum() < filter.getIndex() + 1) { + return false; + } else { + return deviceID.segment(filter.getIndex() + 1).equals(filter.getValue()); + } + } + + @Override + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, DeviceEntry deviceEntry) { + return filter + .getValue() + .equals(deviceEntry.getAttributeColumnValues().get(attributeIndexMap.get(filter.getKey()))); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java new file mode 100644 index 000000000000..7bc05c939255 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import java.util.List; + +public interface ITableDeviceSchemaValidation { + + String getDatabase(); + + String getTableName(); + + List getDeviceIdList(); + + List getAttributeColumnNameList(); + + List> getAttributeValue(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java new file mode 100644 index 000000000000..784bcc743eee --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -0,0 +1,701 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.AndFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertSchemaPredicateToFilterVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceId; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.read.common.type.UnknownType; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; + +public class TableModelSchemaFetcher { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableModelSchemaFetcher.class); + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final Coordinator coordinator = Coordinator.getInstance(); + + private final TableDeviceSchemaCache cache = new TableDeviceSchemaCache(); + + private final ClusterConfigTaskExecutor configTaskExecutor = + ClusterConfigTaskExecutor.getInstance(); + + private static class TableModelSchemaFetcherHolder { + private static final TableModelSchemaFetcher INSTANCE = new TableModelSchemaFetcher(); + } + + public static TableModelSchemaFetcher getInstance() { + return TableModelSchemaFetcherHolder.INSTANCE; + } + + private TableModelSchemaFetcher() { + // do nothing + } + + public void invalidateDeviceCache(String database, String tableName) { + cache.invalidate(database, tableName); + } + + // This method return all the existing column schemas in the target table. + // When table or column is missing, this method will execute auto creation. + // When using SQL, the columnSchemaList could be null and there won't be any validation. + // All input column schemas will be validated and auto created when necessary. + // When the input dataType or category of one column is null, the column cannot be auto created. + public TableSchema validateTableHeaderSchema( + String database, TableSchema tableSchema, MPPQueryContext context) { + List inputColumnList = tableSchema.getColumns(); + TsTable table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); + List missingColumnList = new ArrayList<>(); + List resultColumnList = new ArrayList<>(); + + // first round validate, check existing schema + if (table == null) { + if (inputColumnList == null) { + throw new SemanticException("Unknown column names. Cannot auto create table."); + } + // check arguments for table auto creation + for (ColumnSchema columnSchema : inputColumnList) { + if (columnSchema.getColumnCategory() == null) { + throw new SemanticException("Unknown column category. Cannot auto create table."); + } + if (columnSchema.getType() == null) { + throw new IllegalArgumentException("Unknown column data type. Cannot auto create table."); + } + missingColumnList.add(columnSchema); + } + } else if (inputColumnList == null) { + // SQL insert without columnName, nothing to check + } else { + for (int i = 0; i < inputColumnList.size(); i++) { + ColumnSchema columnSchema = inputColumnList.get(i); + TsTableColumnSchema existingColumn = table.getColumnSchema(columnSchema.getName()); + if (existingColumn == null) { + // check arguments for column auto creation + if (columnSchema.getColumnCategory() == null) { + throw new IllegalArgumentException( + "Unknown column category. Cannot auto create column."); + } + if (columnSchema.getType() == null) { + throw new IllegalArgumentException( + "Unknown column data type. Cannot auto create column."); + } + missingColumnList.add(columnSchema); + } else { + // check and validate column data type and category + if (!columnSchema.getType().equals(UnknownType.UNKNOWN) + && !TypeFactory.getType(existingColumn.getDataType()) + .equals(columnSchema.getType())) { + throw new SemanticException( + String.format("Wrong data type at column %s.", columnSchema.getName())); + } + if (columnSchema.getColumnCategory() != null + && !existingColumn.getColumnCategory().equals(columnSchema.getColumnCategory())) { + throw new SemanticException( + String.format("Wrong category at column %s.", columnSchema.getName())); + } + } + } + } + + // auto create missing table or columns + if (table == null) { + autoCreateTable(database, tableSchema, context); + table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); + } else if (inputColumnList == null) { + // do nothing + } else { + if (!missingColumnList.isEmpty()) { + autoCreateColumn(database, tableSchema.getTableName(), missingColumnList, context); + } + } + table + .getColumnList() + .forEach( + o -> + resultColumnList.add( + new ColumnSchema( + o.getColumnName(), + TypeFactory.getType(o.getDataType()), + false, + o.getColumnCategory()))); + return new TableSchema(tableSchema.getTableName(), resultColumnList); + } + + public void autoCreateTable(String database, TableSchema tableSchema, MPPQueryContext context) { + throw new SemanticException(new TableNotExistsException(database, tableSchema.getTableName())); + } + + private void autoCreateColumn( + String database, + String tableName, + List inputColumnList, + MPPQueryContext context) { + AlterTableAddColumnTask task = + new AlterTableAddColumnTask( + database, tableName, inputColumnList, context.getQueryId().getId()); + try { + ListenableFuture future = task.execute(configTaskExecutor); + ConfigTaskResult result = future.get(); + if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + "Auto add table column failed.", result.getStatusCode().getStatusCode())); + } + } catch (ExecutionException | InterruptedException e) { + LOGGER.warn("Auto add table column failed.", e); + throw new RuntimeException(e); + } + } + + public void validateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) { + ValidateResult validateResult = validateDeviceSchemaInCache(schemaValidation); + + if (!validateResult.missingDeviceIndexList.isEmpty() + || !validateResult.attributeMissingInCacheDeviceIndexList.isEmpty()) { + validateResult = fetchAndValidateDeviceSchema(schemaValidation, validateResult, context); + } + + if (!validateResult.missingDeviceIndexList.isEmpty() + || !validateResult.attributeUpdateDeviceIndexList.isEmpty()) { + autoCreateDeviceSchema(schemaValidation, validateResult, context); + } + } + + private ValidateResult validateDeviceSchemaInCache( + ITableDeviceSchemaValidation schemaValidation) { + ValidateResult result = new ValidateResult(); + String database = schemaValidation.getDatabase(); + String tableName = schemaValidation.getTableName(); + List deviceIdList = schemaValidation.getDeviceIdList(); + List attributeKeyList = schemaValidation.getAttributeColumnNameList(); + List> attributeValueList = schemaValidation.getAttributeValue(); + + for (int i = 0, size = deviceIdList.size(); i < size; i++) { + Map attributeMap = + cache.getDeviceAttribute(database, tableName, deviceIdList.get(i)); + if (attributeMap == null) { + result.missingDeviceIndexList.add(i); + continue; + } + List deviceAttributeValueList = attributeValueList.get(i); + for (int j = 0; j < attributeKeyList.size(); j++) { + String value = attributeMap.get(attributeKeyList.get(j)); + if (value == null) { + result.attributeMissingInCacheDeviceIndexList.add(i); + break; + } else if (!value.equals(deviceAttributeValueList.get(j))) { + result.attributeUpdateDeviceIndexList.add(i); + break; + } + } + } + return result; + } + + private ValidateResult fetchAndValidateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, + ValidateResult previousValidateResult, + MPPQueryContext context) { + List targetDeviceList = + new ArrayList<>( + previousValidateResult.missingDeviceIndexList.size() + + previousValidateResult.attributeMissingInCacheDeviceIndexList.size()); + for (int index : previousValidateResult.missingDeviceIndexList) { + targetDeviceList.add(schemaValidation.getDeviceIdList().get(index)); + } + for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) { + targetDeviceList.add(schemaValidation.getDeviceIdList().get(index)); + } + + Map> fetchedDeviceSchema = + fetchMissingDeviceSchema( + new FetchTableDevicesStatement( + schemaValidation.getDatabase(), schemaValidation.getTableName(), targetDeviceList), + context); + + for (Map.Entry> entry : fetchedDeviceSchema.entrySet()) { + cache.put( + schemaValidation.getDatabase(), + schemaValidation.getTableName(), + entry.getKey().getIdValues(), + entry.getValue()); + } + + ValidateResult result = new ValidateResult(); + for (int index : previousValidateResult.missingDeviceIndexList) { + String[] deviceId = schemaValidation.getDeviceIdList().get(index); + Map attributeMap = fetchedDeviceSchema.get(new TableDeviceId(deviceId)); + if (attributeMap == null) { + result.missingDeviceIndexList.add(index); + } else { + for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { + String key = schemaValidation.getAttributeColumnNameList().get(j); + String value = attributeMap.get(key); + if (value == null + || !value.equals(schemaValidation.getAttributeValue().get(index).get(j))) { + result.attributeUpdateDeviceIndexList.add(index); + break; + } + } + } + } + + for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) { + String[] deviceId = schemaValidation.getDeviceIdList().get(index); + Map attributeMap = fetchedDeviceSchema.get(new TableDeviceId(deviceId)); + if (attributeMap == null) { + throw new IllegalStateException("Device shall exist but not exist."); + } else { + for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { + String key = schemaValidation.getAttributeColumnNameList().get(j); + String value = attributeMap.get(key); + if (value == null + || !value.equals(schemaValidation.getAttributeValue().get(index).get(j))) { + result.attributeUpdateDeviceIndexList.add(index); + break; + } + } + } + } + + result.attributeUpdateDeviceIndexList.addAll( + previousValidateResult.attributeUpdateDeviceIndexList); + + return result; + } + + private Map> fetchMissingDeviceSchema( + FetchTableDevicesStatement statement, MPPQueryContext context) { + long queryId = SessionManager.getInstance().requestQueryId(); + Throwable t = null; + + String database = statement.getDatabase(); + String table = statement.getTableName(); + TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + statement, + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + config.getQueryTimeoutThreshold()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + + List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + Map> fetchedDeviceSchema = new HashMap<>(); + + try { + while (coordinator.getQueryExecution(queryId).hasNextResult()) { + Optional tsBlock; + try { + tsBlock = coordinator.getQueryExecution(queryId).getBatchResult(); + } catch (IoTDBException e) { + t = e; + throw new RuntimeException("Fetch Table Device Schema failed. ", e); + } + if (!tsBlock.isPresent() || tsBlock.get().isEmpty()) { + break; + } + Column[] columns = tsBlock.get().getValueColumns(); + for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { + String[] nodes = new String[idLength]; + int idIndex = 0; + Map attributeMap = new HashMap<>(); + for (int j = 0; j < columnHeaderList.size(); j++) { + TsTableColumnSchema columnSchema = + tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + if (columns[j].isNull(i)) { + nodes[idIndex] = null; + } else { + nodes[idIndex] = columns[j].getBinary(i).toString(); + } + idIndex++; + } else { + if (columns[j].isNull(i)) { + attributeMap.put(columnSchema.getColumnName(), null); + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } + } + } + fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap); + } + } + } catch (Throwable throwable) { + t = throwable; + throw throwable; + } finally { + coordinator.cleanupQueryExecution(queryId, null, t); + } + return fetchedDeviceSchema; + } + + private void autoCreateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, + ValidateResult previousValidateResult, + MPPQueryContext context) { + List deviceIdList = + new ArrayList<>( + previousValidateResult.missingDeviceIndexList.size() + + previousValidateResult.attributeUpdateDeviceIndexList.size()); + List> attributeValueList = new ArrayList<>(deviceIdList.size()); + for (int index : previousValidateResult.missingDeviceIndexList) { + deviceIdList.add(schemaValidation.getDeviceIdList().get(index)); + attributeValueList.add(schemaValidation.getAttributeValue().get(index)); + } + for (int index : previousValidateResult.attributeUpdateDeviceIndexList) { + deviceIdList.add(schemaValidation.getDeviceIdList().get(index)); + attributeValueList.add(schemaValidation.getAttributeValue().get(index)); + } + + CreateTableDeviceStatement statement = + new CreateTableDeviceStatement( + schemaValidation.getDatabase(), + schemaValidation.getTableName(), + deviceIdList, + schemaValidation.getAttributeColumnNameList(), + attributeValueList); + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + statement, + SessionManager.getInstance().requestQueryId(), + context == null ? null : context.getSession(), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + context == null || context.getQueryType().equals(QueryType.WRITE) + ? config.getQueryTimeoutThreshold() + : context.getTimeOut()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + } + + private static class ValidateResult { + final List missingDeviceIndexList = new ArrayList<>(); + final List attributeMissingInCacheDeviceIndexList = new ArrayList<>(); + final List attributeUpdateDeviceIndexList = new ArrayList<>(); + } + + public List fetchDeviceSchema( + String database, + String table, + List expressionList, + List attributeColumns) { + List deviceEntryList = new ArrayList<>(); + + TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + Pair, List> filters = + transformExpression(expressionList, tableInstance); + List idFilters = filters.getLeft(); + List attributeFilters = filters.getRight(); + DeviceInCacheFilterVisitor filterVisitor = new DeviceInCacheFilterVisitor(attributeColumns); + SchemaFilter attributeFilter = getAttributeFilter(attributeFilters); + + List> idPatternList = + DeviceFilterUtil.convertSchemaFilterToOrConcatList(idFilters); + List> idFilterListForFetch = new ArrayList<>(); + boolean cacheFetchedDevice = true; + for (int i = 0; i < idPatternList.size(); i++) { + SchemaFilterCheckResult checkResult = + checkIdFilterAndTryGetDeviceInCache( + deviceEntryList, + database, + tableInstance, + idPatternList.get(i), + o -> attributeFilter == null || filterVisitor.process(attributeFilter, o), + attributeColumns); + if (checkResult.needFetch) { + idFilterListForFetch.add(idPatternList.get(i)); + if (!checkResult.isIdDetermined) { + cacheFetchedDevice = false; + } + } + } + + if (!idFilterListForFetch.isEmpty()) { + fetchMissingDeviceSchemaForQuery( + database, + tableInstance, + attributeColumns, + idFilterListForFetch, + attributeFilter, + deviceEntryList, + cacheFetchedDevice); + } + + Set set = new LinkedHashSet<>(deviceEntryList); + return new ArrayList<>(set); + } + + private Pair, List> transformExpression( + List expressionList, TsTable table) { + List idDeterminedFilters = new ArrayList<>(); + List idFuzzyFilters = new ArrayList<>(); + ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor(); + ConvertSchemaPredicateToFilterVisitor.Context context = + new ConvertSchemaPredicateToFilterVisitor.Context(table); + for (Expression expression : expressionList) { + if (expression == null) { + continue; + } + context.reset(); + SchemaFilter schemaFilter = expression.accept(visitor, context); + if (context.hasAttribute()) { + idFuzzyFilters.add(schemaFilter); + } else { + idDeterminedFilters.add(schemaFilter); + } + } + return new Pair<>(idDeterminedFilters, idFuzzyFilters); + } + + // return whether this condition shall be used for remote fetch + private SchemaFilterCheckResult checkIdFilterAndTryGetDeviceInCache( + List deviceEntryList, + String database, + TsTable tableInstance, + List idFilters, + Predicate check, + List attributeColumns) { + String[] idValues = new String[tableInstance.getIdNums()]; + for (SchemaFilter schemaFilter : idFilters) { + DeviceIdFilter idFilter = (DeviceIdFilter) schemaFilter; + if (idValues[idFilter.getIndex()] == null) { + idValues[idFilter.getIndex()] = idFilter.getValue(); + } else { + // conflict filter + return new SchemaFilterCheckResult(false, false); + } + } + if (idFilters.size() < idValues.length) { + return new SchemaFilterCheckResult(true, false); + } + Map attributeMap = + cache.getDeviceAttribute(database, tableInstance.getTableName(), idValues); + if (attributeMap == null) { + return new SchemaFilterCheckResult(true, true); + } + List attributeValues = new ArrayList<>(attributeColumns.size()); + for (String attributeKey : attributeColumns) { + String value = attributeMap.get(attributeKey); + if (value == null) { + return new SchemaFilterCheckResult(true, true); + } else { + attributeValues.add(value); + } + } + String[] deviceIdNodes = new String[idValues.length + 1]; + deviceIdNodes[0] = database + PATH_SEPARATOR + tableInstance.getTableName(); + System.arraycopy(idValues, 0, deviceIdNodes, 1, idValues.length); + DeviceEntry deviceEntry = + new DeviceEntry(new StringArrayDeviceID(deviceIdNodes), attributeValues); + if (check.test(deviceEntry)) { + deviceEntryList.add(deviceEntry); + } + return new SchemaFilterCheckResult(false, true); + } + + private static class SchemaFilterCheckResult { + boolean needFetch; + boolean isIdDetermined; + + SchemaFilterCheckResult(boolean needFetch, boolean isIdDetermined) { + this.needFetch = needFetch; + this.isIdDetermined = isIdDetermined; + } + } + + private void fetchMissingDeviceSchemaForQuery( + String database, + TsTable tableInstance, + List attributeColumns, + List> idPatternList, + SchemaFilter attributeFilter, + List deviceEntryList, + boolean cacheFetchedDevice) { + + String table = tableInstance.getTableName(); + + long queryId = SessionManager.getInstance().requestQueryId(); + ShowTableDevicesStatement statement = + new ShowTableDevicesStatement(database, table, idPatternList, attributeFilter); + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + statement, + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + config.getQueryTimeoutThreshold()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + + List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + Map attributeMap; + + Throwable t = null; + try { + while (coordinator.getQueryExecution(queryId).hasNextResult()) { + Optional tsBlock; + try { + tsBlock = coordinator.getQueryExecution(queryId).getBatchResult(); + } catch (IoTDBException e) { + t = e; + throw new RuntimeException("Fetch Table Device Schema failed. ", e); + } + if (!tsBlock.isPresent() || tsBlock.get().isEmpty()) { + break; + } + Column[] columns = tsBlock.get().getValueColumns(); + for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { + String[] nodes = new String[idLength + 1]; + nodes[0] = database + PATH_SEPARATOR + table; + int idIndex = 0; + attributeMap = new HashMap<>(); + for (int j = 0; j < columnHeaderList.size(); j++) { + TsTableColumnSchema columnSchema = + tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + if (columns[j].isNull(i)) { + nodes[idIndex + 1] = null; + } else { + nodes[idIndex + 1] = columns[j].getBinary(i).toString(); + } + idIndex++; + } else { + if (columns[j].isNull(i)) { + attributeMap.put(columnSchema.getColumnName(), null); + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } + } + } + IDeviceID deviceID = new StringArrayDeviceID(nodes); + deviceEntryList.add( + new DeviceEntry( + deviceID, + attributeColumns.stream().map(attributeMap::get).collect(Collectors.toList()))); + if (cacheFetchedDevice) { + cache.put(database, table, Arrays.copyOfRange(nodes, 1, nodes.length), attributeMap); + } + } + } + } catch (Throwable throwable) { + t = throwable; + throw throwable; + } finally { + coordinator.cleanupQueryExecution(queryId, null, t); + } + } + + private SchemaFilter getAttributeFilter(List filterList) { + if (filterList.isEmpty()) { + return null; + } + AndFilter andFilter; + SchemaFilter latestFilter = filterList.get(0); + for (int i = 1; i < filterList.size(); i++) { + andFilter = new AndFilter(latestFilter, filterList.get(i)); + latestFilter = andFilter; + } + return latestFilter; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java new file mode 100644 index 000000000000..cdbee6b2a7cc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java @@ -0,0 +1,3 @@ +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +public class TableModelSchemaValidator {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java new file mode 100644 index 000000000000..b37f57a9574a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java @@ -0,0 +1,9 @@ +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +public class CacheMemoryControlUtil { + + public static int estimateStringSize(String string) { + // each char takes 2B in Java + return string == null ? 0 : 32 + 2 * string.length(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java new file mode 100644 index 000000000000..7d60e91111c4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableDeviceCacheEntry { + + private final Map attributeMap; + + public TableDeviceCacheEntry() { + attributeMap = new ConcurrentHashMap<>(); + } + + public TableDeviceCacheEntry(Map attributeMap) { + this.attributeMap = new HashMap<>(attributeMap); + } + + public String getAttribute(String key) { + return attributeMap.get(key); + } + + public Map getAttributeMap() { + return attributeMap; + } + + public int estimateSize() { + int size = 8; + for (Map.Entry entry : attributeMap.entrySet()) { + size += estimateStringSize(entry.getKey()) + estimateStringSize(entry.getValue()); + } + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java new file mode 100644 index 000000000000..a40bdf54821d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.Arrays; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableDeviceId { + + private final String[] idValues; + + public TableDeviceId(String[] idValues) { + this.idValues = idValues; + } + + public String getIdValue(int index) { + return idValues[index]; + } + + public String[] getIdValues() { + return idValues; + } + + public int estimateSize() { + int size = 8 + 8 + 8 + 4; // object header + reference + String[] header + String.length + for (String node : idValues) { + size += estimateStringSize(node); + } + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceId)) return false; + TableDeviceId that = (TableDeviceId) o; + return Arrays.equals(idValues, that.idValues); + } + + @Override + public int hashCode() { + return Arrays.hashCode(idValues); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java new file mode 100644 index 000000000000..4c17dbfadff5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; + +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TableDeviceSchemaCache { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final IDualKeyCache dualKeyCache; + + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); + + public TableDeviceSchemaCache() { + DualKeyCacheBuilder dualKeyCacheBuilder = + new DualKeyCacheBuilder<>(); + dualKeyCache = + dualKeyCacheBuilder + .cacheEvictionPolicy( + DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) + .memoryCapacity(config.getAllocateMemoryForSchemaCache()) + .firstKeySizeComputer(TableId::estimateSize) + .secondKeySizeComputer(TableDeviceId::estimateSize) + .valueSizeComputer(TableDeviceCacheEntry::estimateSize) + .build(); + } + + public Map getDeviceAttribute( + String database, String tableName, String[] deviceId) { + readWriteLock.readLock().lock(); + try { + TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(database, tableName), new TableDeviceId(deviceId)); + return entry == null ? null : entry.getAttributeMap(); + } finally { + readWriteLock.readLock().unlock(); + } + } + + public void put( + String database, String tableName, String[] deviceId, Map attributeMap) { + readWriteLock.readLock().lock(); + try { + dualKeyCache.put( + new TableId(database, tableName), + new TableDeviceId(deviceId), + new TableDeviceCacheEntry(attributeMap)); + } finally { + readWriteLock.readLock().unlock(); + } + } + + public void invalidate(String database, String tableName) { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.invalidateAll(); + } finally { + readWriteLock.writeLock().unlock(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java new file mode 100644 index 000000000000..f858a591bfb5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.Objects; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableId { + + private final String database; + + private final String tableName; + + public TableId(String database, String tableName) { + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public int estimateSize() { + return 8 + 8 + 8 + estimateStringSize(database) + estimateStringSize(tableName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableId)) return false; + TableId tableId = (TableId) o; + return Objects.equals(database, tableId.database) + && Objects.equals(tableName, tableId.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java index 396352ab78da..09bb03daf885 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class DeviceEntry { @@ -73,6 +74,20 @@ public static DeviceEntry deserialize(ByteBuffer byteBuffer) { return new DeviceEntry(iDeviceID, attributeColumnValues); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeviceEntry)) return false; + DeviceEntry that = (DeviceEntry) o; + return Objects.equals(deviceID, that.deviceID) + && Objects.equals(attributeColumnValues, that.attributeColumnValues); + } + + @Override + public int hashCode() { + return Objects.hash(deviceID, attributeColumnValues); + } + @Override public String toString() { return "DeviceEntry{" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index d25e93903cd3..0b5ec9f4458b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -19,10 +19,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; import org.apache.iotdb.commons.udf.builtin.BuiltinScalarFunction; +import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; @@ -30,17 +33,16 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature; import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.TypeFactory; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -53,14 +55,31 @@ public class TableMetadataImpl implements Metadata { private final TypeManager typeManager = new InternalTypeManager(); + private final DataNodeTableCache tableCache = DataNodeTableCache.getInstance(); + @Override public boolean tableExists(QualifiedObjectName name) { - return false; + return tableCache.getTable(name.getDatabaseName(), name.getObjectName()) != null; } @Override public Optional getTableSchema(SessionInfo session, QualifiedObjectName name) { - return null; + TsTable table = tableCache.getTable(name.getDatabaseName(), name.getObjectName()); + if (table == null) { + throw new SemanticException( + new TableNotExistsException(name.getDatabaseName(), name.getObjectName())); + } + List columnSchemaList = + table.getColumnList().stream() + .map( + o -> + new ColumnSchema( + o.getColumnName(), + TypeFactory.getType(o.getDataType()), + false, + o.getColumnCategory())) + .collect(Collectors.toList()); + return Optional.of(new TableSchema(table.getTableName(), columnSchemaList)); } @Override @@ -262,13 +281,12 @@ public List indexScan( QualifiedObjectName tableName, List expressionList, List attributeColumns) { - // fixme, perfect the real metadata impl - List result = new ArrayList<>(); - IDeviceID deviceID1 = new StringArrayDeviceID("db.table1", "beijing", "a_1"); - IDeviceID deviceID2 = new StringArrayDeviceID("db.table1", "beijing", "b_1"); - result.add(new DeviceEntry(deviceID1, Arrays.asList("old", "low"))); - result.add(new DeviceEntry(deviceID2, Arrays.asList("new", "high"))); - return result; + return TableModelSchemaFetcher.getInstance() + .fetchDeviceSchema( + tableName.getDatabaseName(), + tableName.getObjectName(), + expressionList, + attributeColumns); } public static boolean isTwoNumericType(List argumentTypes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8dba82f490b..1473eda7b7f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -121,6 +122,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.SetThrottleQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; /** * This class provides a visitor of {@link StatementNode}, which can be extended to create a visitor @@ -600,4 +604,21 @@ public R visitShowCurrentTimestamp( ShowCurrentTimestampStatement showCurrentTimestampStatement, C context) { return visitStatement(showCurrentTimestampStatement, context); } + + public R visitInsertTable(InsertTableStatement insertTableStatement, C context) { + return visitStatement(insertTableStatement, context); + } + + public R visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, C context) { + return visitStatement(createTableDeviceStatement, context); + } + + public R visitShowTableDevices(ShowTableDevicesStatement statement, C context) { + return visitStatement(statement, context); + } + + public R visitFetchTableDevices(FetchTableDevicesStatement statement, C context) { + return visitStatement(statement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java new file mode 100644 index 000000000000..4fe5fac42ab9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.ITableDeviceSchemaValidation; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.Insert; +import org.apache.iotdb.db.relational.sql.tree.LongLiteral; +import org.apache.iotdb.db.relational.sql.tree.NullLiteral; +import org.apache.iotdb.db.relational.sql.tree.Row; +import org.apache.iotdb.db.relational.sql.tree.Values; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.UnknownType; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class InsertTableStatement extends Statement implements ITableDeviceSchemaValidation { + + private final Insert insert; + + private final String database; + + private List deviceIdList; + + private List attributeNameList; + + private List> attributeValueList; + + private InsertRowStatement insertRowStatement; + + private TableSchema tableSchema; + + public InsertTableStatement(IClientSession clientSession, Insert insert) { + this.insert = insert; + this.database = parseDatabase(clientSession); + this.tableSchema = parseTable(); + } + + private String parseDatabase(IClientSession clientSession) { + String database = clientSession.getDatabaseName(); + if (database == null) { + database = insert.getTable().getName().getPrefix().get().getSuffix(); + } + return database; + } + + private TableSchema parseTable() { + String tableName = insert.getTable().getName().getSuffix().toString(); + List columnSchemaList = new ArrayList<>(); + boolean hasColumn = insert.getColumns().isPresent(); + if (!hasColumn) { + return new TableSchema(tableName, null); + } + int size = insert.getColumns().get().size(); + List columnNameList = insert.getColumns().get(); + for (int i = 0; i < size; i++) { + String columnName = columnNameList.get(i).getValue(); + if (columnName.equalsIgnoreCase("time")) { + continue; + } + columnSchemaList.add(new ColumnSchema(columnName, UnknownType.UNKNOWN, false, null)); + } + return new TableSchema(tableName, columnSchemaList); + } + + private InsertRowStatement parseInsert() { + InsertRowStatement insertStatement = new InsertRowStatement(); + String tableName = tableSchema.getTableName(); + TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); + List values = + ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); + Map idColumnMap = new HashMap<>(); + Map attrColumnMap = new HashMap<>(); + Map measurementColumnMap = new HashMap<>(); + long time = 0L; + boolean hasColumn = insert.getColumns().isPresent(); + int size = hasColumn ? insert.getColumns().get().size() : table.getColumnNum(); + List columnNameList = hasColumn ? insert.getColumns().get() : null; + for (int i = 0; i < size; i++) { + String columnName = + hasColumn + ? columnNameList.get(i).getValue() + : table.getColumnList().get(i).getColumnName(); + if (columnName.equalsIgnoreCase("time")) { + columnName = "Time"; + } + TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); + if (columnSchema == null) { + throw new SemanticException(String.format("Unknown Column %s", columnName)); + } + TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); + if (values.get(i) instanceof NullLiteral) { + continue; + } + if (category.equals(TsTableColumnCategory.ID)) { + idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.ATTRIBUTE)) { + attrColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + measurementColumnMap.put(columnName, ((LongLiteral) values.get(i)).getValue()); + } else { + time = Long.parseLong(((LongLiteral) values.get(i)).getValue()); + } + } + String[] deviceIds = new String[table.getIdNums() + 3]; + deviceIds[0] = PATH_ROOT; + deviceIds[1] = database; + deviceIds[2] = tableName; + String[] measurements = new String[measurementColumnMap.size()]; + Object[] valueList = new Object[measurements.length]; + MeasurementSchema[] schemas = new MeasurementSchema[measurements.length]; + TSDataType[] dataTypes = new TSDataType[measurements.length]; + int idIndex = 0; + int measurementIndex = 0; + for (int i = 0; i < table.getColumnNum(); i++) { + TsTableColumnCategory category = table.getColumnList().get(i).getColumnCategory(); + if (category.equals(TsTableColumnCategory.ID)) { + String id = idColumnMap.get(table.getColumnList().get(i).getColumnName()); + deviceIds[3 + idIndex] = id; + idIndex++; + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + String measurement = table.getColumnList().get(i).getColumnName(); + if (measurementColumnMap.containsKey(measurement)) { + measurements[measurementIndex] = measurement; + valueList[measurementIndex] = measurementColumnMap.get(measurement); + schemas[measurementIndex] = + ((MeasurementColumnSchema) table.getColumnList().get(i)).getMeasurementSchema(); + dataTypes[measurementIndex] = schemas[measurementIndex].getType(); + measurementIndex++; + } + } + } + + this.deviceIdList = + Collections.singletonList(Arrays.copyOfRange(deviceIds, 3, deviceIds.length)); + this.attributeNameList = new ArrayList<>(attrColumnMap.keySet()); + this.attributeValueList = + Collections.singletonList( + attributeNameList.stream().map(attrColumnMap::get).collect(Collectors.toList())); + + insertStatement.setDevicePath(new PartialPath(deviceIds)); + TimestampPrecisionUtils.checkTimestampPrecision(time); + insertStatement.setTime(time); + insertStatement.setMeasurements(measurements); + insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]); + insertStatement.setValues(valueList); + insertStatement.setNeedInferType(true); + insertStatement.setAligned(true); + insertStatement.setMeasurementSchemas(schemas); + insertStatement.setDataTypes(dataTypes); + try { + for (int i = 0; i < measurements.length; i++) { + insertStatement.selfCheckDataTypes(i); + } + insertStatement.updateAfterSchemaValidation(); + } catch (Exception e) { + throw new SemanticException(e); + } + return insertStatement; + } + + public TableSchema getTableSchema() { + return tableSchema; + } + + public void setTableSchema(TableSchema tableSchema) { + this.tableSchema = tableSchema; + } + + public InsertRowStatement getInsertRowStatement() { + if (insertRowStatement == null) { + insertRowStatement = parseInsert(); + } + return insertRowStatement; + } + + @Override + public List getPaths() { + return insertRowStatement.getPaths(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitInsertTable(this, context); + } + + @Override + public String getDatabase() { + return database; + } + + @Override + public String getTableName() { + return tableSchema.getTableName(); + } + + @Override + public List getDeviceIdList() { + return deviceIdList; + } + + @Override + public List getAttributeColumnNameList() { + return attributeNameList; + } + + @Override + public List> getAttributeValue() { + return attributeValueList; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java new file mode 100644 index 000000000000..d16b016f3400 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java @@ -0,0 +1,84 @@ +package org.apache.iotdb.db.queryengine.plan.statement.table; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class CreateTableDeviceStatement extends Statement { + + private final String database; + + private final String table; + + private final List deviceIdList; + + private final List attributeNameList; + + private final List> attributeValueList; + + private transient List devicePathList; + + public CreateTableDeviceStatement( + String database, + String table, + List deviceIdList, + List attributeNameList, + List> attributeValueList) { + this.database = database; + this.table = table; + this.deviceIdList = deviceIdList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + } + + public String getDatabase() { + return database; + } + + public String getTable() { + return table; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + public List getAttributeNameList() { + return attributeNameList; + } + + public List> getAttributeValueList() { + return attributeValueList; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitCreateTableDevice(this, context); + } + + @Override + public List getPaths() { + if (devicePathList == null) { + generateDevicePaths(); + } + return devicePathList; + } + + private void generateDevicePaths() { + List result = new ArrayList<>(deviceIdList.size()); + for (String[] deviceId : deviceIdList) { + String[] nodes = new String[3 + deviceId.length]; + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = table; + System.arraycopy(deviceId, 0, nodes, 3, deviceId.length); + result.add(new PartialPath(nodes)); + } + this.devicePathList = result; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java new file mode 100644 index 000000000000..e1ea9582ffa4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.table; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class FetchTableDevicesStatement extends Statement { + + private final String database; + + private final String tableName; + + private final List deviceIdList; + + public FetchTableDevicesStatement( + String database, String tableName, List deviceIdList) { + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitFetchTableDevices(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java new file mode 100644 index 000000000000..ec3f9ffb266b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.table; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; + +import java.util.List; + +public class ShowTableDevicesStatement extends ShowStatement { + + private final String database; + + private final String tableName; + + private final List> idDeterminedFilterList; + + private final SchemaFilter idFuzzyFilter; + + public ShowTableDevicesStatement( + String database, + String tableName, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter) { + super(); + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilter = idFuzzyFilter; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public SchemaFilter getIdFuzzyFilter() { + return idFuzzyFilter; + } + + public List> getIdDeterminedFilterList() { + return idDeterminedFilterList; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowTableDevices(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 421f47ea238d..2b0e5d85a8d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -293,6 +294,18 @@ long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) // endregion + // region table device management + + void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException; + + void deleteTableDevice(String table) throws MetadataException; + + // endregion + // region Interfaces for SchemaReader ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) @@ -304,5 +317,10 @@ ISchemaReader getTimeSeriesReader(IShowTimeSeriesPlan sho ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan) throws MetadataException; + ISchemaReader getTableDeviceReader(ShowTableDevicesPlan showTableDevicesPlan) + throws MetadataException; + + ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException; // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java new file mode 100644 index 000000000000..6a65fe17f1ad --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute; + +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.utils.FileUtils; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DeviceAttributeStore implements IDeviceAttributeStore { + + private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeStore.class); + + public List> deviceAttributeList = new ArrayList<>(); + + @Override + public void clear() { + deviceAttributeList = new ArrayList<>(); + } + + @Override + public synchronized boolean createSnapshot(File targetDir) { + File snapshotTmp = + SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT_TMP); + File snapshot = + SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT); + + try { + FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try { + serialize(outputStream); + } finally { + outputStream.flush(); + fileOutputStream.getFD().sync(); + outputStream.close(); + } + if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { + logger.error( + "Failed to delete old snapshot {} while creating device attribute snapshot.", + snapshot.getName()); + return false; + } + if (!snapshotTmp.renameTo(snapshot)) { + logger.error( + "Failed to rename {} to {} while creating device attribute snapshot.", + snapshotTmp.getName(), + snapshot.getName()); + FileUtils.deleteFileIfExist(snapshot); + return false; + } + + return true; + } catch (IOException e) { + logger.error("Failed to create mtree snapshot due to {}", e.getMessage(), e); + FileUtils.deleteFileIfExist(snapshot); + return false; + } finally { + FileUtils.deleteFileIfExist(snapshotTmp); + } + } + + @Override + public void loadFromSnapshot(File snapshotDir, String sgSchemaDirPath) throws IOException { + try (BufferedInputStream inputStream = + new BufferedInputStream( + Files.newInputStream( + SystemFileFactory.INSTANCE + .getFile(snapshotDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT) + .toPath()))) { + deserialize(inputStream); + } catch (IOException e) { + logger.warn("Load device attribute snapshot from {} failed", snapshotDir); + throw e; + } + } + + @Override + public synchronized int createAttribute(List nameList, List valueList) { + Map attributeMap = new HashMap<>(); + for (int i = 0; i < nameList.size(); i++) { + attributeMap.put(nameList.get(i), valueList.get(i)); + } + deviceAttributeList.add(attributeMap); + return deviceAttributeList.size() - 1; + } + + @Override + public void alterAttribute(int pointer, List nameList, List valueList) { + Map attributeMap = deviceAttributeList.get(pointer); + for (int i = 0; i < nameList.size(); i++) { + attributeMap.put(nameList.get(i), valueList.get(i)); + } + } + + @Override + public String getAttribute(int pointer, String name) { + return deviceAttributeList.get(pointer).get(name); + } + + private void serialize(OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(deviceAttributeList.size(), outputStream); + for (Map attributeMap : deviceAttributeList) { + ReadWriteIOUtils.write(attributeMap, outputStream); + } + } + + private void deserialize(InputStream inputStream) throws IOException { + int size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; i++) { + deviceAttributeList.add(ReadWriteIOUtils.readMap(inputStream)); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java new file mode 100644 index 000000000000..79f44d20bd20 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public interface IDeviceAttributeStore { + + void clear(); + + boolean createSnapshot(File targetDir); + + void loadFromSnapshot(File snapshotDir, String sgSchemaDirPath) throws IOException; + + int createAttribute(List nameList, List valueList); + + void alterAttribute(int pointer, List nameList, List valueList); + + String getAttribute(int pointer, String name); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 46d264fb0200..ad7eedb237f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; @@ -35,6 +36,7 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.metadata.SeriesOverflowException; @@ -49,6 +51,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.DeviceAttributeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.IDeviceAttributeStore; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Deserializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Serializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.SchemaLogReader; @@ -57,12 +61,15 @@ import org.apache.iotdb.db.schemaengine.schemaregion.logfile.visitor.SchemaRegionPlanSerializer; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MTreeBelowSGMemoryImpl; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowDevicesResult; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.schemaregion.tag.TagManager; import org.apache.iotdb.db.schemaengine.schemaregion.utils.filter.FilterContainsVisitor; @@ -87,6 +94,7 @@ import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SchemaUtils; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Pair; @@ -98,8 +106,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -158,6 +168,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { private MTreeBelowSGMemoryImpl mtree; private TagManager tagManager; + private IDeviceAttributeStore deviceAttributeStore; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionMemoryImpl(ISchemaRegionParams schemaRegionParams) throws MetadataException { @@ -196,6 +207,7 @@ public synchronized void init() throws MetadataException { // do not write log when recover isRecovering = true; + deviceAttributeStore = new DeviceAttributeStore(); tagManager = new TagManager(schemaRegionDirPath); mtree = new MTreeBelowSGMemoryImpl( @@ -432,6 +444,13 @@ public synchronized boolean createSnapshot(File snapshotDir) { schemaRegionId, System.currentTimeMillis() - tagSnapshotStartTime); + long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); + isSuccess = isSuccess && deviceAttributeStore.createSnapshot(snapshotDir); + logger.info( + "Device attribute snapshot creation of schemaRegion {} costs {}ms", + schemaRegionId, + System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + logger.info( "Snapshot creation of schemaRegion {} costs {}ms.", schemaRegionId, @@ -454,6 +473,14 @@ public void loadSnapshot(File latestSnapshotRootDir) { isRecovering = true; + long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); + deviceAttributeStore = new DeviceAttributeStore(); + deviceAttributeStore.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); + logger.info( + "Device attribute snapshot loading of schemaRegion {} costs {}ms.", + schemaRegionId, + System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + long tagSnapshotStartTime = System.currentTimeMillis(); tagManager = TagManager.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); logger.info( @@ -1204,10 +1231,35 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) return result; } + @Override + public void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException { + for (int i = 0; i < devicePathList.size(); i++) { + int finalI = i; + mtree.createTableDevice( + devicePathList.get(i), + () -> + deviceAttributeStore.createAttribute( + attributeNameList, attributeValueList.get(finalI)), + pointer -> + deviceAttributeStore.alterAttribute( + pointer, attributeNameList, attributeValueList.get(finalI))); + } + } + + @Override + public void deleteTableDevice(String table) throws MetadataException { + mtree.deleteTableDevice(table); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { - return mtree.getDeviceReader(showDevicesPlan); + return mtree.getDeviceReader( + showDevicesPlan, (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); } @Override @@ -1237,6 +1289,95 @@ public ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan return mtree.getNodeReader(showNodesPlan); } + @Override + public ISchemaReader getTableDeviceReader( + ShowTableDevicesPlan showTableDevicesPlan) throws MetadataException { + return mtree.getDeviceReader( + showTableDevicesPlan.getDevicePattern(), + showTableDevicesPlan.getAttributeFilter(), + (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); + } + + @Override + public ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException { + return new ISchemaReader() { + + Iterator devicePathIterator = devicePathList.listIterator(); + + IDeviceSchemaInfo next = null; + + Throwable t = null; + + @Override + public boolean isSuccess() { + return t == null; + } + + @Override + public Throwable getFailure() { + return t; + } + + @Override + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + @Override + public boolean hasNext() { + if (next == null) { + tryGetNext(); + } + return next != null; + } + + @Override + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + IDeviceSchemaInfo result = next; + next = null; + return result; + } + + private void tryGetNext() { + while (devicePathIterator.hasNext()) { + try { + IMemMNode node = mtree.getNodeByPath(devicePathIterator.next()); + if (!node.isDevice()) { + continue; + } + IDeviceMNode deviceNode = node.getAsDeviceMNode(); + ShowDevicesResult result = + new ShowDevicesResult( + deviceNode.getFullPath(), + deviceNode.isAlignedNullable(), + deviceNode.getSchemaTemplateId(), + deviceNode.getPartialPath().getNodes()); + result.setAttributeProvider( + k -> + deviceAttributeStore.getAttribute( + ((TableDeviceInfo) deviceNode.getDeviceInfo()) + .getAttributePointer(), + k)); + next = result; + break; + } catch (PathNotExistException e) { + continue; + } catch (Throwable e) { + t = e; + return; + } + } + } + + @Override + public void close() throws Exception {} + }; + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 47618defb6a6..d54d4c194fc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -63,6 +63,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -1313,6 +1314,20 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) return result; } + @Override + public void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteTableDevice(String table) throws MetadataException { + throw new UnsupportedOperationException(); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { @@ -1345,6 +1360,19 @@ public ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan throws MetadataException { return mtree.getNodeReader(showNodesPlan); } + + @Override + public ISchemaReader getTableDeviceReader( + ShowTableDevicesPlan showTableDevicesPlan) { + throw new UnsupportedOperationException(); + } + + @Override + public ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException { + throw new UnsupportedOperationException(); + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 72295115324b..dec75300cb0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; @@ -46,6 +47,7 @@ import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.EntityCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MNodeCollector; @@ -90,8 +92,11 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; +import java.util.function.IntSupplier; /** * The hierarchical struct of the Metadata Tree is implemented in this class. @@ -948,7 +953,8 @@ public long countPathsUsingTemplate(PartialPath pathPattern, int templateId) // region Interfaces for schema reader @SuppressWarnings("java:S2095") - public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) + public ISchemaReader getDeviceReader( + IShowDevicesPlan showDevicesPlan, BiFunction attributeProvider) throws MetadataException { EntityCollector collector = new EntityCollector( @@ -960,8 +966,15 @@ public ISchemaReader getDeviceReader(IShowDevicesPlan showDev protected IDeviceSchemaInfo collectEntity(IDeviceMNode node) { PartialPath device = getPartialPathFromRootToNode(node.getAsMNode()); - return new ShowDevicesResult( - device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + ShowDevicesResult result = + new ShowDevicesResult( + device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + result.setAttributeProvider( + k -> + attributeProvider.apply( + ((TableDeviceInfo) node.getDeviceInfo()).getAttributePointer(), + k)); + return result; } }; if (showDevicesPlan.usingSchemaTemplate()) { @@ -992,7 +1005,8 @@ public ListenableFuture isBlocked() { public boolean hasNext() { while (next == null && collector.hasNext()) { IDeviceSchemaInfo temp = collector.next(); - if (filterVisitor.process(showDevicesPlan.getSchemaFilter(), temp)) { + if (showDevicesPlan.getSchemaFilter() == null + || filterVisitor.process(showDevicesPlan.getSchemaFilter(), temp)) { next = temp; } } @@ -1016,6 +1030,72 @@ public IDeviceSchemaInfo next() { } } + public ISchemaReader getDeviceReader( + PartialPath pattern, + SchemaFilter attributeFilter, + BiFunction attributeProvider) + throws MetadataException { + EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, null) { + + protected IDeviceSchemaInfo collectEntity(IDeviceMNode node) { + PartialPath device = getPartialPathFromRootToNode(node.getAsMNode()); + ShowDevicesResult result = + new ShowDevicesResult( + device.getFullPath(), + node.isAlignedNullable(), + node.getSchemaTemplateId(), + node.getPartialPath().getNodes()); + result.setAttributeProvider( + k -> + attributeProvider.apply( + ((TableDeviceInfo) node.getDeviceInfo()).getAttributePointer(), + k)); + return result; + } + }; + return new ISchemaReader() { + + private final DeviceFilterVisitor filterVisitor = new DeviceFilterVisitor(); + private IDeviceSchemaInfo next; + + public boolean isSuccess() { + return collector.isSuccess(); + } + + public Throwable getFailure() { + return collector.getFailure(); + } + + public void close() { + collector.close(); + } + + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + public boolean hasNext() { + while (next == null && collector.hasNext()) { + IDeviceSchemaInfo temp = collector.next(); + if (attributeFilter == null || filterVisitor.process(attributeFilter, temp)) { + next = temp; + } + } + return next != null; + } + + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + IDeviceSchemaInfo result = next; + next = null; + return result; + } + }; + } + public ISchemaReader getTimeSeriesReader( IShowTimeSeriesPlan showTimeSeriesPlan, Function, Map>> tagAndAttributeProvider) @@ -1252,4 +1332,48 @@ protected Void collectMeasurement(IMeasurementMNode node) { return result; } // endregion + + // region table device management + + public void createTableDevice( + PartialPath devicePath, IntSupplier attributePointerGetter, IntConsumer attributeUppdater) + throws MetadataException { + String[] nodeNames = devicePath.getNodes(); + IMemMNode cur = storageGroupMNode; + IMemMNode child; + for (int i = levelOfSG + 1; i < nodeNames.length; i++) { + child = cur.getChild(nodeNames[i]); + if (child == null) { + child = + store.addChild(cur, nodeNames[i], nodeFactory.createInternalMNode(cur, nodeNames[i])); + } + cur = child; + } + + IDeviceMNode entityMNode; + + synchronized (this) { + if (cur.isDevice()) { + entityMNode = cur.getAsDeviceMNode(); + if (!(entityMNode.getDeviceInfo() instanceof TableDeviceInfo)) { + throw new MetadataException("Table device shall not create under tree model"); + } + TableDeviceInfo deviceInfo = + (TableDeviceInfo) entityMNode.getDeviceInfo(); + attributeUppdater.accept(deviceInfo.getAttributePointer()); + } else { + entityMNode = store.setToEntity(cur); + TableDeviceInfo deviceInfo = new TableDeviceInfo<>(); + deviceInfo.setAttributePointer(attributePointerGetter.getAsInt()); + entityMNode.getAsInternalMNode().setDeviceInfo(deviceInfo); + regionStatistics.addDevice(); + } + } + } + + public void deleteTableDevice(String tableName) { + storageGroupMNode.deleteChild(tableName); + } + + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java index ccacef6c7ebe..ea1ff409ecbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java @@ -84,7 +84,8 @@ public String getFullPath() { } String concatFullPath() { - StringBuilder builder = new StringBuilder(getName()); + StringBuilder builder = new StringBuilder(); + builder.insert(0, getName()); IMemMNode curr = this; while (curr.getParent() != null) { curr = curr.getParent(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java new file mode 100644 index 000000000000..d5bfaa4717a9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.container; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +// The value in this map shall not be null. +// Therefore, when using compute method, use v==null to judge if there's existing value. +public class KeyNullableConcurrentHashMap implements Map { + + private final Map, V> map = new ConcurrentHashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(Optional.ofNullable(key)); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(value); + } + + @Override + public V get(Object key) { + return map.get(Optional.ofNullable(key)); + } + + @Override + public V put(K key, V value) { + return map.put(Optional.ofNullable(key), value); + } + + @Override + public V remove(Object key) { + return map.remove(Optional.ofNullable(key)); + } + + @Override + public void putAll(Map m) { + m.forEach((k, v) -> map.put(Optional.ofNullable(k), v)); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Set keySet() { + return map.keySet().stream().map(k -> k.orElse(null)).collect(Collectors.toSet()); + } + + @Override + public Collection values() { + return map.values(); + } + + @Override + public Set> entrySet() { + return map.entrySet().stream() + .map( + o -> + new Entry() { + @Override + public K getKey() { + return o.getKey().orElse(null); + } + + @Override + public V getValue() { + return o.getValue(); + } + + @Override + public V setValue(V value) { + return o.setValue(value); + } + }) + .collect(Collectors.toSet()); + } + + @Override + public V getOrDefault(Object key, V defaultValue) { + return map.getOrDefault(Optional.ofNullable(key), defaultValue); + } + + @Override + public void forEach(BiConsumer action) { + map.forEach((k, v) -> action.accept(k.orElse(null), v)); + } + + @Override + public void replaceAll(BiFunction function) { + map.replaceAll((k, v) -> function.apply(k.orElse(null), v)); + } + + @Override + public V putIfAbsent(K key, V value) { + return map.putIfAbsent(Optional.ofNullable(key), value); + } + + @Override + public boolean remove(Object key, Object value) { + return map.remove(Optional.ofNullable(key), value); + } + + @Override + public boolean replace(K key, V oldValue, V newValue) { + return map.replace(Optional.ofNullable(key), oldValue, newValue); + } + + @Override + public V replace(K key, V value) { + return map.replace(Optional.ofNullable(key), value); + } + + @Override + public V computeIfAbsent(K key, Function mappingFunction) { + return map.computeIfAbsent( + Optional.ofNullable(key), k -> mappingFunction.apply(k.orElse(null))); + } + + @Override + public V computeIfPresent( + K key, BiFunction remappingFunction) { + return map.computeIfPresent( + Optional.ofNullable(key), (k, v) -> remappingFunction.apply(k.orElse(null), v)); + } + + @Override + public V compute(K key, BiFunction remappingFunction) { + return map.compute( + Optional.ofNullable(key), (k, v) -> remappingFunction.apply(k.orElse(null), v)); + } + + @Override + public V merge(K key, V value, BiFunction remappingFunction) { + return map.merge(Optional.ofNullable(key), value, remappingFunction); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java index 44c16126ea87..97de504f4b5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java @@ -28,11 +28,10 @@ import java.util.Collection; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import static java.util.Collections.emptySet; -public class MemMNodeContainer extends ConcurrentHashMap +public class MemMNodeContainer extends KeyNullableConcurrentHashMap implements IMNodeContainer { private static final IMNodeContainer EMPTY_CONTAINER = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java index 36f92bfda56b..97d13ee6757d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java @@ -37,6 +37,6 @@ public void setName(String name) { public int estimateSize() { // object header, 8B // name reference, name length and name hash code, 8 + 4 + 4 = 16B - return 8 + 16 + 2 * name.length(); + return 8 + 16 + 2 * (name == null ? 0 : name.length()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java new file mode 100644 index 000000000000..857cf20a40d2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info; + +import org.apache.iotdb.commons.schema.node.IMNode; +import org.apache.iotdb.commons.schema.node.info.IDeviceInfo; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; + +import java.util.Map; + +import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; + +public class TableDeviceInfo> implements IDeviceInfo { + + private int attributePointer = -1; + + public int getAttributePointer() { + return attributePointer; + } + + public void setAttributePointer(int attributePointer) { + this.attributePointer = attributePointer; + } + + @Override + public void moveDataToNewMNode(IDeviceMNode newMNode) {} + + @Override + public boolean addAlias(String alias, IMeasurementMNode child) { + return false; + } + + @Override + public void deleteAliasChild(String alias) {} + + @Override + public Map> getAliasChildren() { + return null; + } + + @Override + public void setAliasChildren(Map> aliasChildren) {} + + @Override + public boolean hasAliasChild(String name) { + return false; + } + + @Override + public N getAliasChild(String name) { + return null; + } + + @Override + public boolean isUseTemplate() { + return false; + } + + @Override + public void setUseTemplate(boolean useTemplate) {} + + @Override + public void setSchemaTemplateId(int schemaTemplateId) {} + + @Override + public int getSchemaTemplateId() { + return NON_TEMPLATE; + } + + @Override + public int getSchemaTemplateIdWithState() { + return 0; + } + + @Override + public boolean isPreDeactivateTemplate() { + return false; + } + + @Override + public void preDeactivateTemplate() {} + + @Override + public void rollbackPreDeactivateTemplate() {} + + @Override + public void deactivateTemplate() {} + + @Override + public Boolean isAligned() { + return true; + } + + @Override + public void setAligned(Boolean isAligned) {} + + @Override + public int estimateSize() { + return 12; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java index 9508221ec0e9..565183b38fd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -59,6 +60,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.MEASUREMENT_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_ENTITY_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_MNODE_TYPE; +import static org.apache.iotdb.commons.schema.SchemaConstant.TABLE_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.isStorageGroupType; public class MemMTreeSnapshotUtil { @@ -235,6 +237,11 @@ private static void deserializeMNode( node = deserializer.deserializeEntityMNode(inputStream); deviceProcess.accept(node.getAsDeviceMNode()); break; + case TABLE_MNODE_TYPE: + childrenNum = ReadWriteIOUtils.readInt(inputStream); + node = deserializer.deserializeTableDeviceMNode(inputStream); + deviceProcess.accept(node.getAsDeviceMNode()); + break; case STORAGE_GROUP_ENTITY_MNODE_TYPE: childrenNum = ReadWriteIOUtils.readInt(inputStream); node = deserializer.deserializeStorageGroupEntityMNode(inputStream); @@ -280,12 +287,20 @@ private static class MNodeSerializer extends MNodeVisitor public Boolean visitBasicMNode(IMNode node, OutputStream outputStream) { try { if (node.isDevice()) { - ReadWriteIOUtils.write(ENTITY_MNODE_TYPE, outputStream); - serializeBasicMNode(node, outputStream); - IDeviceMNode deviceMNode = node.getAsDeviceMNode(); - ReadWriteIOUtils.write(deviceMNode.getSchemaTemplateIdWithState(), outputStream); - ReadWriteIOUtils.write(deviceMNode.isUseTemplate(), outputStream); - ReadWriteIOUtils.write(deviceMNode.isAlignedNullable(), outputStream); + if (node.getAsDeviceMNode().getDeviceInfo() instanceof TableDeviceInfo) { + ReadWriteIOUtils.write(TABLE_MNODE_TYPE, outputStream); + TableDeviceInfo tableDeviceInfo = + (TableDeviceInfo) (node.getAsDeviceMNode().getDeviceInfo()); + serializeBasicMNode(node, outputStream); + ReadWriteIOUtils.write(tableDeviceInfo.getAttributePointer(), outputStream); + } else { + ReadWriteIOUtils.write(ENTITY_MNODE_TYPE, outputStream); + serializeBasicMNode(node, outputStream); + IDeviceMNode deviceMNode = node.getAsDeviceMNode(); + ReadWriteIOUtils.write(deviceMNode.getSchemaTemplateIdWithState(), outputStream); + ReadWriteIOUtils.write(deviceMNode.isUseTemplate(), outputStream); + ReadWriteIOUtils.write(deviceMNode.isAlignedNullable(), outputStream); + } } else { ReadWriteIOUtils.write(INTERNAL_MNODE_TYPE, outputStream); serializeBasicMNode(node, outputStream); @@ -394,6 +409,15 @@ public IMemMNode deserializeEntityMNode(InputStream inputStream) throws IOExcept return node.getAsMNode(); } + public IMemMNode deserializeTableDeviceMNode(InputStream inputStream) throws IOException { + String name = ReadWriteIOUtils.readString(inputStream); + IDeviceMNode node = nodeFactory.createDeviceMNode(null, name); + TableDeviceInfo tableDeviceInfo = new TableDeviceInfo<>(); + tableDeviceInfo.setAttributePointer(ReadWriteIOUtils.readInt(inputStream)); + node.getAsInternalMNode().setDeviceInfo(tableDeviceInfo); + return node.getAsMNode(); + } + public IMemMNode deserializeMeasurementMNode(InputStream inputStream) throws IOException { String name = ReadWriteIOUtils.readString(inputStream); MeasurementSchema schema = MeasurementSchema.deserializeFrom(inputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java new file mode 100644 index 000000000000..e579d235f6bf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java @@ -0,0 +1,40 @@ +package org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.AndFilter; + +import java.util.List; + +public class ShowTableDevicesPlan { + + private PartialPath devicePattern; + + private SchemaFilter attributeFilter; + + public ShowTableDevicesPlan(PartialPath devicePattern, SchemaFilter attributeFilter) { + this.devicePattern = devicePattern; + this.attributeFilter = attributeFilter; + } + + private SchemaFilter getAttributeFilter(List filterList) { + if (filterList.isEmpty()) { + return null; + } + AndFilter andFilter; + SchemaFilter latestFilter = filterList.get(0); + for (int i = 1; i < filterList.size(); i++) { + andFilter = new AndFilter(latestFilter, filterList.get(i)); + latestFilter = andFilter; + } + return latestFilter; + } + + public PartialPath getDevicePattern() { + return devicePattern; + } + + public SchemaFilter getAttributeFilter() { + return attributeFilter; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java index 9a670f9b44bf..a144e10c9c15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java @@ -24,4 +24,8 @@ public interface IDeviceSchemaInfo extends ISchemaInfo { Boolean isAligned(); int getTemplateId(); + + String getAttributeValue(String attributeKey); + + String[] getRawNodes(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java index 19aa19710082..fda9cf1f300e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java @@ -18,20 +18,33 @@ */ package org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import java.util.Objects; +import java.util.function.Function; public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchemaInfo { private Boolean isAligned; private int templateId; + private Function attributeProvider; + + private String[] rawNodes = null; + public ShowDevicesResult(String name, Boolean isAligned, int templateId) { super(name); this.isAligned = isAligned; this.templateId = templateId; } + public ShowDevicesResult(String name, Boolean isAligned, int templateId, String[] rawNodes) { + super(name); + this.isAligned = isAligned; + this.templateId = templateId; + this.rawNodes = rawNodes; + } + public Boolean isAligned() { return isAligned; } @@ -40,6 +53,24 @@ public int getTemplateId() { return templateId; } + public void setAttributeProvider(Function attributeProvider) { + this.attributeProvider = attributeProvider; + } + + @Override + public String getAttributeValue(String attributeKey) { + return attributeProvider.apply(attributeKey); + } + + public String[] getRawNodes() { + return rawNodes; + } + + @Override + public PartialPath getPartialPath() { + return new PartialPath(rawNodes); + } + @Override public String toString() { return "ShowDevicesResult{" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java index e1723193abfe..f80dfba0c2a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java @@ -21,11 +21,15 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; +import java.util.Objects; + public class DeviceFilterVisitor extends SchemaFilterVisitor { @Override public boolean visitNode(SchemaFilter filter, IDeviceSchemaInfo info) { @@ -59,4 +63,19 @@ public boolean visitTemplateFilter(TemplateFilter templateFilter, IDeviceSchemaI return false; } } + + @Override + public boolean visitDeviceIdFilter(DeviceIdFilter filter, IDeviceSchemaInfo info) { + String[] nodes = info.getPartialPath().getNodes(); + if (nodes.length < filter.getIndex() + 3) { + return false; + } else { + return nodes[filter.getIndex() + 3].equals(filter.getValue()); + } + } + + @Override + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, IDeviceSchemaInfo info) { + return Objects.equals(filter.getValue(), info.getAttributeValue(filter.getKey())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 4b92c961c801..24f592fd0be4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -146,6 +147,64 @@ public void commitCreateTable(String database, String tableName) { } } + @Override + public void invalidateTable(String database, String tableName) { + readWriteLock.writeLock().lock(); + try { + databaseTableMap.compute( + database, + (db, map) -> { + if (map == null) { + return null; + } + map.remove(tableName); + if (map.isEmpty()) { + return null; + } else { + return map; + } + }); + LOGGER.info("Invalidate table {}.{} successfully", database, tableName); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void invalidateTable(String database) { + readWriteLock.writeLock().lock(); + try { + databaseTableMap.remove(database); + preCreateTableMap.remove(database); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void addTableColumn( + String database, String tableName, List columnSchemaList) { + readWriteLock.writeLock().lock(); + try { + TsTable table = databaseTableMap.get(database).get(tableName); + columnSchemaList.forEach(table::addColumnSchema); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void rollbackAddColumn( + String database, String tableName, List columnSchemaList) { + readWriteLock.writeLock().lock(); + try { + TsTable table = databaseTableMap.get(database).get(tableName); + columnSchemaList.forEach(o -> table.removeColumnSchema(o.getColumnName())); + } finally { + readWriteLock.writeLock().unlock(); + } + } + public TsTable getTable(String database, String tableName) { readWriteLock.readLock().lock(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index 16d263004734..9efeeb940161 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -20,6 +20,9 @@ package org.apache.iotdb.db.schemaengine.table; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; + +import java.util.List; public interface ITableCache { @@ -30,4 +33,14 @@ public interface ITableCache { void rollbackCreateTable(String database, String tableName); void commitCreateTable(String database, String tableName); + + void invalidateTable(String database, String tableName); + + void invalidateTable(String database); + + void addTableColumn( + String database, String tableName, List columnSchemaList); + + void rollbackAddColumn( + String database, String tableName, List columnSchemaList); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 2ffeca565963..af42e170a796 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; @@ -698,7 +699,7 @@ public boolean equals(Object obj) { return false; } else { for (int i = 0; i < this.nodes.length; i++) { - if (!nodes[i].equals(otherNodes[i])) { + if (!Objects.equals(nodes[i], otherNodes[i])) { return false; } } @@ -715,7 +716,7 @@ public boolean equals(String obj) { public int hashCode() { int h = 0; for (String node : nodes) { - h += 31 * h + node.hashCode(); + h += 31 * h + Objects.hashCode(node); } return h; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java index a8ba920813b6..6ee736453500 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java @@ -33,7 +33,8 @@ private PathPatternUtil() {} * patternNode that can match batch explicit node names. e.g. *, e.g. *, **, d*, *d*. */ public static boolean hasWildcard(String node) { - return node.startsWith(ONE_LEVEL_PATH_WILDCARD) || node.endsWith(ONE_LEVEL_PATH_WILDCARD); + return node != null + && (node.startsWith(ONE_LEVEL_PATH_WILDCARD) || node.endsWith(ONE_LEVEL_PATH_WILDCARD)); } public static boolean isMultiLevelMatchWildcard(String node) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java index 698f27b37c85..4a68861af1d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java @@ -29,6 +29,7 @@ import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.regex.Pattern; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; @@ -123,6 +124,8 @@ private SinglePathPatternNode getNextNode(SinglePathPatternNode currentNode) { if (patternNodes[nextIndex] == null) { if (nextIndex == rawNodes.length) { patternNodes[nextIndex] = new PrefixMatchNode(nextIndex, currentNode.getTracebackNode()); + } else if (rawNodes[nextIndex] == null) { + patternNodes[nextIndex] = new NameMatchNode(nextIndex, currentNode.getTracebackNode()); } else if (rawNodes[nextIndex].equals(MULTI_LEVEL_PATH_WILDCARD)) { patternNodes[nextIndex] = new MultiLevelWildcardMatchNode(nextIndex); } else if (rawNodes[nextIndex].equals(ONE_LEVEL_PATH_WILDCARD)) { @@ -439,7 +442,7 @@ private NameMatchNode(int patternIndex, SinglePathPatternNode tracebackNode) { @Override public boolean isMatch(String event) { - return rawNodes[patternIndex].equals(event); + return Objects.equals(rawNodes[patternIndex], event); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java index 7c67375d31df..f045527778b8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java @@ -56,6 +56,10 @@ private SchemaConstant() { public static final String TAG_LOG_SNAPSHOT_TMP = "tlog.txt.snapshot.tmp"; public static final String MTREE_SNAPSHOT = "mtree.snapshot"; public static final String MTREE_SNAPSHOT_TMP = "mtree.snapshot.tmp"; + + public static final String DEVICE_ATTRIBUTE_SNAPSHOT = "device_attribute.snapshot"; + public static final String DEVICE_ATTRIBUTE_SNAPSHOT_TMP = "device_attribute.snapshot.tmp"; + public static final String SYSTEM_DATABASE = "root.__system"; public static final String[] ALL_RESULT_NODES = new String[] {"root", "**"}; public static final PartialPath ALL_MATCH_PATTERN = new PartialPath(ALL_RESULT_NODES); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java index a9e5fea0b73c..ad3c4679a394 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java @@ -21,6 +21,9 @@ import org.apache.iotdb.commons.schema.filter.impl.AndFilter; import org.apache.iotdb.commons.schema.filter.impl.DataTypeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TagFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; @@ -73,6 +76,12 @@ public static SchemaFilter deserialize(ByteBuffer byteBuffer) { return new AndFilter(byteBuffer); case TEMPLATE_FILTER: return new TemplateFilter(byteBuffer); + case OR: + return new OrFilter(byteBuffer); + case DEVICE_ID: + return new DeviceIdFilter(byteBuffer); + case DEVICE_ATTRIBUTE: + return new DeviceAttributeFilter(byteBuffer); default: throw new IllegalArgumentException("Unsupported schema filter type: " + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java index 285130f778e9..66e9f15e939e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java @@ -26,6 +26,11 @@ public enum SchemaFilterType { VIEW_TYPE((short) 4), AND((short) 5), TEMPLATE_FILTER((short) 6), + + DEVICE_ID((short) 7), + DEVICE_ATTRIBUTE((short) 8), + + OR((short) 9), ; private final short code; @@ -54,6 +59,12 @@ public static SchemaFilterType getSchemaFilterType(short code) { return AND; case 6: return TEMPLATE_FILTER; + case 7: + return DEVICE_ID; + case 8: + return DEVICE_ATTRIBUTE; + case 9: + return OR; default: throw new IllegalArgumentException("Invalid input: " + code); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java index d6861baab683..f503da64b15f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java @@ -20,6 +20,9 @@ import org.apache.iotdb.commons.schema.filter.impl.AndFilter; import org.apache.iotdb.commons.schema.filter.impl.DataTypeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TagFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; @@ -69,4 +72,16 @@ public boolean visitTemplateFilter(TemplateFilter templateFilter, C context) { public boolean visitAndFilter(AndFilter andFilter, C context) { return andFilter.getLeft().accept(this, context) && andFilter.getRight().accept(this, context); } + + public boolean visitOrFilter(OrFilter orFilter, C context) { + return orFilter.getLeft().accept(this, context) || orFilter.getRight().accept(this, context); + } + + public boolean visitDeviceIdFilter(DeviceIdFilter filter, C context) { + return visitFilter(filter, context); + } + + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, C context) { + return visitDeviceAttributeFilter(filter, context); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java new file mode 100644 index 000000000000..d85b437e9c42 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DeviceAttributeFilter extends SchemaFilter { + + private final String key; + + private final String value; + + public DeviceAttributeFilter(String key, String value) { + this.key = key; + this.value = value; + } + + public DeviceAttributeFilter(ByteBuffer byteBuffer) { + this.key = ReadWriteIOUtils.readString(byteBuffer); + this.value = ReadWriteIOUtils.readString(byteBuffer); + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitDeviceAttributeFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.DEVICE_ATTRIBUTE; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(key, byteBuffer); + ReadWriteIOUtils.write(value, byteBuffer); + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(key, stream); + ReadWriteIOUtils.write(value, stream); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java new file mode 100644 index 000000000000..c65ccf2579d8 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.table.TsTable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class DeviceFilterUtil { + + private DeviceFilterUtil() { + // do nothing + } + + // if the element in idDeterminedFilterList isEmpty, the corresponding pattern is + // root.db.table.*.*.. + // e.g. input (db, table[c1, c2], [[]]), return [root.db.table.*.*] + public static List convertToDevicePattern( + String database, TsTable table, List> idDeterminedFilterList) { + List pathList = new ArrayList<>(); + int length = table.getIdNums() + 3; + for (List idFilterList : idDeterminedFilterList) { + String[] nodes = new String[length]; + Arrays.fill(nodes, ONE_LEVEL_PATH_WILDCARD); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = table.getTableName(); + for (SchemaFilter schemaFilter : idFilterList) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else { + throw new IllegalStateException("Input single filter must be DeviceIdFilter"); + } + } + pathList.add(new PartialPath(nodes)); + } + + return pathList; + } + + public static List convertToDevicePath( + String database, String tableName, List deviceIdList) { + List devicePathList = new ArrayList<>(deviceIdList.size()); + String[] nodes; + for (String[] idValues : deviceIdList) { + nodes = new String[idValues.length + 3]; + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + System.arraycopy(idValues, 0, nodes, 3, idValues.length); + devicePathList.add(new PartialPath(nodes)); + } + return devicePathList; + } + + // input and-concat filter list + // return or concat filter list, inner which all filter is and concat + // e.g. (a OR b) AND (c OR d) -> (a AND c) OR (a AND d) OR (b AND c) OR (b AND d) + // if input is empty, then return [[]] + public static List> convertSchemaFilterToOrConcatList( + List schemaFilterList) { + List> orConcatList = + schemaFilterList.stream() + .map(DeviceFilterUtil::convertOneSchemaFilterToOrConcat) + .collect(Collectors.toList()); + int orSize = orConcatList.size(); + int finalResultSize = 1; + for (List filterList : orConcatList) { + finalResultSize *= filterList.size(); + } + List> result = new ArrayList<>(finalResultSize); + int[] indexes = new int[orSize]; + while (finalResultSize > 0) { + List oneCase = new ArrayList<>(orConcatList.size()); + for (int j = 0; j < orSize; j++) { + oneCase.add(orConcatList.get(j).get(indexes[j])); + } + result.add(oneCase); + for (int k = orSize - 1; k >= 0; k--) { + indexes[k]++; + if (indexes[k] < orConcatList.get(k).size()) { + break; + } + indexes[k] = 0; + } + finalResultSize--; + } + return result; + } + + private static List convertOneSchemaFilterToOrConcat(SchemaFilter schemaFilter) { + List result = new ArrayList<>(); + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + result.addAll(convertOneSchemaFilterToOrConcat(orFilter.getLeft())); + result.addAll(convertOneSchemaFilterToOrConcat(orFilter.getRight())); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.AND)) { + throw new IllegalStateException("Input filter shall not be AND operation"); + } else { + result.add(schemaFilter); + } + return result; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java new file mode 100644 index 000000000000..6ba5cd71f54e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DeviceIdFilter extends SchemaFilter { + + // id column index + // when used in partialPath, the index of node in path shall be [this.index + 3] + // since a partialPath start with {root, db, table} + private final int index; + + private final String value; + + public DeviceIdFilter(int index, String value) { + this.index = index; + this.value = value; + } + + public DeviceIdFilter(ByteBuffer byteBuffer) { + this.index = ReadWriteIOUtils.readInt(byteBuffer); + this.value = ReadWriteIOUtils.readString(byteBuffer); + } + + public int getIndex() { + return index; + } + + public String getValue() { + return value; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitDeviceIdFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.DEVICE_ID; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(index, byteBuffer); + ReadWriteIOUtils.write(value, byteBuffer); + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(index, stream); + ReadWriteIOUtils.write(value, stream); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java new file mode 100644 index 000000000000..e740c23b2c97 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class OrFilter extends SchemaFilter { + private final SchemaFilter left; + private final SchemaFilter right; + + public OrFilter(SchemaFilter left, SchemaFilter right) { + // left and right should not be null + this.left = left; + this.right = right; + } + + public OrFilter(ByteBuffer byteBuffer) { + this.left = SchemaFilter.deserialize(byteBuffer); + this.right = SchemaFilter.deserialize(byteBuffer); + } + + public SchemaFilter getLeft() { + return left; + } + + public SchemaFilter getRight() { + return right; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitOrFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.OR; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + left.serialize(byteBuffer); + right.serialize(byteBuffer); + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + left.serialize(stream); + right.serialize(stream); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java new file mode 100644 index 000000000000..fe9d8b7e4faf --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +public enum AlterTableOperationType { + ADD_COLUMN((byte) 0); + + private final byte type; + + AlterTableOperationType(byte type) { + this.type = type; + } + + public byte getTypeValue() { + return type; + } + + public static AlterTableOperationType getType(byte value) { + switch (value) { + case 0: + return ADD_COLUMN; + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 054928cca632..dd0d4662cc70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.schema.table; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; @@ -52,6 +53,8 @@ public class TsTable { private Map props = null; + private transient int idNums = 0; + public TsTable(String tableName) { this.tableName = tableName; columnSchemaMap.put(TIME_COLUMN_NAME, TIME_COLUMN_SCHEMA); @@ -67,12 +70,23 @@ public TsTableColumnSchema getColumnSchema(String columnName) { public void addColumnSchema(TsTableColumnSchema columnSchema) { columnSchemaMap.put(columnSchema.getColumnName(), columnSchema); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + idNums++; + } + } + + public void removeColumnSchema(String columnName) { + columnSchemaMap.remove(columnName); } public int getColumnNum() { return columnSchemaMap.size(); } + public int getIdNums() { + return idNums; + } + public List getColumnList() { return new ArrayList<>(columnSchemaMap.values()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java index 06e08bb41bec..fb080a266c33 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java @@ -28,7 +28,14 @@ public enum TsTableInternalRPCType { PRE_CREATE((byte) 0), ROLLBACK_CREATE((byte) 1), - COMMIT_CREATE((byte) 2); + COMMIT_CREATE((byte) 2), + + INVALIDATE_CACHE((byte) 3), + DELETE_DATA_IN_DATA_REGION((byte) 4), + DELETE_SCHEMA_IN_SCHEMA_REGION((byte) 5), + + ADD_COLUMN((byte) 6), + ROLLBACK_ADD_COLUMN((byte) 7); private final byte operationType; @@ -57,6 +64,16 @@ public static TsTableInternalRPCType getType(byte type) { return ROLLBACK_CREATE; case 2: return COMMIT_CREATE; + case 3: + return INVALIDATE_CACHE; + case 4: + return DELETE_DATA_IN_DATA_REGION; + case 5: + return DELETE_SCHEMA_IN_SCHEMA_REGION; + case 6: + return ADD_COLUMN; + case 7: + return ROLLBACK_ADD_COLUMN; default: throw new IllegalArgumentException("Unknown table update operation type" + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java index 4ebb8a25231f..eac67b2b3b28 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java @@ -21,10 +21,13 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class TsTableColumnSchemaUtil { @@ -77,4 +80,38 @@ private static TsTableColumnSchema deserialize( throw new IllegalArgumentException(); } } + + public static byte[] serialize(List columnSchemaList) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + serialize(columnSchemaList, stream); + } catch (IOException ignored) { + // won't happen + } + return stream.toByteArray(); + } + + public static void serialize(List columnSchemaList, OutputStream stream) + throws IOException { + if (columnSchemaList == null) { + ReadWriteIOUtils.write(-1, stream); + return; + } + ReadWriteIOUtils.write(columnSchemaList.size(), stream); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + serialize(columnSchema, stream); + } + } + + public static List deserializeColumnSchemaList(ByteBuffer buffer) { + int size = ReadWriteIOUtils.readInt(buffer); + if (size == -1) { + return null; + } + List columnSchemaList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnSchemaList.add(deserialize(buffer)); + } + return columnSchemaList; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java index caa8019ef871..606d1b00dafd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java @@ -121,7 +121,9 @@ protected AbstractTreeVisitor( boolean usingDFA = false; // Use DFA if there are ** and no regex node in pathPattern for (String pathNode : pathPattern.getNodes()) { - if (IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(pathNode)) { + if (pathNode == null) { + continue; + } else if (IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(pathNode)) { // ** node usingDFA = true; } else if (pathNode.length() > 1 && PathPatternUtil.hasWildcard(pathNode)) { diff --git a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java index a50e7e82b65a..68c08e6fff8c 100644 --- a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java @@ -379,11 +379,6 @@ public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementCont return super.visitLoadTsFileStatement(ctx); } - @Override - public Node visitShowDevicesStatement(RelationalSqlParser.ShowDevicesStatementContext ctx) { - return super.visitShowDevicesStatement(ctx); - } - @Override public Node visitCountDevicesStatement(RelationalSqlParser.CountDevicesStatementContext ctx) { return super.visitCountDevicesStatement(ctx); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index b14f67cef03d..3a15aab022e6 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -919,6 +919,23 @@ enum TTestOperation { TEST_SUB_PROCEDURE, } +// ==================================================== +// Table +// ==================================================== +struct TDropTableReq{ + 1: required string database + 2: required string tableName + 3: required string queryId +} + +struct TAlterTableReq{ + 1: required string database + 2: required string tableName + 3: required string queryId + 4: required byte operationType + 5: required binary updateInfo +} + service IConfigNodeRPCService { // ====================================================== @@ -1565,5 +1582,9 @@ service IConfigNodeRPCService { // ====================================================== common.TSStatus createTable(binary tableInfo) + + common.TSStatus dropTable(TDropTableReq req) + + common.TSStatus alterTable(TAlterTableReq req) }