From f2c80eb46dc71b53b41e08953c7f5ad010db2886 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Tue, 16 Apr 2024 21:02:41 +0800 Subject: [PATCH 01/36] implement table insert via tree model --- .../thrift/impl/ClientRPCServiceImpl.java | 112 ++++++++++++++++-- .../plan/analyze/AnalyzeVisitor.java | 2 +- .../iotdb/commons/schema/table/TsTable.java | 10 ++ 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index f970838f31bc..6496edc21d69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -34,6 +34,8 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -104,6 +106,13 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.SetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.Insert; +import org.apache.iotdb.db.relational.sql.tree.LongLiteral; +import org.apache.iotdb.db.relational.sql.tree.Row; +import org.apache.iotdb.db.relational.sql.tree.Values; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -113,6 +122,7 @@ import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.SetThreadName; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -201,6 +211,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.partition.DataPartition.NOT_ASSIGNED; import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; @@ -336,17 +347,29 @@ private TSExecuteStatementResp executeStatementInternal( queryId = SESSION_MANAGER.requestQueryId(clientSession, req.statementId); - // TODO audit log, quota, StatementType - result = - COORDINATOR.executeForTableModel( - s, - relationSqlParser, - clientSession, - queryId, - SESSION_MANAGER.getSessionInfo(clientSession), - statement, - metadata, - req.getTimeout()); + if (s instanceof Insert) { + result = + COORDINATOR.executeForTreeModel( + parseInsert(clientSession, (Insert) s), + queryId, + SESSION_MANAGER.getSessionInfo(clientSession), + statement, + partitionFetcher, + schemaFetcher, + req.getTimeout()); + } else { + // TODO audit log, quota, StatementType + result = + COORDINATOR.executeForTableModel( + s, + relationSqlParser, + clientSession, + queryId, + SESSION_MANAGER.getSessionInfo(clientSession), + statement, + metadata, + req.getTimeout()); + } } if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() @@ -403,6 +426,73 @@ private TSExecuteStatementResp executeStatementInternal( } } + private InsertRowStatement parseInsert(IClientSession clientSession, Insert insert) { + InsertRowStatement insertStatement = new InsertRowStatement(); + String database = clientSession.getDatabaseName(); + if (database == null) { + database = insert.getTable().getName().getPrefix().get().getSuffix(); + } + String tableName = insert.getTable().getName().getSuffix().toString(); + TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); + List values = + ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); + Map idColumnMap = new HashMap<>(); + Map attrColumnMap = new HashMap<>(); + Map measurementColumnMap = new HashMap<>(); + long time = 0L; + boolean hasColumn = insert.getColumns().isPresent(); + int size = hasColumn ? insert.getColumns().get().size() : table.getColumnNum(); + List columnNameList = hasColumn ? insert.getColumns().get() : null; + for (int i = 0; i < size; i++) { + String columnName = + hasColumn + ? columnNameList.get(i).getValue() + : table.getColumnList().get(i).getColumnName(); + TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); + if (category.equals(TsTableColumnCategory.ID)) { + idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.ATTRIBUTE)) { + attrColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + measurementColumnMap.put(columnName, ((LongLiteral) values.get(i)).getValue()); + } else { + time = Long.parseLong(((LongLiteral) values.get(i)).getValue()); + } + } + String[] deviceIds = new String[table.getIdNums() + 3]; + deviceIds[0] = PATH_ROOT; + deviceIds[1] = database; + deviceIds[2] = tableName; + String[] measurements = new String[measurementColumnMap.size()]; + Object[] valueList = new Object[measurements.length]; + int idIndex = 0; + int measurementIndex = 0; + for (int i = 0; i < table.getColumnNum(); i++) { + TsTableColumnCategory category = table.getColumnList().get(i).getColumnCategory(); + if (category.equals(TsTableColumnCategory.ID)) { + String id = idColumnMap.get(table.getColumnList().get(i).getColumnName()); + deviceIds[3 + idIndex] = id == null ? "" : id; + idIndex++; + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + String measurement = table.getColumnList().get(i).getColumnName(); + if (measurementColumnMap.containsKey(measurement)) { + measurements[measurementIndex] = measurement; + valueList[measurementIndex] = measurementColumnMap.get(measurement); + measurementIndex++; + } + } + } + insertStatement.setDevicePath(new PartialPath(deviceIds)); + TimestampPrecisionUtils.checkTimestampPrecision(time); + insertStatement.setTime(time); + insertStatement.setMeasurements(measurements); + insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]); + insertStatement.setValues(valueList); + insertStatement.setNeedInferType(true); + insertStatement.setAligned(true); + return insertStatement; + } + private TSExecuteStatementResp executeRawDataQueryInternal( TSRawDataQueryReq req, SelectResult setResult) { boolean finished = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 2a681119b678..e242c53c7e52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2436,7 +2436,7 @@ public Analysis visitInternalCreateTimeseries( Analysis analysis = new Analysis(); analysis.setStatement(internalCreateTimeSeriesStatement); - checkIsTableCompatible(internalCreateTimeSeriesStatement.getDevicePath(), context); + // checkIsTableCompatible(internalCreateTimeSeriesStatement.getDevicePath(), context); checkIsTemplateCompatible( internalCreateTimeSeriesStatement.getDevicePath(), internalCreateTimeSeriesStatement.getMeasurements(), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 6904c000c6d1..bffa73fdf1a2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.schema.table; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -51,6 +52,8 @@ public class TsTable { private Map props = null; + private transient int idNums = 0; + public TsTable(String tableName) { this.tableName = tableName; columnSchemaMap.put(TIME_COLUMN_NAME, TIME_COLUMN_SCHEMA); @@ -66,12 +69,19 @@ public TsTableColumnSchema getColumnSchema(String columnName) { public void addColumnSchema(TsTableColumnSchema columnSchema) { columnSchemaMap.put(columnSchema.getColumnName(), columnSchema); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + idNums++; + } } public int getColumnNum() { return columnSchemaMap.size(); } + public int getIdNums() { + return idNums; + } + public List getColumnList() { return new ArrayList<>(columnSchemaMap.values()); } From e0dc92ba7aa577803514c67a9dca926a8770ec72 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 17 Apr 2024 09:19:43 +0800 Subject: [PATCH 02/36] refactor insert parse --- .../thrift/impl/ClientRPCServiceImpl.java | 80 +---------- .../plan/analyze/AnalyzeVisitor.java | 24 ++++ .../plan/statement/StatementVisitor.java | 5 + .../statement/crud/InsertTableStatement.java | 132 ++++++++++++++++++ 4 files changed, 163 insertions(+), 78 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 6496edc21d69..e7a8b6baee73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -34,8 +34,6 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.table.TsTable; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -93,6 +91,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateAlignedTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateMultiTimeSeriesStatement; @@ -106,13 +105,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.SetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.relational.sql.parser.SqlParser; -import org.apache.iotdb.db.relational.sql.tree.Expression; -import org.apache.iotdb.db.relational.sql.tree.Identifier; import org.apache.iotdb.db.relational.sql.tree.Insert; -import org.apache.iotdb.db.relational.sql.tree.LongLiteral; -import org.apache.iotdb.db.relational.sql.tree.Row; -import org.apache.iotdb.db.relational.sql.tree.Values; -import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -122,7 +115,6 @@ import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.SetThreadName; -import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -211,7 +203,6 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.partition.DataPartition.NOT_ASSIGNED; import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; @@ -350,7 +341,7 @@ private TSExecuteStatementResp executeStatementInternal( if (s instanceof Insert) { result = COORDINATOR.executeForTreeModel( - parseInsert(clientSession, (Insert) s), + new InsertTableStatement(clientSession, (Insert) s), queryId, SESSION_MANAGER.getSessionInfo(clientSession), statement, @@ -426,73 +417,6 @@ private TSExecuteStatementResp executeStatementInternal( } } - private InsertRowStatement parseInsert(IClientSession clientSession, Insert insert) { - InsertRowStatement insertStatement = new InsertRowStatement(); - String database = clientSession.getDatabaseName(); - if (database == null) { - database = insert.getTable().getName().getPrefix().get().getSuffix(); - } - String tableName = insert.getTable().getName().getSuffix().toString(); - TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); - List values = - ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); - Map idColumnMap = new HashMap<>(); - Map attrColumnMap = new HashMap<>(); - Map measurementColumnMap = new HashMap<>(); - long time = 0L; - boolean hasColumn = insert.getColumns().isPresent(); - int size = hasColumn ? insert.getColumns().get().size() : table.getColumnNum(); - List columnNameList = hasColumn ? insert.getColumns().get() : null; - for (int i = 0; i < size; i++) { - String columnName = - hasColumn - ? columnNameList.get(i).getValue() - : table.getColumnList().get(i).getColumnName(); - TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); - if (category.equals(TsTableColumnCategory.ID)) { - idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); - } else if (category.equals(TsTableColumnCategory.ATTRIBUTE)) { - attrColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); - } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { - measurementColumnMap.put(columnName, ((LongLiteral) values.get(i)).getValue()); - } else { - time = Long.parseLong(((LongLiteral) values.get(i)).getValue()); - } - } - String[] deviceIds = new String[table.getIdNums() + 3]; - deviceIds[0] = PATH_ROOT; - deviceIds[1] = database; - deviceIds[2] = tableName; - String[] measurements = new String[measurementColumnMap.size()]; - Object[] valueList = new Object[measurements.length]; - int idIndex = 0; - int measurementIndex = 0; - for (int i = 0; i < table.getColumnNum(); i++) { - TsTableColumnCategory category = table.getColumnList().get(i).getColumnCategory(); - if (category.equals(TsTableColumnCategory.ID)) { - String id = idColumnMap.get(table.getColumnList().get(i).getColumnName()); - deviceIds[3 + idIndex] = id == null ? "" : id; - idIndex++; - } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { - String measurement = table.getColumnList().get(i).getColumnName(); - if (measurementColumnMap.containsKey(measurement)) { - measurements[measurementIndex] = measurement; - valueList[measurementIndex] = measurementColumnMap.get(measurement); - measurementIndex++; - } - } - } - insertStatement.setDevicePath(new PartialPath(deviceIds)); - TimestampPrecisionUtils.checkTimestampPrecision(time); - insertStatement.setTime(time); - insertStatement.setMeasurements(measurements); - insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]); - insertStatement.setValues(valueList); - insertStatement.setNeedInferType(true); - insertStatement.setAligned(true); - return insertStatement; - } - private TSExecuteStatementResp executeRawDataQueryInternal( TSRawDataQueryReq req, SelectResult setResult) { boolean finished = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index e242c53c7e52..27026437d603 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -101,6 +101,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -3644,4 +3645,27 @@ public Analysis visitShowCurrentTimestamp( analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowCurrentTimestampHeader()); return analysis; } + + @Override + public Analysis visitInsertTable( + InsertTableStatement insertTableStatement, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + Analysis analysis = new Analysis(); + InsertRowStatement insertRowStatement = insertTableStatement.getInsertRowStatement(); + validateSchema(analysis, insertRowStatement, context); + if (analysis.isFinishQueryAfterAnalyze()) { + return analysis; + } + analysis.setStatement(insertRowStatement); + + DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); + dataPartitionQueryParam.setDevicePath(insertRowStatement.getDevicePath().getFullPath()); + dataPartitionQueryParam.setTimePartitionSlotList( + Collections.singletonList(insertRowStatement.getTimePartitionSlot())); + + return getAnalysisForWriting( + analysis, + Collections.singletonList(dataPartitionQueryParam), + context.getSession().getUserName()); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8dba82f490b..2e8f04b8dfa8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTableStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -600,4 +601,8 @@ public R visitShowCurrentTimestamp( ShowCurrentTimestampStatement showCurrentTimestampStatement, C context) { return visitStatement(showCurrentTimestampStatement, context); } + + public R visitInsertTable(InsertTableStatement insertTableStatement, C context) { + return visitStatement(insertTableStatement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java new file mode 100644 index 000000000000..605c608efc91 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.Insert; +import org.apache.iotdb.db.relational.sql.tree.LongLiteral; +import org.apache.iotdb.db.relational.sql.tree.Row; +import org.apache.iotdb.db.relational.sql.tree.Values; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class InsertTableStatement extends Statement { + + private final InsertRowStatement insertRowStatement; + + public InsertTableStatement(IClientSession clientSession, Insert insert) { + insertRowStatement = parseInsert(clientSession, insert); + } + + private InsertRowStatement parseInsert(IClientSession clientSession, Insert insert) { + InsertRowStatement insertStatement = new InsertRowStatement(); + String database = clientSession.getDatabaseName(); + if (database == null) { + database = insert.getTable().getName().getPrefix().get().getSuffix(); + } + String tableName = insert.getTable().getName().getSuffix().toString(); + TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); + List values = + ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); + Map idColumnMap = new HashMap<>(); + Map attrColumnMap = new HashMap<>(); + Map measurementColumnMap = new HashMap<>(); + long time = 0L; + boolean hasColumn = insert.getColumns().isPresent(); + int size = hasColumn ? insert.getColumns().get().size() : table.getColumnNum(); + List columnNameList = hasColumn ? insert.getColumns().get() : null; + for (int i = 0; i < size; i++) { + String columnName = + hasColumn + ? columnNameList.get(i).getValue() + : table.getColumnList().get(i).getColumnName(); + TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); + if (category.equals(TsTableColumnCategory.ID)) { + idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.ATTRIBUTE)) { + attrColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + measurementColumnMap.put(columnName, ((LongLiteral) values.get(i)).getValue()); + } else { + time = Long.parseLong(((LongLiteral) values.get(i)).getValue()); + } + } + String[] deviceIds = new String[table.getIdNums() + 3]; + deviceIds[0] = PATH_ROOT; + deviceIds[1] = database; + deviceIds[2] = tableName; + String[] measurements = new String[measurementColumnMap.size()]; + Object[] valueList = new Object[measurements.length]; + int idIndex = 0; + int measurementIndex = 0; + for (int i = 0; i < table.getColumnNum(); i++) { + TsTableColumnCategory category = table.getColumnList().get(i).getColumnCategory(); + if (category.equals(TsTableColumnCategory.ID)) { + String id = idColumnMap.get(table.getColumnList().get(i).getColumnName()); + deviceIds[3 + idIndex] = id == null ? "" : id; + idIndex++; + } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { + String measurement = table.getColumnList().get(i).getColumnName(); + if (measurementColumnMap.containsKey(measurement)) { + measurements[measurementIndex] = measurement; + valueList[measurementIndex] = measurementColumnMap.get(measurement); + measurementIndex++; + } + } + } + insertStatement.setDevicePath(new PartialPath(deviceIds)); + TimestampPrecisionUtils.checkTimestampPrecision(time); + insertStatement.setTime(time); + insertStatement.setMeasurements(measurements); + insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]); + insertStatement.setValues(valueList); + insertStatement.setNeedInferType(true); + insertStatement.setAligned(true); + return insertStatement; + } + + public InsertRowStatement getInsertRowStatement() { + return insertRowStatement; + } + + @Override + public List getPaths() { + return insertRowStatement.getPaths(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitInsertTable(this, context); + } +} From a3e868536c8f890f71f727543bf328144238391f Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 17 Apr 2024 11:35:39 +0800 Subject: [PATCH 03/36] set up device creation --- .../schemaregion/SchemaExecutionVisitor.java | 13 ++ .../queryengine/common/MPPQueryContext.java | 10 + .../plan/analyze/AnalyzeVisitor.java | 52 ++++- .../plan/planner/LogicalPlanVisitor.java | 12 + .../plan/planner/plan/node/PlanNodeType.java | 5 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../metedata/write/CreateTableDeviceNode.java | 216 ++++++++++++++++++ .../schema/ITableDeviceSchemaValidation.java | 35 +++ .../schema/TableModelSchemaFetcher.java | 69 ++++++ .../metadata/TableMetadataImpl.java | 5 +- .../plan/statement/StatementVisitor.java | 6 + .../statement/crud/InsertTableStatement.java | 86 ++++++- .../internal/CreateTableDeviceStatement.java | 84 +++++++ .../schemaregion/ISchemaRegion.java | 10 + .../impl/SchemaRegionMemoryImpl.java | 9 + .../impl/SchemaRegionPBTreeImpl.java | 9 + 16 files changed, 610 insertions(+), 16 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 172b42ae38cb..5e284814bab2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; @@ -525,6 +526,18 @@ public TSStatus visitDeleteLogicalView(DeleteLogicalViewNode node, ISchemaRegion } } + @Override + public TSStatus visitCreateTableDevice(CreateTableDeviceNode node, ISchemaRegion schemaRegion) { + try { + schemaRegion.createTableDevice( + node.getDevicePathList(), node.getAttributeNameList(), node.getAttributeValueList()); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( PipeEnrichedWritePlanNode node, ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 56b738b4e4cb..d8b0c98208e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -74,6 +74,8 @@ public class MPPQueryContext { QueryPlanStatistics queryPlanStatistics = null; + private boolean skipSchemaValidate = false; + public MPPQueryContext(QueryId queryId) { this.queryId = queryId; this.endPointBlackList = new LinkedList<>(); @@ -289,4 +291,12 @@ public void setLogicalOptimizationCost(long logicalOptimizeCost) { } queryPlanStatistics.setLogicalOptimizationCost(logicalOptimizeCost); } + + public boolean isSkipSchemaValidate() { + return skipSchemaValidate; + } + + public void setSkipSchemaValidate(boolean skipSchemaValidate) { + this.skipSchemaValidate = skipSchemaValidate; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 27026437d603..3374178878d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -79,6 +79,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByVariationParameter; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; @@ -105,6 +106,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -2720,6 +2722,9 @@ public Analysis visitPipeEnrichedStatement( private void validateSchema( Analysis analysis, InsertBaseStatement insertStatement, MPPQueryContext context) { + if (context.isSkipSchemaValidate()) { + return; + } final long startTime = System.nanoTime(); try { SchemaValidator.validate(schemaFetcher, insertStatement, context); @@ -3652,20 +3657,47 @@ public Analysis visitInsertTable( context.setQueryType(QueryType.WRITE); Analysis analysis = new Analysis(); InsertRowStatement insertRowStatement = insertTableStatement.getInsertRowStatement(); - validateSchema(analysis, insertRowStatement, context); + + final long startTime = System.nanoTime(); + try { + TableModelSchemaFetcher.getInstance().validateDeviceSchema(insertTableStatement, context); + } catch (SemanticException e) { + analysis.setFinishQueryAfterAnalyze(true); + if (e.getCause() instanceof IoTDBException) { + IoTDBException exception = (IoTDBException) e.getCause(); + analysis.setFailStatus( + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); + } else { + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); + } + } finally { + PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost(System.nanoTime() - startTime); + } + if (analysis.isFinishQueryAfterAnalyze()) { return analysis; } - analysis.setStatement(insertRowStatement); + context.setSkipSchemaValidate(true); + return insertRowStatement.accept(this, context); + } - DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); - dataPartitionQueryParam.setDevicePath(insertRowStatement.getDevicePath().getFullPath()); - dataPartitionQueryParam.setTimePartitionSlotList( - Collections.singletonList(insertRowStatement.getTimePartitionSlot())); + @Override + public Analysis visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + Analysis analysis = new Analysis(); + analysis.setStatement(createTableDeviceStatement); - return getAnalysisForWriting( - analysis, - Collections.singletonList(dataPartitionQueryParam), - context.getSession().getUserName()); + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath devicePath : createTableDeviceStatement.getPaths()) { + patternTree.appendFullPath(devicePath.concatNode(ONE_LEVEL_PATH_WILDCARD)); + } + SchemaPartition partition = + partitionFetcher.getOrCreateSchemaPartition( + patternTree, context.getSession().getUserName()); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 87c04426010c..58d2a410d578 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -67,6 +68,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -1010,4 +1012,14 @@ public PlanNode visitShowLogicalView( .planLimit(showLogicalViewStatement.getLimit()) .getRoot(); } + + @Override + public PlanNode visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, MPPQueryContext context) { + return new CreateTableDeviceNode( + context.getQueryId().genPlanNodeId(), + createTableDeviceStatement.getPaths(), + createTableDeviceStatement.getAttributeNameList(), + createTableDeviceStatement.getAttributeValueList()); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index bda66843a979..c6fc8eaca39c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; @@ -209,6 +210,8 @@ public enum PlanNodeType { EXPLAIN_ANALYZE((short) 90), PIPE_OPERATE_SCHEMA_QUEUE_REFERENCE((short) 91), + + CREATE_TABLE_DEVICE((short) 92), ; public static final int BYTES = Short.BYTES; @@ -440,6 +443,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return ExplainAnalyzeNode.deserialize(buffer); case 91: return PipeOperateSchemaQueueNode.deserialize(buffer); + case 92: + return CreateTableDeviceNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index a9ce391d3534..1fa71fe9aa24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; @@ -553,4 +554,8 @@ public R visitTopK( org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode node, C context) { return visitPlan(node, context); } + + public R visitCreateTableDevice(CreateTableDeviceNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java new file mode 100644 index 000000000000..c36594039298 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java @@ -0,0 +1,216 @@ +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class CreateTableDeviceNode extends WritePlanNode { + + private final List devicePathList; + + private final List attributeNameList; + + private final List> attributeValueList; + + private TRegionReplicaSet regionReplicaSet; + + public CreateTableDeviceNode( + PlanNodeId id, + List devicePathList, + List attributeNameList, + List> attributeValueList) { + super(id); + this.devicePathList = devicePathList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + } + + public CreateTableDeviceNode( + PlanNodeId id, + TRegionReplicaSet regionReplicaSet, + List devicePathList, + List attributeNameList, + List> attributeValueList) { + super(id); + this.devicePathList = devicePathList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + this.regionReplicaSet = regionReplicaSet; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.CREATE_TABLE_DEVICE; + } + + public List getDevicePathList() { + return devicePathList; + } + + public List getAttributeNameList() { + return attributeNameList; + } + + public List> getAttributeValueList() { + return attributeValueList; + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return regionReplicaSet; + } + + @Override + public List getChildren() { + return new ArrayList<>(); + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return new CreateTableDeviceNode( + getPlanNodeId(), regionReplicaSet, devicePathList, attributeNameList, attributeValueList); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.CREATE_TABLE_DEVICE.serialize(byteBuffer); + ReadWriteIOUtils.write(devicePathList.size(), byteBuffer); + for (PartialPath deviceId : devicePathList) { + deviceId.serialize(byteBuffer); + } + ReadWriteIOUtils.write(attributeNameList.size(), byteBuffer); + for (String attributeName : attributeNameList) { + ReadWriteIOUtils.write(attributeName, byteBuffer); + } + ReadWriteIOUtils.write(attributeValueList.size(), byteBuffer); + for (List deviceValueList : attributeValueList) { + ReadWriteIOUtils.write(deviceValueList.size(), byteBuffer); + for (String value : deviceValueList) { + ReadWriteIOUtils.write(value, byteBuffer); + } + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.CREATE_TABLE_DEVICE.serialize(stream); + ReadWriteIOUtils.write(devicePathList.size(), stream); + for (PartialPath deviceId : devicePathList) { + deviceId.serialize(stream); + } + ReadWriteIOUtils.write(attributeNameList.size(), stream); + for (String attributeName : attributeNameList) { + ReadWriteIOUtils.write(attributeName, stream); + } + for (List deviceValueList : attributeValueList) { + for (String value : deviceValueList) { + ReadWriteIOUtils.write(value, stream); + } + } + } + + public static CreateTableDeviceNode deserialize(ByteBuffer buffer) { + int deviceNum = ReadWriteIOUtils.readInt(buffer); + List devicePathList = new ArrayList<>(deviceNum); + for (int i = 0; i < deviceNum; i++) { + devicePathList.add((PartialPath) PathDeserializeUtil.deserialize(buffer)); + } + int attributeNameNum = ReadWriteIOUtils.readInt(buffer); + List attributeNameList = new ArrayList<>(attributeNameNum); + for (int i = 0; i < attributeNameNum; i++) { + attributeNameList.add(ReadWriteIOUtils.readString(buffer)); + } + List> attributeValueList = new ArrayList<>(deviceNum); + for (int i = 0; i < deviceNum; i++) { + List deviceValueList = new ArrayList<>(attributeNameNum); + for (int j = 0; j < attributeNameNum; j++) { + deviceValueList.add(ReadWriteIOUtils.readString(buffer)); + } + attributeValueList.add(deviceValueList); + } + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new CreateTableDeviceNode( + planNodeId, devicePathList, attributeNameList, attributeValueList); + } + + @Override + public List splitByPartition(Analysis analysis) { + Map> splitMap = new HashMap<>(); + for (int i = 0; i < devicePathList.size(); i++) { + TRegionReplicaSet regionReplicaSet = + analysis + .getSchemaPartitionInfo() + .getSchemaRegionReplicaSet(devicePathList.get(i).getFullPath()); + splitMap.computeIfAbsent(regionReplicaSet, k -> new ArrayList<>()).add(i); + } + List result = new ArrayList<>(splitMap.size()); + for (Map.Entry> entry : splitMap.entrySet()) { + List subDevicePathList = new ArrayList<>(entry.getValue().size()); + List> subAttributeValueList = new ArrayList<>(entry.getValue().size()); + for (Integer index : entry.getValue()) { + subDevicePathList.add(devicePathList.get(index)); + subAttributeValueList.add(attributeValueList.get(index)); + } + result.add( + new CreateTableDeviceNode( + getPlanNodeId(), + entry.getKey(), + subDevicePathList, + attributeNameList, + subAttributeValueList)); + } + return result; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateTableDevice(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof CreateTableDeviceNode)) return false; + if (!super.equals(o)) return false; + CreateTableDeviceNode that = (CreateTableDeviceNode) o; + return Objects.equals(devicePathList, that.devicePathList) + && Objects.equals(attributeNameList, that.attributeNameList) + && Objects.equals(attributeValueList, that.attributeValueList) + && Objects.equals(regionReplicaSet, that.regionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), devicePathList, attributeNameList, attributeValueList, regionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java new file mode 100644 index 000000000000..7bc05c939255 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/ITableDeviceSchemaValidation.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import java.util.List; + +public interface ITableDeviceSchemaValidation { + + String getDatabase(); + + String getTableName(); + + List getDeviceIdList(); + + List getAttributeColumnNameList(); + + List> getAttributeValue(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java new file mode 100644 index 000000000000..74554645a0a0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; + +public class TableModelSchemaFetcher { + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private static class TableModelSchemaFetcherHolder { + private static final TableModelSchemaFetcher INSTANCE = new TableModelSchemaFetcher(); + } + + public static TableModelSchemaFetcher getInstance() { + return TableModelSchemaFetcherHolder.INSTANCE; + } + + private TableModelSchemaFetcher() { + // do nothing + } + + public void validateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) { + CreateTableDeviceStatement statement = + new CreateTableDeviceStatement( + schemaValidation.getDatabase(), + schemaValidation.getTableName(), + schemaValidation.getDeviceIdList(), + schemaValidation.getAttributeColumnNameList(), + schemaValidation.getAttributeValue()); + Coordinator.getInstance() + .executeForTreeModel( + statement, + SessionManager.getInstance().requestQueryId(), + context == null ? null : context.getSession(), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + context == null || context.getQueryType().equals(QueryType.WRITE) + ? config.getQueryTimeoutThreshold() + : context.getTimeOut()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index e5cfa4befb2e..dc9f5cfb339a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature; import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.tsfile.read.common.type.Type; @@ -48,9 +49,11 @@ public class TableMetadataImpl implements Metadata { private final TypeManager typeManager = new InternalTypeManager(); + private final DataNodeTableCache tableCache = DataNodeTableCache.getInstance(); + @Override public boolean tableExists(QualifiedObjectName name) { - return false; + return tableCache.getTable(name.getDatabaseName(), name.getObjectName()) != null; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 2e8f04b8dfa8..acb722fc933f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -605,4 +606,9 @@ public R visitShowCurrentTimestamp( public R visitInsertTable(InsertTableStatement insertTableStatement, C context) { return visitStatement(insertTableStatement, context); } + + public R visitCreateTableDevice( + CreateTableDeviceStatement createTableDeviceStatement, C context) { + return visitStatement(createTableDeviceStatement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index 605c608efc91..74a3641f3c68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -21,8 +21,11 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.ITableDeviceSchemaValidation; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.relational.sql.tree.Expression; @@ -34,28 +37,53 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; -public class InsertTableStatement extends Statement { +public class InsertTableStatement extends Statement implements ITableDeviceSchemaValidation { + + private final String database; + + private final String table; + + private List deviceIdList; + + private List attributeNameList; + + private List> attributeValueList; private final InsertRowStatement insertRowStatement; public InsertTableStatement(IClientSession clientSession, Insert insert) { - insertRowStatement = parseInsert(clientSession, insert); + this.database = parseDatabase(clientSession, insert); + this.table = parseTable(insert); + insertRowStatement = parseInsert(insert); } - private InsertRowStatement parseInsert(IClientSession clientSession, Insert insert) { - InsertRowStatement insertStatement = new InsertRowStatement(); + private String parseDatabase(IClientSession clientSession, Insert insert) { String database = clientSession.getDatabaseName(); if (database == null) { database = insert.getTable().getName().getPrefix().get().getSuffix(); } - String tableName = insert.getTable().getName().getSuffix().toString(); + return database; + } + + private String parseTable(Insert insert) { + return insert.getTable().getName().getSuffix().toString(); + } + + private InsertRowStatement parseInsert(Insert insert) { + InsertRowStatement insertStatement = new InsertRowStatement(); + String tableName = table; TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); List values = ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); @@ -88,6 +116,8 @@ private InsertRowStatement parseInsert(IClientSession clientSession, Insert inse deviceIds[2] = tableName; String[] measurements = new String[measurementColumnMap.size()]; Object[] valueList = new Object[measurements.length]; + MeasurementSchema[] schemas = new MeasurementSchema[measurements.length]; + TSDataType[] dataTypes = new TSDataType[measurements.length]; int idIndex = 0; int measurementIndex = 0; for (int i = 0; i < table.getColumnNum(); i++) { @@ -101,10 +131,21 @@ private InsertRowStatement parseInsert(IClientSession clientSession, Insert inse if (measurementColumnMap.containsKey(measurement)) { measurements[measurementIndex] = measurement; valueList[measurementIndex] = measurementColumnMap.get(measurement); + schemas[measurementIndex] = + ((MeasurementColumnSchema) table.getColumnList().get(i)).getMeasurementSchema(); + dataTypes[measurementIndex] = schemas[measurementIndex].getType(); measurementIndex++; } } } + + this.deviceIdList = + Collections.singletonList(Arrays.copyOfRange(deviceIds, 3, deviceIds.length)); + this.attributeNameList = new ArrayList<>(attrColumnMap.keySet()); + this.attributeValueList = + Collections.singletonList( + attributeNameList.stream().map(attrColumnMap::get).collect(Collectors.toList())); + insertStatement.setDevicePath(new PartialPath(deviceIds)); TimestampPrecisionUtils.checkTimestampPrecision(time); insertStatement.setTime(time); @@ -113,6 +154,16 @@ private InsertRowStatement parseInsert(IClientSession clientSession, Insert inse insertStatement.setValues(valueList); insertStatement.setNeedInferType(true); insertStatement.setAligned(true); + insertStatement.setMeasurementSchemas(schemas); + insertStatement.setDataTypes(dataTypes); + try { + for (int i = 0; i < measurements.length; i++) { + insertStatement.selfCheckDataTypes(i); + } + insertStatement.updateAfterSchemaValidation(); + } catch (Exception e) { + throw new SemanticException(e); + } return insertStatement; } @@ -129,4 +180,29 @@ public List getPaths() { public R accept(StatementVisitor visitor, C context) { return visitor.visitInsertTable(this, context); } + + @Override + public String getDatabase() { + return database; + } + + @Override + public String getTableName() { + return table; + } + + @Override + public List getDeviceIdList() { + return deviceIdList; + } + + @Override + public List getAttributeColumnNameList() { + return attributeNameList; + } + + @Override + public List> getAttributeValue() { + return attributeValueList; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java new file mode 100644 index 000000000000..11b0b0a0a0dd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java @@ -0,0 +1,84 @@ +package org.apache.iotdb.db.queryengine.plan.statement.internal; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class CreateTableDeviceStatement extends Statement { + + private final String database; + + private final String table; + + private final List deviceIdList; + + private final List attributeNameList; + + private final List> attributeValueList; + + private transient List devicePathList; + + public CreateTableDeviceStatement( + String database, + String table, + List deviceIdList, + List attributeNameList, + List> attributeValueList) { + this.database = database; + this.table = table; + this.deviceIdList = deviceIdList; + this.attributeNameList = attributeNameList; + this.attributeValueList = attributeValueList; + } + + public String getDatabase() { + return database; + } + + public String getTable() { + return table; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + public List getAttributeNameList() { + return attributeNameList; + } + + public List> getAttributeValueList() { + return attributeValueList; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitCreateTableDevice(this, context); + } + + @Override + public List getPaths() { + if (devicePathList == null) { + generateDevicePaths(); + } + return devicePathList; + } + + private void generateDevicePaths() { + List result = new ArrayList<>(deviceIdList.size()); + for (String[] deviceId : deviceIdList) { + String[] nodes = new String[3 + deviceId.length]; + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = table; + System.arraycopy(deviceId, 0, nodes, 3, deviceId.length); + result.add(new PartialPath(nodes)); + } + this.devicePathList = result; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 421f47ea238d..43120723863a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -293,6 +293,16 @@ long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) // endregion + // region table device management + + void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException; + + // endregion + // region Interfaces for SchemaReader ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index a7431b431682..9fb2a0cefbdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1204,6 +1204,15 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) return result; } + @Override + public void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException { + System.out.println(); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index c99b37c80073..668ff078b9b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1313,6 +1313,15 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) return result; } + @Override + public void createTableDevice( + List devicePathList, + List attributeNameList, + List> attributeValueList) + throws MetadataException { + throw new UnsupportedOperationException(); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { From d098bf7932addc90159bc06f730adbe91bb9761e Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 17 Apr 2024 15:27:15 +0800 Subject: [PATCH 04/36] implement device creation and attribute store --- .../schema/TableModelSchemaFetcher.java | 31 ++-- .../attribute/DeviceAttributeStore.java | 142 ++++++++++++++++++ .../attribute/IDeviceAttributeStore.java | 37 +++++ .../impl/SchemaRegionMemoryImpl.java | 31 +++- .../impl/mem/MTreeBelowSGMemoryImpl.java | 43 ++++++ .../impl/mem/mnode/info/TableDeviceInfo.java | 117 +++++++++++++++ .../iotdb/commons/schema/SchemaConstant.java | 4 + 7 files changed, 393 insertions(+), 12 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 74554645a0a0..7bec6837bfaa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.SessionManager; @@ -27,7 +28,9 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; +import org.apache.iotdb.rpc.TSStatusCode; public class TableModelSchemaFetcher { @@ -54,16 +57,22 @@ public void validateDeviceSchema( schemaValidation.getDeviceIdList(), schemaValidation.getAttributeColumnNameList(), schemaValidation.getAttributeValue()); - Coordinator.getInstance() - .executeForTreeModel( - statement, - SessionManager.getInstance().requestQueryId(), - context == null ? null : context.getSession(), - "", - ClusterPartitionFetcher.getInstance(), - ClusterSchemaFetcher.getInstance(), - context == null || context.getQueryType().equals(QueryType.WRITE) - ? config.getQueryTimeoutThreshold() - : context.getTimeOut()); + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + statement, + SessionManager.getInstance().requestQueryId(), + context == null ? null : context.getSession(), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + context == null || context.getQueryType().equals(QueryType.WRITE) + ? config.getQueryTimeoutThreshold() + : context.getTimeOut()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java new file mode 100644 index 000000000000..e1c5c1c676c3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute; + +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DeviceAttributeStore implements IDeviceAttributeStore { + + private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeStore.class); + + public List> deviceAttributeList = new ArrayList<>(); + + @Override + public void clear() { + deviceAttributeList = new ArrayList<>(); + } + + @Override + public synchronized boolean createSnapshot(File targetDir) { + File snapshotTmp = + SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT_TMP); + File snapshot = + SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT); + + try { + FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try { + serialize(outputStream); + } finally { + outputStream.flush(); + fileOutputStream.getFD().sync(); + outputStream.close(); + } + if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { + logger.error( + "Failed to delete old snapshot {} while creating device attribute snapshot.", + snapshot.getName()); + return false; + } + if (!snapshotTmp.renameTo(snapshot)) { + logger.error( + "Failed to rename {} to {} while creating device attribute snapshot.", + snapshotTmp.getName(), + snapshot.getName()); + FileUtils.deleteFileIfExist(snapshot); + return false; + } + + return true; + } catch (IOException e) { + logger.error("Failed to create mtree snapshot due to {}", e.getMessage(), e); + FileUtils.deleteFileIfExist(snapshot); + return false; + } finally { + FileUtils.deleteFileIfExist(snapshotTmp); + } + } + + @Override + public void loadFromSnapshot(File snapshotDir, String sgSchemaDirPath) throws IOException { + try (BufferedInputStream inputStream = + new BufferedInputStream( + Files.newInputStream( + SystemFileFactory.INSTANCE + .getFile(snapshotDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT) + .toPath()))) { + deserialize(inputStream); + } catch (IOException e) { + logger.warn("Load device attribute snapshot from {} failed", snapshotDir); + throw e; + } + } + + @Override + public synchronized int createAttribute(List nameList, List valueList) { + Map attributeMap = new HashMap<>(); + for (int i = 0; i < nameList.size(); i++) { + attributeMap.put(nameList.get(i), valueList.get(i)); + } + deviceAttributeList.add(attributeMap); + return deviceAttributeList.size() - 1; + } + + @Override + public void alterAttribute(int pointer, List nameList, List valueList) { + Map attributeMap = deviceAttributeList.get((int) pointer); + for (int i = 0; i < nameList.size(); i++) { + attributeMap.put(nameList.get(i), valueList.get(i)); + } + } + + private void serialize(OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(deviceAttributeList.size(), outputStream); + for (Map attributeMap : deviceAttributeList) { + ReadWriteIOUtils.write(attributeMap, outputStream); + } + } + + private void deserialize(InputStream inputStream) throws IOException { + int size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; i++) { + deviceAttributeList.add(ReadWriteIOUtils.readMap(inputStream)); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java new file mode 100644 index 000000000000..c5395a93a675 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public interface IDeviceAttributeStore { + + void clear(); + + boolean createSnapshot(File targetDir); + + void loadFromSnapshot(File snapshotDir, String sgSchemaDirPath) throws IOException; + + int createAttribute(List nameList, List valueList); + + void alterAttribute(int pointer, List nameList, List valueList); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 9fb2a0cefbdf..0cb39a678704 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -49,6 +49,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.DeviceAttributeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.IDeviceAttributeStore; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Deserializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Serializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.SchemaLogReader; @@ -158,6 +160,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { private MTreeBelowSGMemoryImpl mtree; private TagManager tagManager; + private IDeviceAttributeStore deviceAttributeStore; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionMemoryImpl(ISchemaRegionParams schemaRegionParams) throws MetadataException { @@ -196,6 +199,7 @@ public synchronized void init() throws MetadataException { // do not write log when recover isRecovering = true; + deviceAttributeStore = new DeviceAttributeStore(); tagManager = new TagManager(schemaRegionDirPath); mtree = new MTreeBelowSGMemoryImpl( @@ -432,6 +436,13 @@ public synchronized boolean createSnapshot(File snapshotDir) { schemaRegionId, System.currentTimeMillis() - tagSnapshotStartTime); + long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); + isSuccess = isSuccess && deviceAttributeStore.createSnapshot(snapshotDir); + logger.info( + "Device attribute snapshot creation of schemaRegion {} costs {}ms", + schemaRegionId, + System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + logger.info( "Snapshot creation of schemaRegion {} costs {}ms.", schemaRegionId, @@ -454,6 +465,14 @@ public void loadSnapshot(File latestSnapshotRootDir) { isRecovering = true; + long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); + deviceAttributeStore = new DeviceAttributeStore(); + deviceAttributeStore.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); + logger.info( + "Device attribute snapshot loading of schemaRegion {} costs {}ms.", + schemaRegionId, + System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + long tagSnapshotStartTime = System.currentTimeMillis(); tagManager = TagManager.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); logger.info( @@ -1210,7 +1229,17 @@ public void createTableDevice( List attributeNameList, List> attributeValueList) throws MetadataException { - System.out.println(); + for (int i = 0; i < devicePathList.size(); i++) { + int finalI = i; + mtree.createTableDevice( + devicePathList.get(i), + () -> + deviceAttributeStore.createAttribute( + attributeNameList, attributeValueList.get(finalI)), + pointer -> + deviceAttributeStore.alterAttribute( + pointer, attributeNameList, attributeValueList.get(finalI))); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 12d1a2c3d885..cd5a05512ff2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.EntityCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MNodeCollector; @@ -92,6 +93,8 @@ import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; +import java.util.function.IntSupplier; /** * The hierarchical struct of the Metadata Tree is implemented in this class. @@ -1252,4 +1255,44 @@ protected Void collectMeasurement(IMeasurementMNode node) { return result; } // endregion + + // region table device management + + public void createTableDevice( + PartialPath devicePath, IntSupplier attributePointerGetter, IntConsumer attributeUppdater) + throws MetadataException { + String[] nodeNames = devicePath.getNodes(); + IMemMNode cur = storageGroupMNode; + IMemMNode child; + for (int i = levelOfSG + 1; i < nodeNames.length; i++) { + child = cur.getChild(nodeNames[i]); + if (child == null) { + child = + store.addChild(cur, nodeNames[i], nodeFactory.createInternalMNode(cur, nodeNames[i])); + } + cur = child; + } + + IDeviceMNode entityMNode; + + synchronized (this) { + if (cur.isDevice()) { + entityMNode = cur.getAsDeviceMNode(); + if (!(entityMNode.getDeviceInfo() instanceof TableDeviceInfo)) { + throw new MetadataException("Table device shall not create under tree model"); + } + TableDeviceInfo deviceInfo = + (TableDeviceInfo) entityMNode.getDeviceInfo(); + attributeUppdater.accept(deviceInfo.getAttributePointer()); + } else { + entityMNode = store.setToEntity(cur); + TableDeviceInfo deviceInfo = new TableDeviceInfo<>(); + deviceInfo.setAttributePointer(attributePointerGetter.getAsInt()); + entityMNode.getAsInternalMNode().setDeviceInfo(deviceInfo); + regionStatistics.addDevice(); + } + } + } + + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java new file mode 100644 index 000000000000..dc067caf337d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info; + +import org.apache.iotdb.commons.schema.node.IMNode; +import org.apache.iotdb.commons.schema.node.info.IDeviceInfo; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; + +import java.util.Map; + +public class TableDeviceInfo> implements IDeviceInfo { + + private int attributePointer = -1; + + public int getAttributePointer() { + return attributePointer; + } + + public void setAttributePointer(int attributePointer) { + this.attributePointer = attributePointer; + } + + @Override + public void moveDataToNewMNode(IDeviceMNode newMNode) {} + + @Override + public boolean addAlias(String alias, IMeasurementMNode child) { + return false; + } + + @Override + public void deleteAliasChild(String alias) {} + + @Override + public Map> getAliasChildren() { + return null; + } + + @Override + public void setAliasChildren(Map> aliasChildren) {} + + @Override + public boolean hasAliasChild(String name) { + return false; + } + + @Override + public N getAliasChild(String name) { + return null; + } + + @Override + public boolean isUseTemplate() { + return false; + } + + @Override + public void setUseTemplate(boolean useTemplate) {} + + @Override + public void setSchemaTemplateId(int schemaTemplateId) {} + + @Override + public int getSchemaTemplateId() { + return 0; + } + + @Override + public int getSchemaTemplateIdWithState() { + return 0; + } + + @Override + public boolean isPreDeactivateTemplate() { + return false; + } + + @Override + public void preDeactivateTemplate() {} + + @Override + public void rollbackPreDeactivateTemplate() {} + + @Override + public void deactivateTemplate() {} + + @Override + public Boolean isAligned() { + return null; + } + + @Override + public void setAligned(Boolean isAligned) {} + + @Override + public int estimateSize() { + return 12; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java index 7c67375d31df..f045527778b8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java @@ -56,6 +56,10 @@ private SchemaConstant() { public static final String TAG_LOG_SNAPSHOT_TMP = "tlog.txt.snapshot.tmp"; public static final String MTREE_SNAPSHOT = "mtree.snapshot"; public static final String MTREE_SNAPSHOT_TMP = "mtree.snapshot.tmp"; + + public static final String DEVICE_ATTRIBUTE_SNAPSHOT = "device_attribute.snapshot"; + public static final String DEVICE_ATTRIBUTE_SNAPSHOT_TMP = "device_attribute.snapshot.tmp"; + public static final String SYSTEM_DATABASE = "root.__system"; public static final String[] ALL_RESULT_NODES = new String[] {"root", "**"}; public static final PartialPath ALL_MATCH_PATTERN = new PartialPath(ALL_RESULT_NODES); From 9417229f8d85321965b33922ecd7d5e168c719b8 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 17 Apr 2024 16:34:52 +0800 Subject: [PATCH 05/36] remove comment --- .../iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 3374178878d3..31cd6f2c127d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2439,7 +2439,7 @@ public Analysis visitInternalCreateTimeseries( Analysis analysis = new Analysis(); analysis.setStatement(internalCreateTimeSeriesStatement); - // checkIsTableCompatible(internalCreateTimeSeriesStatement.getDevicePath(), context); + checkIsTableCompatible(internalCreateTimeSeriesStatement.getDevicePath(), context); checkIsTemplateCompatible( internalCreateTimeSeriesStatement.getDevicePath(), internalCreateTimeSeriesStatement.getMeasurements(), From 238e2a72cb47c024fed82b7b7a0ecf1e5d9a8a53 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 17 Apr 2024 20:56:00 +0800 Subject: [PATCH 06/36] implement getTableSchema --- .../metadata/TableMetadataImpl.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index dc9f5cfb339a..a03f65fd8a3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; import org.apache.iotdb.commons.udf.builtin.BuiltinScalarFunction; +import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; @@ -33,10 +35,12 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.tsfile.read.common.type.Type; +import org.apache.iotdb.tsfile.read.common.type.TypeFactory; import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.iotdb.tsfile.read.common.type.BinaryType.TEXT; import static org.apache.iotdb.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -58,7 +62,22 @@ public boolean tableExists(QualifiedObjectName name) { @Override public Optional getTableSchema(SessionInfo session, QualifiedObjectName name) { - return null; + TsTable table = tableCache.getTable(name.getDatabaseName(), name.getObjectName()); + if (table == null) { + throw new SemanticException( + new TableNotExistsException(name.getDatabaseName(), name.getObjectName())); + } + List columnSchemaList = + table.getColumnList().stream() + .map( + o -> + new ColumnSchema( + o.getColumnName(), + TypeFactory.getType(o.getDataType()), + false, + o.getColumnCategory())) + .collect(Collectors.toList()); + return Optional.of(new TableSchema(table.getTableName(), columnSchemaList)); } @Override From 7a1728ce6c907dfda3a46055e6c205e608fedbb3 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Thu, 18 Apr 2024 10:17:17 +0800 Subject: [PATCH 07/36] implement getTableSchema --- .../analyzer/schema/TableModelSchemaFetcher.java | 12 ++++++++++++ .../plan/relational/metadata/TableMetadataImpl.java | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 7bec6837bfaa..ca4c7fee853d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -29,9 +29,13 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; +import org.apache.iotdb.db.relational.sql.tree.Expression; import org.apache.iotdb.rpc.TSStatusCode; +import java.util.List; + public class TableModelSchemaFetcher { private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -75,4 +79,12 @@ public void validateDeviceSchema( executionResult.status.getMessage(), executionResult.status.getCode())); } } + + public List fetchDeviceSchema( + String database, + String table, + List expressionList, + List attributeColumns) { + return null; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index a03f65fd8a3e..a993523ba22b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; @@ -255,7 +256,12 @@ public List indexScan( QualifiedObjectName tableName, List expressionList, List attributeColumns) { - return null; + return TableModelSchemaFetcher.getInstance() + .fetchDeviceSchema( + tableName.getDatabaseName(), + tableName.getObjectName(), + expressionList, + attributeColumns); } public static boolean isTwoNumericType(List argumentTypes) { From 7636d9b654c7c950e352fe9849205ef8b0a60f85 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Thu, 18 Apr 2024 10:24:10 +0800 Subject: [PATCH 08/36] mock --- .../relational/metadata/TableMetadataImpl.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index a993523ba22b..fa7302837a72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -35,9 +35,13 @@ import org.apache.iotdb.db.relational.sql.tree.Expression; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; +import org.apache.iotdb.tsfile.file.metadata.IDeviceID; +import org.apache.iotdb.tsfile.file.metadata.StringArrayDeviceID; import org.apache.iotdb.tsfile.read.common.type.Type; import org.apache.iotdb.tsfile.read.common.type.TypeFactory; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -256,12 +260,10 @@ public List indexScan( QualifiedObjectName tableName, List expressionList, List attributeColumns) { - return TableModelSchemaFetcher.getInstance() - .fetchDeviceSchema( - tableName.getDatabaseName(), - tableName.getObjectName(), - expressionList, - attributeColumns); + List result = new ArrayList<>(); + IDeviceID deviceID = new StringArrayDeviceID("beijing", "a_1"); + result.add(new DeviceEntry(deviceID, Arrays.asList("new", "low"))); + return result; } public static boolean isTwoNumericType(List argumentTypes) { From 59321ec74e04f2ba78b93b3dc617b0aa122e81a4 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Thu, 18 Apr 2024 14:19:16 +0800 Subject: [PATCH 09/36] fix snapshot --- .../impl/mem/mnode/info/TableDeviceInfo.java | 6 ++-- .../mem/snapshot/MemMTreeSnapshotUtil.java | 36 +++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java index dc067caf337d..857cf20a40d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/TableDeviceInfo.java @@ -26,6 +26,8 @@ import java.util.Map; +import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; + public class TableDeviceInfo> implements IDeviceInfo { private int attributePointer = -1; @@ -80,7 +82,7 @@ public void setSchemaTemplateId(int schemaTemplateId) {} @Override public int getSchemaTemplateId() { - return 0; + return NON_TEMPLATE; } @Override @@ -104,7 +106,7 @@ public void deactivateTemplate() {} @Override public Boolean isAligned() { - return null; + return true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java index c06f52067d68..e7a22f32a5c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -59,6 +60,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.MEASUREMENT_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_ENTITY_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_MNODE_TYPE; +import static org.apache.iotdb.commons.schema.SchemaConstant.TABLE_MNODE_TYPE; import static org.apache.iotdb.commons.schema.SchemaConstant.isStorageGroupType; public class MemMTreeSnapshotUtil { @@ -235,6 +237,11 @@ private static void deserializeMNode( node = deserializer.deserializeEntityMNode(inputStream); deviceProcess.accept(node.getAsDeviceMNode()); break; + case TABLE_MNODE_TYPE: + childrenNum = ReadWriteIOUtils.readInt(inputStream); + node = deserializer.deserializeTableDeviceMNode(inputStream); + deviceProcess.accept(node.getAsDeviceMNode()); + break; case STORAGE_GROUP_ENTITY_MNODE_TYPE: childrenNum = ReadWriteIOUtils.readInt(inputStream); node = deserializer.deserializeStorageGroupEntityMNode(inputStream); @@ -280,12 +287,20 @@ private static class MNodeSerializer extends MNodeVisitor public Boolean visitBasicMNode(IMNode node, OutputStream outputStream) { try { if (node.isDevice()) { - ReadWriteIOUtils.write(ENTITY_MNODE_TYPE, outputStream); - serializeBasicMNode(node, outputStream); - IDeviceMNode deviceMNode = node.getAsDeviceMNode(); - ReadWriteIOUtils.write(deviceMNode.getSchemaTemplateIdWithState(), outputStream); - ReadWriteIOUtils.write(deviceMNode.isUseTemplate(), outputStream); - ReadWriteIOUtils.write(deviceMNode.isAlignedNullable(), outputStream); + if (node.getAsDeviceMNode().getDeviceInfo() instanceof TableDeviceInfo) { + ReadWriteIOUtils.write(TABLE_MNODE_TYPE, outputStream); + TableDeviceInfo tableDeviceInfo = + (TableDeviceInfo) (node.getAsDeviceMNode().getDeviceInfo()); + serializeBasicMNode(node, outputStream); + ReadWriteIOUtils.write(tableDeviceInfo.getAttributePointer(), outputStream); + } else { + ReadWriteIOUtils.write(ENTITY_MNODE_TYPE, outputStream); + serializeBasicMNode(node, outputStream); + IDeviceMNode deviceMNode = node.getAsDeviceMNode(); + ReadWriteIOUtils.write(deviceMNode.getSchemaTemplateIdWithState(), outputStream); + ReadWriteIOUtils.write(deviceMNode.isUseTemplate(), outputStream); + ReadWriteIOUtils.write(deviceMNode.isAlignedNullable(), outputStream); + } } else { ReadWriteIOUtils.write(INTERNAL_MNODE_TYPE, outputStream); serializeBasicMNode(node, outputStream); @@ -394,6 +409,15 @@ public IMemMNode deserializeEntityMNode(InputStream inputStream) throws IOExcept return node.getAsMNode(); } + public IMemMNode deserializeTableDeviceMNode(InputStream inputStream) throws IOException { + String name = ReadWriteIOUtils.readString(inputStream); + IDeviceMNode node = nodeFactory.createDeviceMNode(null, name); + TableDeviceInfo tableDeviceInfo = new TableDeviceInfo<>(); + tableDeviceInfo.setAttributePointer(ReadWriteIOUtils.readInt(inputStream)); + node.getAsInternalMNode().setDeviceInfo(tableDeviceInfo); + return node.getAsMNode(); + } + public IMemMNode deserializeMeasurementMNode(InputStream inputStream) throws IOException { String name = ReadWriteIOUtils.readString(inputStream); MeasurementSchema schema = MeasurementSchema.deserializeFrom(inputStream); From 323ceb132ed460cb63508e980c15ecf717a3816a Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sat, 20 Apr 2024 10:30:18 +0800 Subject: [PATCH 10/36] merge --- .../plan/node/metedata/write/CreateTableDeviceNode.java | 3 ++- .../plan/relational/metadata/TableMetadataImpl.java | 1 - .../plan/statement/crud/InsertTableStatement.java | 5 +++-- .../schemaregion/attribute/DeviceAttributeStore.java | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java index c36594039298..229191129b4d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTableDeviceNode.java @@ -9,7 +9,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 4e50a6e1b307..09947ddcd762 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index 74a3641f3c68..28d5804c8482 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -36,8 +36,9 @@ import org.apache.iotdb.db.relational.sql.tree.Values; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import java.util.ArrayList; import java.util.Arrays; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index e1c5c1c676c3..048d6eb5285e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 34e5b0e5ac3a7a69906d50f7a6f9e4e62ea69d34 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 21 Apr 2024 21:18:23 +0800 Subject: [PATCH 11/36] implement schema filter --- .../confignode/conf/ConfigNodeConfig.java | 2 +- .../schema/source/SchemaSourceFactory.java | 11 + .../source/TableDeviceSchemaSource.java | 226 ++++++++++++++++++ .../plan/analyze/AnalyzeVisitor.java | 70 ++++++ .../plan/planner/LogicalPlanBuilder.java | 20 ++ .../plan/planner/LogicalPlanVisitor.java | 40 ++++ .../plan/planner/OperatorTreeGenerator.java | 22 ++ .../distribution/ExchangeNodeAdder.java | 6 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../metedata/read/TableDeviceScanNode.java | 141 +++++++++++ .../schema/TableModelSchemaFetcher.java | 212 +++++++++++++++- .../plan/relational/metadata/DeviceEntry.java | 10 + .../metadata/TableMetadataImpl.java | 18 +- .../planner/optimizations/IndexScan.java | 3 + .../plan/statement/StatementVisitor.java | 5 + .../metadata/ShowTableDevicesStatement.java | 69 ++++++ .../schemaregion/ISchemaRegion.java | 3 + .../attribute/DeviceAttributeStore.java | 7 +- .../attribute/IDeviceAttributeStore.java | 2 + .../impl/SchemaRegionMemoryImpl.java | 13 +- .../impl/SchemaRegionPBTreeImpl.java | 7 + .../impl/mem/MTreeBelowSGMemoryImpl.java | 82 ++++++- .../read/req/impl/ShowTableDevicesPlan.java | 40 ++++ .../read/resp/info/IDeviceSchemaInfo.java | 2 + .../resp/info/impl/ShowDevicesResult.java | 12 + .../utils/filter/DeviceFilterVisitor.java | 12 + .../schema/filter/SchemaFilterType.java | 11 + .../schema/filter/SchemaFilterVisitor.java | 15 ++ .../filter/impl/DeviceAttributeFilter.java | 63 +++++ .../schema/filter/impl/DeviceIdFilter.java | 63 +++++ .../commons/schema/filter/impl/OrFilter.java | 74 ++++++ .../db/relational/sql/parser/AstBuilder.java | 4 +- .../db/relational/sql/tree/ShowDevice.java | 67 ++++++ 33 files changed, 1317 insertions(+), 20 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java create mode 100644 iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 3875bf090806..3177bf0c0fd0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -101,7 +101,7 @@ public class ConfigNodeConfig { * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 2; + private int defaultDataRegionGroupNumPerDatabase = 1; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 7fe108425795..9e3028b62f00 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -98,4 +99,14 @@ public static ISchemaSource getLogicalViewSchemaSource( PathPatternTree scope) { return new LogicalViewSchemaSource(pathPattern, limit, offset, schemaFilter, scope); } + + public static ISchemaSource getTableDeviceSchemaSource( + String database, + String tableName, + List idDeterminedFilterList, + List idFuzzyFilterList, + List columnHeaderList) { + return new TableDeviceSchemaSource( + database, tableName, idDeterminedFilterList, idFuzzyFilterList, columnHeaderList); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java new file mode 100644 index 000000000000..529997611421 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -0,0 +1,226 @@ +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class TableDeviceSchemaSource implements ISchemaSource { + + private String database; + + private String tableName; + + private List idDeterminedFilterList; + + private List idFuzzyFilterList; + + private List columnHeaderList; + + public TableDeviceSchemaSource( + String database, + String tableName, + List idDeterminedFilterList, + List idFuzzyFilterList, + List columnHeaderList) { + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilterList = idFuzzyFilterList; + this.columnHeaderList = columnHeaderList; + } + + @Override + public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { + List devicePatternList = getDevicePatternList(); + return new ISchemaReader() { + + private ISchemaReader deviceReader; + private Throwable throwable; + private int index = 0; + + @Override + public boolean isSuccess() { + return throwable == null && (deviceReader == null || deviceReader.isSuccess()); + } + + @Override + public Throwable getFailure() { + if (throwable != null) { + return throwable; + } else if (deviceReader != null) { + return deviceReader.getFailure(); + } + return null; + } + + @Override + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + @Override + public boolean hasNext() { + try { + if (throwable != null) { + return false; + } + if (deviceReader != null) { + if (deviceReader.hasNext()) { + return true; + } else { + deviceReader.close(); + if (!deviceReader.isSuccess()) { + throwable = deviceReader.getFailure(); + return false; + } + } + } + + while (index < devicePatternList.size()) { + deviceReader = + schemaRegion.getTableDeviceReader( + new ShowTableDevicesPlan(devicePatternList.get(index), idFuzzyFilterList)); + index++; + if (deviceReader.hasNext()) { + return true; + } else { + deviceReader.close(); + } + } + return false; + } catch (Exception e) { + throw new SchemaExecutionException(e.getMessage(), e); + } + } + + @Override + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return deviceReader.next(); + } + + @Override + public void close() throws Exception { + if (deviceReader != null) { + deviceReader.close(); + } + } + }; + } + + private List getDevicePatternList() { + int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3; + String[] nodes = new String[length]; + Arrays.fill(nodes, "*"); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + Map> orValueMap = new HashMap<>(); + for (SchemaFilter schemaFilter : idDeterminedFilterList) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) + && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + orValueMap + .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) + .add(deviceIdFilter.getValue()); + } + } + } + + PartialPath path = new PartialPath(nodes); + List pathList = new ArrayList<>(); + pathList.add(path); + for (Map.Entry> entry : orValueMap.entrySet()) { + for (int i = 0, size = pathList.size(); i < size; i++) { + for (String value : entry.getValue()) { + nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); + nodes[entry.getKey() + 3] = value; + path = new PartialPath(nodes); + pathList.add(path); + } + } + } + + return pathList; + } + + @Override + public List getInfoQueryColumnHeaders() { + return columnHeaderList; + } + + @Override + public void transformToTsBlockColumns( + IDeviceSchemaInfo schemaInfo, TsBlockBuilder builder, String database) { + builder.getTimeColumnBuilder().writeLong(0L); + int resultIndex = 0; + int idIndex = 0; + PartialPath devicePath = schemaInfo.getPartialPath(); + TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); + TsTableColumnSchema columnSchema; + for (ColumnHeader columnHeader : columnHeaderList) { + columnSchema = table.getColumnSchema(columnHeader.getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary(devicePath.getNodes()[idIndex + 3], TSFileConfig.STRING_CHARSET)); + idIndex++; + } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } + resultIndex++; + } + builder.declarePosition(); + } + + @Override + public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { + return false; + } + + @Override + public long getSchemaStatistic(ISchemaRegion schemaRegion) { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index f60893d2cd71..bd94bf2b0fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -33,6 +33,11 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -127,6 +132,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement; @@ -180,6 +186,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND; import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME; @@ -3700,4 +3707,67 @@ public Analysis visitCreateTableDevice( return analysis; } + + @Override + public Analysis visitShowTableDevices( + ShowTableDevicesStatement statement, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + Analysis analysis = new Analysis(); + analysis.setStatement(statement); + + String database = statement.getDatabase(); + String tableName = statement.getTableName(); + List columnSchemaList = + DataNodeTableCache.getInstance().getTable(database, tableName).getColumnList(); + + int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3 + 1; + String[] nodes = new String[length]; + Arrays.fill(nodes, "*"); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + nodes[nodes.length - 1] = ONE_LEVEL_PATH_WILDCARD; + Map> orValueMap = new HashMap<>(); + for (SchemaFilter schemaFilter : statement.getIdDeterminedFilterList()) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) + && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + orValueMap + .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) + .add(deviceIdFilter.getValue()); + } + } + } + PathPatternTree patternTree = new PathPatternTree(); + PartialPath path = new PartialPath(nodes); + patternTree.appendFullPath(path); + List pathList = new ArrayList<>(); + pathList.add(path); + for (Map.Entry> entry : orValueMap.entrySet()) { + for (int i = 0, size = pathList.size(); i < size; i++) { + for (String value : entry.getValue()) { + nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); + nodes[entry.getKey() + 3] = value; + path = new PartialPath(nodes); + pathList.add(path); + patternTree.appendFullPath(path); + } + } + } + + SchemaPartition partition = + partitionFetcher.getOrCreateSchemaPartition( + patternTree, context.getSession().getUserName()); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 2a56d0e81406..1d31828c51b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; import org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil; @@ -54,6 +55,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; @@ -1642,4 +1644,22 @@ public LogicalPlanBuilder planEndTimeColumnInject( new SlidingTimeColumnGeneratorParameter(groupByTimeParameter, ascending)); return this; } + + public LogicalPlanBuilder planTableDeviceSource( + String database, + String tableName, + List idDeterminedFilterList, + List idFuzzyFilterList, + List columnHeaderList) { + this.root = + new TableDeviceScanNode( + context.getQueryId().genPlanNodeId(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilterList, + columnHeaderList, + null); + return this; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index ffd75a5b42c1..d61ba35b1645 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -21,10 +21,14 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer; import org.apache.iotdb.db.queryengine.plan.expression.Expression; @@ -84,6 +88,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowChildNodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowChildPathsStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement; @@ -93,6 +98,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.commons.lang3.StringUtils; @@ -1022,4 +1028,38 @@ public PlanNode visitCreateTableDevice( createTableDeviceStatement.getAttributeNameList(), createTableDeviceStatement.getAttributeValueList()); } + + @Override + public PlanNode visitShowTableDevices( + ShowTableDevicesStatement statement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + + List columnSchemaList = + DataNodeTableCache.getInstance() + .getTable(statement.getDatabase(), statement.getTableName()) + .getColumnList(); + + List columnHeaderList = new ArrayList<>(columnSchemaList.size()); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID) + || columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + columnHeaderList.add( + new ColumnHeader(columnSchema.getColumnName(), columnSchema.getDataType())); + } + } + + analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true)); + + planBuilder = + planBuilder + .planTableDeviceSource( + statement.getDatabase(), + statement.getTableName(), + statement.getIdDeterminedFilterList(), + statement.getIdFuzzyFilterList(), + columnHeaderList) + .planSchemaQueryMerge(false); + + return planBuilder.getRoot(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 3b13ac78a9d6..cc3326e2cf6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -173,6 +173,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; @@ -3262,4 +3263,25 @@ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanC return new ExplainAnalyzeOperator( operatorContext, operator, node.getQueryId(), node.isVerbose(), node.getTimeout()); } + + @Override + public Operator visitTableDeviceScan( + TableDeviceScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaQueryScanOperator.class.getSimpleName()); + return new SchemaQueryScanOperator<>( + node.getPlanNodeId(), + operatorContext, + SchemaSourceFactory.getTableDeviceSchemaSource( + node.getDatabase(), + node.getTableName(), + node.getIdDeterminedFilterList(), + node.getIdFuzzyFilterList(), + node.getColumnHeaderList())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index 3ae7a3f126f5..0c19a738a5d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.DeviceMergeNode; @@ -154,6 +155,11 @@ public PlanNode visitSchemaQueryScan(SchemaQueryScanNode node, NodeGroupContext return processNoChildSourceNode(node, context); } + @Override + public PlanNode visitTableDeviceScan(TableDeviceScanNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + @Override public PlanNode visitSchemaFetchScan(SchemaFetchScanNode node, NodeGroupContext context) { return processNoChildSourceNode(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 1fa71fe9aa24..a417be3f9b70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -558,4 +559,8 @@ public R visitTopK( public R visitCreateTableDevice(CreateTableDeviceNode node, C context) { return visitPlan(node, context); } + + public R visitTableDeviceScan(TableDeviceScanNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java new file mode 100644 index 000000000000..3605963c6bc6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java @@ -0,0 +1,141 @@ +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class TableDeviceScanNode extends SchemaQueryScanNode { + + private String database; + + private String tableName; + + private List idDeterminedFilterList; + + private List idFuzzyFilterList; + + private List columnHeaderList; + + private TRegionReplicaSet schemaRegionReplicaSet; + + public TableDeviceScanNode(PlanNodeId id) { + super(id); + } + + public TableDeviceScanNode( + PlanNodeId id, + String database, + String tableName, + List idDeterminedFilterList, + List idFuzzyFilterList, + List columnHeaderList, + TRegionReplicaSet regionReplicaSet) { + super(id); + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilterList = idFuzzyFilterList; + this.columnHeaderList = columnHeaderList; + this.schemaRegionReplicaSet = regionReplicaSet; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getIdDeterminedFilterList() { + return idDeterminedFilterList; + } + + public List getIdFuzzyFilterList() { + return idFuzzyFilterList; + } + + public List getColumnHeaderList() { + return columnHeaderList; + } + + @Override + public void open() throws Exception {} + + @Override + public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { + this.schemaRegionReplicaSet = regionReplicaSet; + } + + @Override + public void close() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public PlanNode clone() { + return new TableDeviceScanNode( + getPlanNodeId(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilterList, + columnHeaderList, + schemaRegionReplicaSet); + } + + @Override + public List getOutputColumnNames() { + return columnHeaderList.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException {} + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitTableDeviceScan(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceScanNode)) return false; + if (!super.equals(o)) return false; + TableDeviceScanNode that = (TableDeviceScanNode) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(idDeterminedFilterList, that.idDeterminedFilterList) + && Objects.equals(idFuzzyFilterList, that.idFuzzyFilterList) + && Objects.equals(columnHeaderList, that.columnHeaderList) + && Objects.equals(schemaRegionReplicaSet, that.schemaRegionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + database, + tableName, + idDeterminedFilterList, + idFuzzyFilterList, + columnHeaderList, + schemaRegionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index ca4c7fee853d..231708e71b89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -20,10 +20,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; @@ -31,10 +40,31 @@ import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.Literal; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.StringLiteral; +import org.apache.iotdb.db.relational.sql.tree.SymbolReference; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.Pair; + +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; public class TableModelSchemaFetcher { @@ -85,6 +115,186 @@ public List fetchDeviceSchema( String table, List expressionList, List attributeColumns) { - return null; + List deviceEntryList = new ArrayList<>(); + + Coordinator coordinator = Coordinator.getInstance(); + long queryId = SessionManager.getInstance().requestQueryId(); + Throwable t = null; + + TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + Pair, List> filters = + transformExpression(expressionList, tableInstance); + List idFilters = filters.getLeft(); + List attributeFilters = filters.getRight(); + ShowTableDevicesStatement statement = + new ShowTableDevicesStatement(database, table, idFilters, attributeFilters); + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + statement, + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + config.getQueryTimeoutThreshold()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + + List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + Map attributeMap; + + try { + while (coordinator.getQueryExecution(queryId).hasNextResult()) { + Optional tsBlock; + try { + tsBlock = coordinator.getQueryExecution(queryId).getBatchResult(); + } catch (IoTDBException e) { + t = e; + throw new RuntimeException("Fetch Table Device Schema failed. ", e); + } + if (!tsBlock.isPresent() || tsBlock.get().isEmpty()) { + break; + } + Column[] columns = tsBlock.get().getValueColumns(); + for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { + String[] nodes = new String[idLength + 1]; + nodes[0] = database + PATH_SEPARATOR + table; + int idIndex = 0; + attributeMap = new HashMap<>(); + for (int j = 0; j < columnHeaderList.size(); j++) { + TsTableColumnSchema columnSchema = + tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + nodes[idIndex + 1] = columns[j].getBinary(i).toString(); + idIndex++; + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } + } + IDeviceID deviceID = new StringArrayDeviceID(nodes); + deviceEntryList.add( + new DeviceEntry( + deviceID, + attributeColumns.stream().map(attributeMap::get).collect(Collectors.toList()))); + } + } + } catch (Throwable throwable) { + t = throwable; + throw throwable; + } finally { + coordinator.cleanupQueryExecution(queryId, null, t); + } + System.out.println(deviceEntryList); + return deviceEntryList; + } + + private Pair, List> transformExpression( + List expressionList, TsTable table) { + List idDeterminedFilters = new ArrayList<>(); + List idFuzzyFilters = new ArrayList<>(); + Map indexMap = getIdColumnIndex(table); + for (Expression expression : expressionList) { + if (expression == null) { + continue; + } + if (expression instanceof LogicalExpression) { + LogicalExpression logicalExpression = (LogicalExpression) expression; + SchemaFilter schemaFilter = transformToSchemaFilter(logicalExpression, table, indexMap); + if (hasAttribute(schemaFilter)) { + idFuzzyFilters.add(schemaFilter); + } else { + idDeterminedFilters.add(schemaFilter); + } + } else { + SchemaFilter schemaFilter = + transformToSchemaFilter((ComparisonExpression) expression, table, indexMap); + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE)) { + idFuzzyFilters.add(schemaFilter); + } else { + idDeterminedFilters.add(schemaFilter); + } + } + } + return new Pair<>(idDeterminedFilters, idFuzzyFilters); + } + + private boolean hasAttribute(SchemaFilter schemaFilter) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + return hasAttribute(((OrFilter) schemaFilter).getLeft()) + || hasAttribute(((OrFilter) schemaFilter).getRight()); + } + + return schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE); + } + + private SchemaFilter transformToSchemaFilter( + LogicalExpression logicalExpression, TsTable table, Map indexMap) { + SchemaFilter left; + SchemaFilter right; + if (logicalExpression.getTerms().get(0) instanceof LogicalExpression) { + left = + transformToSchemaFilter( + (LogicalExpression) (logicalExpression.getChildren().get(0)), table, indexMap); + } else { + left = + transformToSchemaFilter( + (ComparisonExpression) (logicalExpression.getChildren().get(0)), table, indexMap); + } + if (logicalExpression.getTerms().get(1) instanceof LogicalExpression) { + right = + transformToSchemaFilter( + (LogicalExpression) (logicalExpression.getChildren().get(1)), table, indexMap); + } else { + right = + transformToSchemaFilter( + (ComparisonExpression) (logicalExpression.getChildren().get(1)), table, indexMap); + } + return new OrFilter(left, right); + } + + private SchemaFilter transformToSchemaFilter( + ComparisonExpression comparisonExpression, TsTable table, Map indexMap) { + String columnName; + String value; + if (comparisonExpression.getLeft() instanceof Literal) { + value = ((StringLiteral) (comparisonExpression.getLeft())).getValue(); + if (comparisonExpression.getRight() instanceof Identifier) { + columnName = ((Identifier) (comparisonExpression.getRight())).getValue(); + } else { + columnName = ((SymbolReference) (comparisonExpression.getRight())).getName(); + } + } else { + value = ((StringLiteral) (comparisonExpression.getRight())).getValue(); + if (comparisonExpression.getLeft() instanceof Identifier) { + columnName = ((Identifier) (comparisonExpression.getLeft())).getValue(); + } else { + columnName = ((SymbolReference) (comparisonExpression.getLeft())).getName(); + } + } + if (table.getColumnSchema(columnName).getColumnCategory().equals(TsTableColumnCategory.ID)) { + return new DeviceIdFilter(indexMap.get(columnName), value); + } else { + return new DeviceAttributeFilter(columnName, value); + } + } + + private Map getIdColumnIndex(TsTable table) { + Map map = new HashMap<>(); + List columnSchemaList = table.getColumnList(); + int idIndex = 0; + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + map.put(columnSchema.getColumnName(), idIndex); + idIndex++; + } + } + return map; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java index 81b34e32c8b3..cac441ef1f1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java @@ -40,4 +40,14 @@ public IDeviceID getDeviceID() { public List getAttributeColumnValues() { return attributeColumnValues; } + + @Override + public String toString() { + return "DeviceEntry{" + + "deviceID=" + + deviceID + + ", attributeColumnValues=" + + attributeColumnValues + + '}'; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 09947ddcd762..ce0d72e104b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; @@ -35,13 +36,9 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeFactory; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -260,13 +257,12 @@ public List indexScan( QualifiedObjectName tableName, List expressionList, List attributeColumns) { - // fixme, perfect the real metadata impl - List result = new ArrayList<>(); - IDeviceID deviceID1 = new StringArrayDeviceID("db.table1", "beijing", "a_1"); - IDeviceID deviceID2 = new StringArrayDeviceID("db.table1", "beijing", "b_1"); - result.add(new DeviceEntry(deviceID1, Arrays.asList("old", "low"))); - result.add(new DeviceEntry(deviceID2, Arrays.asList("new", "high"))); - return result; + return TableModelSchemaFetcher.getInstance() + .fetchDeviceSchema( + tableName.getDatabaseName(), + tableName.getObjectName(), + expressionList, + attributeColumns); } public static boolean isTwoNumericType(List argumentTypes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java index 0307f9bb0feb..a13551d3164f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java @@ -83,6 +83,9 @@ public PlanNode visitPlan(PlanNode node, RewriterContext context) { @Override public PlanNode visitFilter(FilterNode node, RewriterContext context) { context.setPredicate(node.getPredicate()); + for (PlanNode child : node.getChildren()) { + child.accept(this, context); + } return node; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index acb722fc933f..be7f36de3fb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -71,6 +71,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowFunctionsStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; @@ -611,4 +612,8 @@ public R visitCreateTableDevice( CreateTableDeviceStatement createTableDeviceStatement, C context) { return visitStatement(createTableDeviceStatement, context); } + + public R visitShowTableDevices(ShowTableDevicesStatement statement, C context) { + return visitStatement(statement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java new file mode 100644 index 000000000000..0cb634830cd7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.List; + +public class ShowTableDevicesStatement extends ShowStatement { + + private final String database; + + private final String tableName; + + private final List idDeterminedFilterList; + + private final List idFuzzyFilterList; + + public ShowTableDevicesStatement( + String database, + String tableName, + List idDeterminedFilterList, + List idFuzzyFilterList) { + super(); + this.database = database; + this.tableName = tableName; + this.idDeterminedFilterList = idDeterminedFilterList; + this.idFuzzyFilterList = idFuzzyFilterList; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getIdDeterminedFilterList() { + return idDeterminedFilterList; + } + + public List getIdFuzzyFilterList() { + return idFuzzyFilterList; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowTableDevices(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 43120723863a..a8f814d4a9ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -314,5 +315,7 @@ ISchemaReader getTimeSeriesReader(IShowTimeSeriesPlan sho ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan) throws MetadataException; + ISchemaReader getTableDeviceReader(ShowTableDevicesPlan showTableDevicesPlan) + throws MetadataException; // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 048d6eb5285e..6a65fe17f1ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -120,12 +120,17 @@ public synchronized int createAttribute(List nameList, List valu @Override public void alterAttribute(int pointer, List nameList, List valueList) { - Map attributeMap = deviceAttributeList.get((int) pointer); + Map attributeMap = deviceAttributeList.get(pointer); for (int i = 0; i < nameList.size(); i++) { attributeMap.put(nameList.get(i), valueList.get(i)); } } + @Override + public String getAttribute(int pointer, String name) { + return deviceAttributeList.get(pointer).get(name); + } + private void serialize(OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(deviceAttributeList.size(), outputStream); for (Map attributeMap : deviceAttributeList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java index c5395a93a675..79f44d20bd20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java @@ -34,4 +34,6 @@ public interface IDeviceAttributeStore { int createAttribute(List nameList, List valueList); void alterAttribute(int pointer, List nameList, List valueList); + + String getAttribute(int pointer, String name); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 13913fbb2ec7..3aae464def17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -62,6 +62,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -1245,7 +1246,8 @@ public void createTableDevice( @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { - return mtree.getDeviceReader(showDevicesPlan); + return mtree.getDeviceReader( + showDevicesPlan, (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); } @Override @@ -1275,6 +1277,15 @@ public ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan return mtree.getNodeReader(showNodesPlan); } + @Override + public ISchemaReader getTableDeviceReader( + ShowTableDevicesPlan showTableDevicesPlan) throws MetadataException { + return mtree.getDeviceReader( + showTableDevicesPlan.getDevicePattern(), + showTableDevicesPlan.getAttributeFilter(), + (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index f0095babf9ac..6cd1fcd1f937 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -63,6 +63,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTableDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; @@ -1354,6 +1355,12 @@ public ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan throws MetadataException { return mtree.getNodeReader(showNodesPlan); } + + @Override + public ISchemaReader getTableDeviceReader( + ShowTableDevicesPlan showTableDevicesPlan) { + throw new UnsupportedOperationException(); + } // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 3c453801c7d1..d78b8325a370 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; @@ -91,6 +92,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntConsumer; @@ -951,7 +953,8 @@ public long countPathsUsingTemplate(PartialPath pathPattern, int templateId) // region Interfaces for schema reader @SuppressWarnings("java:S2095") - public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) + public ISchemaReader getDeviceReader( + IShowDevicesPlan showDevicesPlan, BiFunction attributeProvider) throws MetadataException { EntityCollector collector = new EntityCollector( @@ -963,8 +966,15 @@ public ISchemaReader getDeviceReader(IShowDevicesPlan showDev protected IDeviceSchemaInfo collectEntity(IDeviceMNode node) { PartialPath device = getPartialPathFromRootToNode(node.getAsMNode()); - return new ShowDevicesResult( - device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + ShowDevicesResult result = + new ShowDevicesResult( + device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + result.setAttributeProvider( + k -> + attributeProvider.apply( + ((TableDeviceInfo) node.getDeviceInfo()).getAttributePointer(), + k)); + return result; } }; if (showDevicesPlan.usingSchemaTemplate()) { @@ -995,7 +1005,8 @@ public ListenableFuture isBlocked() { public boolean hasNext() { while (next == null && collector.hasNext()) { IDeviceSchemaInfo temp = collector.next(); - if (filterVisitor.process(showDevicesPlan.getSchemaFilter(), temp)) { + if (showDevicesPlan.getSchemaFilter() == null + || filterVisitor.process(showDevicesPlan.getSchemaFilter(), temp)) { next = temp; } } @@ -1019,6 +1030,69 @@ public IDeviceSchemaInfo next() { } } + public ISchemaReader getDeviceReader( + PartialPath pattern, + SchemaFilter attributeFilter, + BiFunction attributeProvider) + throws MetadataException { + EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, null) { + + protected IDeviceSchemaInfo collectEntity(IDeviceMNode node) { + PartialPath device = getPartialPathFromRootToNode(node.getAsMNode()); + ShowDevicesResult result = + new ShowDevicesResult( + device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + result.setAttributeProvider( + k -> + attributeProvider.apply( + ((TableDeviceInfo) node.getDeviceInfo()).getAttributePointer(), + k)); + return result; + } + }; + return new ISchemaReader() { + + private final DeviceFilterVisitor filterVisitor = new DeviceFilterVisitor(); + private IDeviceSchemaInfo next; + + public boolean isSuccess() { + return collector.isSuccess(); + } + + public Throwable getFailure() { + return collector.getFailure(); + } + + public void close() { + collector.close(); + } + + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + public boolean hasNext() { + while (next == null && collector.hasNext()) { + IDeviceSchemaInfo temp = collector.next(); + if (attributeFilter == null || filterVisitor.process(attributeFilter, temp)) { + next = temp; + } + } + return next != null; + } + + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + IDeviceSchemaInfo result = next; + next = null; + return result; + } + }; + } + public ISchemaReader getTimeSeriesReader( IShowTimeSeriesPlan showTimeSeriesPlan, Function, Map>> tagAndAttributeProvider) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java new file mode 100644 index 000000000000..d3af79e093d8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java @@ -0,0 +1,40 @@ +package org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.AndFilter; + +import java.util.List; + +public class ShowTableDevicesPlan { + + private PartialPath devicePattern; + + private SchemaFilter attributeFilter; + + public ShowTableDevicesPlan(PartialPath devicePattern, List attributeFilterList) { + this.devicePattern = devicePattern; + this.attributeFilter = getAttributeFilter(attributeFilterList); + } + + private SchemaFilter getAttributeFilter(List filterList) { + if (filterList.isEmpty()) { + return null; + } + AndFilter andFilter; + SchemaFilter latestFilter = filterList.get(0); + for (int i = 1; i < filterList.size(); i++) { + andFilter = new AndFilter(latestFilter, filterList.get(i)); + latestFilter = andFilter; + } + return latestFilter; + } + + public PartialPath getDevicePattern() { + return devicePattern; + } + + public SchemaFilter getAttributeFilter() { + return attributeFilter; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java index 9a670f9b44bf..634f7d654615 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java @@ -24,4 +24,6 @@ public interface IDeviceSchemaInfo extends ISchemaInfo { Boolean isAligned(); int getTemplateId(); + + String getAttributeValue(String attributeKey); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java index 19aa19710082..8f4c5b95a6bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java @@ -21,11 +21,14 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import java.util.Objects; +import java.util.function.Function; public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchemaInfo { private Boolean isAligned; private int templateId; + private Function attributeProvider; + public ShowDevicesResult(String name, Boolean isAligned, int templateId) { super(name); this.isAligned = isAligned; @@ -40,6 +43,15 @@ public int getTemplateId() { return templateId; } + public void setAttributeProvider(Function attributeProvider) { + this.attributeProvider = attributeProvider; + } + + @Override + public String getAttributeValue(String attributeKey) { + return attributeProvider.apply(attributeKey); + } + @Override public String toString() { return "ShowDevicesResult{" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java index e1723193abfe..0555aecf5ea4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; @@ -59,4 +61,14 @@ public boolean visitTemplateFilter(TemplateFilter templateFilter, IDeviceSchemaI return false; } } + + @Override + public boolean visitDeviceIdFilter(DeviceIdFilter filter, IDeviceSchemaInfo info) { + return info.getPartialPath().getNodes()[filter.getIndex() + 3].equals(filter.getValue()); + } + + @Override + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, IDeviceSchemaInfo info) { + return info.getAttributeValue(filter.getKey()).equals(filter.getValue()); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java index 285130f778e9..66e9f15e939e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterType.java @@ -26,6 +26,11 @@ public enum SchemaFilterType { VIEW_TYPE((short) 4), AND((short) 5), TEMPLATE_FILTER((short) 6), + + DEVICE_ID((short) 7), + DEVICE_ATTRIBUTE((short) 8), + + OR((short) 9), ; private final short code; @@ -54,6 +59,12 @@ public static SchemaFilterType getSchemaFilterType(short code) { return AND; case 6: return TEMPLATE_FILTER; + case 7: + return DEVICE_ID; + case 8: + return DEVICE_ATTRIBUTE; + case 9: + return OR; default: throw new IllegalArgumentException("Invalid input: " + code); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java index d6861baab683..f503da64b15f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilterVisitor.java @@ -20,6 +20,9 @@ import org.apache.iotdb.commons.schema.filter.impl.AndFilter; import org.apache.iotdb.commons.schema.filter.impl.DataTypeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TagFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; @@ -69,4 +72,16 @@ public boolean visitTemplateFilter(TemplateFilter templateFilter, C context) { public boolean visitAndFilter(AndFilter andFilter, C context) { return andFilter.getLeft().accept(this, context) && andFilter.getRight().accept(this, context); } + + public boolean visitOrFilter(OrFilter orFilter, C context) { + return orFilter.getLeft().accept(this, context) || orFilter.getRight().accept(this, context); + } + + public boolean visitDeviceIdFilter(DeviceIdFilter filter, C context) { + return visitFilter(filter, context); + } + + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, C context) { + return visitDeviceAttributeFilter(filter, context); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java new file mode 100644 index 000000000000..09736bf5b190 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DeviceAttributeFilter extends SchemaFilter { + + private final String key; + + private final String value; + + public DeviceAttributeFilter(String key, String value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitDeviceAttributeFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.DEVICE_ATTRIBUTE; + } + + @Override + public void serialize(ByteBuffer byteBuffer) {} + + @Override + public void serialize(DataOutputStream stream) throws IOException {} +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java new file mode 100644 index 000000000000..eda64d90a8d2 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DeviceIdFilter extends SchemaFilter { + + private final int index; + + private final String value; + + public DeviceIdFilter(int index, String value) { + this.index = index; + this.value = value; + } + + public int getIndex() { + return index; + } + + public String getValue() { + return value; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitDeviceIdFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.DEVICE_ID; + } + + @Override + public void serialize(ByteBuffer byteBuffer) {} + + @Override + public void serialize(DataOutputStream stream) throws IOException {} +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java new file mode 100644 index 000000000000..1f57622719d7 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class OrFilter extends SchemaFilter { + private final SchemaFilter left; + private final SchemaFilter right; + + public OrFilter(SchemaFilter left, SchemaFilter right) { + // left and right should not be null + this.left = left; + this.right = right; + } + + public OrFilter(ByteBuffer byteBuffer) { + this.left = SchemaFilter.deserialize(byteBuffer); + this.right = SchemaFilter.deserialize(byteBuffer); + } + + public SchemaFilter getLeft() { + return left; + } + + public SchemaFilter getRight() { + return right; + } + + @Override + public boolean accept(SchemaFilterVisitor visitor, C node) { + return visitor.visitOrFilter(this, node); + } + + @Override + public SchemaFilterType getSchemaFilterType() { + return SchemaFilterType.AND; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + left.serialize(byteBuffer); + right.serialize(byteBuffer); + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + left.serialize(stream); + right.serialize(stream); + } +} diff --git a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java index 7fbc148ffdb0..760b364bbe80 100644 --- a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java @@ -101,6 +101,7 @@ import org.apache.iotdb.db.relational.sql.tree.SelectItem; import org.apache.iotdb.db.relational.sql.tree.SetProperties; import org.apache.iotdb.db.relational.sql.tree.ShowDB; +import org.apache.iotdb.db.relational.sql.tree.ShowDevice; import org.apache.iotdb.db.relational.sql.tree.ShowIndex; import org.apache.iotdb.db.relational.sql.tree.ShowTables; import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; @@ -382,7 +383,8 @@ public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementCont @Override public Node visitShowDevicesStatement(RelationalSqlParser.ShowDevicesStatementContext ctx) { - return super.visitShowDevicesStatement(ctx); + // todo parse where clause + return new ShowDevice(getLocation(ctx)); } @Override diff --git a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java new file mode 100644 index 000000000000..4427238fe9af --- /dev/null +++ b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.relational.sql.tree; + +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Objects; + +public class ShowDevice extends Statement { + + private Expression whereClause; + + public ShowDevice(@Nullable NodeLocation location) { + super(location); + } + + public void setWhereClause(Expression whereClause) { + this.whereClause = whereClause; + } + + public Expression getWhereClause() { + return whereClause; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ShowDevice)) return false; + ShowDevice that = (ShowDevice) o; + return Objects.equals(whereClause, that.whereClause); + } + + @Override + public String toString() { + return "ShowDevice{" + "whereClause=" + whereClause + '}'; + } +} From 0dfdb3008fb107b9b8997cd11e23df5e7de4ccc6 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Mon, 22 Apr 2024 09:35:19 +0800 Subject: [PATCH 12/36] add warn --- .../plan/statement/crud/InsertTableStatement.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index 28d5804c8482..a884e8d69dad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.ITableDeviceSchemaValidation; @@ -100,6 +101,10 @@ private InsertRowStatement parseInsert(Insert insert) { hasColumn ? columnNameList.get(i).getValue() : table.getColumnList().get(i).getColumnName(); + TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); + if (columnSchema == null) { + throw new SemanticException(String.format("Unknown Column %s", columnName)); + } TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); if (category.equals(TsTableColumnCategory.ID)) { idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); From 2adf50bbffae2de02fde0693c1bf2c6e7cba1777 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 28 Apr 2024 11:20:32 +0800 Subject: [PATCH 13/36] refactor expression to filter via visitor --- ...ConvertSchemaPredicateToFilterVisitor.java | 167 ++++++++++++++++++ .../schema/TableModelSchemaFetcher.java | 96 +--------- 2 files changed, 175 insertions(+), 88 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java new file mode 100644 index 000000000000..1e9f116d3e26 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.relational.sql.tree.BetweenPredicate; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; +import org.apache.iotdb.db.relational.sql.tree.Identifier; +import org.apache.iotdb.db.relational.sql.tree.IfExpression; +import org.apache.iotdb.db.relational.sql.tree.InPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNotNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.LikePredicate; +import org.apache.iotdb.db.relational.sql.tree.Literal; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.NotExpression; +import org.apache.iotdb.db.relational.sql.tree.NullIfExpression; +import org.apache.iotdb.db.relational.sql.tree.SearchedCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.StringLiteral; +import org.apache.iotdb.db.relational.sql.tree.SymbolReference; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ConvertSchemaPredicateToFilterVisitor + extends PredicateVisitor { + + @Override + protected SchemaFilter visitInPredicate(InPredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitIsNullPredicate(IsNullPredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitIsNotNullPredicate(IsNotNullPredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitLikePredicate(LikePredicate node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitLogicalExpression(LogicalExpression node, Context context) { + // the operator of the logical expression shall be OR + return new OrFilter( + node.getTerms().get(0).accept(this, context), node.getTerms().get(1).accept(this, context)); + } + + @Override + protected SchemaFilter visitNotExpression(NotExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitComparisonExpression(ComparisonExpression node, Context context) { + String columnName; + String value; + if (node.getLeft() instanceof Literal) { + value = ((StringLiteral) (node.getLeft())).getValue(); + if (node.getRight() instanceof Identifier) { + columnName = ((Identifier) (node.getRight())).getValue(); + } else { + columnName = ((SymbolReference) (node.getRight())).getName(); + } + } else { + value = ((StringLiteral) (node.getRight())).getValue(); + if (node.getLeft() instanceof Identifier) { + columnName = ((Identifier) (node.getLeft())).getValue(); + } else { + columnName = ((SymbolReference) (node.getLeft())).getName(); + } + } + if (context + .table + .getColumnSchema(columnName) + .getColumnCategory() + .equals(TsTableColumnCategory.ID)) { + return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), value); + } else { + return new DeviceAttributeFilter(columnName, value); + } + } + + @Override + protected SchemaFilter visitSimpleCaseExpression(SimpleCaseExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitSearchedCaseExpression(SearchedCaseExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitIfExpression(IfExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitNullIfExpression(NullIfExpression node, Context context) { + return visitExpression(node, context); + } + + @Override + protected SchemaFilter visitBetweenPredicate(BetweenPredicate node, Context context) { + return visitExpression(node, context); + } + + public static class Context { + + private final TsTable table; + private final Map idColumeIndexMap; + + private final List idDeterminedFilters = new ArrayList<>(); + private final List idFuzzyFilters = new ArrayList<>(); + + public Context(TsTable table) { + this.table = table; + this.idColumeIndexMap = getIdColumnIndex(table); + } + + private Map getIdColumnIndex(TsTable table) { + Map map = new HashMap<>(); + List columnSchemaList = table.getColumnList(); + int idIndex = 0; + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + map.put(columnSchema.getColumnName(), idIndex); + idIndex++; + } + } + return map; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 231708e71b89..625575819790 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -22,8 +22,6 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; -import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; @@ -38,16 +36,11 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertSchemaPredicateToFilterVisitor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; -import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; import org.apache.iotdb.db.relational.sql.tree.Expression; -import org.apache.iotdb.db.relational.sql.tree.Identifier; -import org.apache.iotdb.db.relational.sql.tree.Literal; -import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; -import org.apache.iotdb.db.relational.sql.tree.StringLiteral; -import org.apache.iotdb.db.relational.sql.tree.SymbolReference; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; @@ -199,27 +192,18 @@ private Pair, List> transformExpression( List expressionList, TsTable table) { List idDeterminedFilters = new ArrayList<>(); List idFuzzyFilters = new ArrayList<>(); - Map indexMap = getIdColumnIndex(table); + ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor(); + ConvertSchemaPredicateToFilterVisitor.Context context = + new ConvertSchemaPredicateToFilterVisitor.Context(table); for (Expression expression : expressionList) { if (expression == null) { continue; } - if (expression instanceof LogicalExpression) { - LogicalExpression logicalExpression = (LogicalExpression) expression; - SchemaFilter schemaFilter = transformToSchemaFilter(logicalExpression, table, indexMap); - if (hasAttribute(schemaFilter)) { - idFuzzyFilters.add(schemaFilter); - } else { - idDeterminedFilters.add(schemaFilter); - } + SchemaFilter schemaFilter = expression.accept(visitor, context); + if (hasAttribute(schemaFilter)) { + idFuzzyFilters.add(schemaFilter); } else { - SchemaFilter schemaFilter = - transformToSchemaFilter((ComparisonExpression) expression, table, indexMap); - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE)) { - idFuzzyFilters.add(schemaFilter); - } else { - idDeterminedFilters.add(schemaFilter); - } + idDeterminedFilters.add(schemaFilter); } } return new Pair<>(idDeterminedFilters, idFuzzyFilters); @@ -233,68 +217,4 @@ private boolean hasAttribute(SchemaFilter schemaFilter) { return schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE); } - - private SchemaFilter transformToSchemaFilter( - LogicalExpression logicalExpression, TsTable table, Map indexMap) { - SchemaFilter left; - SchemaFilter right; - if (logicalExpression.getTerms().get(0) instanceof LogicalExpression) { - left = - transformToSchemaFilter( - (LogicalExpression) (logicalExpression.getChildren().get(0)), table, indexMap); - } else { - left = - transformToSchemaFilter( - (ComparisonExpression) (logicalExpression.getChildren().get(0)), table, indexMap); - } - if (logicalExpression.getTerms().get(1) instanceof LogicalExpression) { - right = - transformToSchemaFilter( - (LogicalExpression) (logicalExpression.getChildren().get(1)), table, indexMap); - } else { - right = - transformToSchemaFilter( - (ComparisonExpression) (logicalExpression.getChildren().get(1)), table, indexMap); - } - return new OrFilter(left, right); - } - - private SchemaFilter transformToSchemaFilter( - ComparisonExpression comparisonExpression, TsTable table, Map indexMap) { - String columnName; - String value; - if (comparisonExpression.getLeft() instanceof Literal) { - value = ((StringLiteral) (comparisonExpression.getLeft())).getValue(); - if (comparisonExpression.getRight() instanceof Identifier) { - columnName = ((Identifier) (comparisonExpression.getRight())).getValue(); - } else { - columnName = ((SymbolReference) (comparisonExpression.getRight())).getName(); - } - } else { - value = ((StringLiteral) (comparisonExpression.getRight())).getValue(); - if (comparisonExpression.getLeft() instanceof Identifier) { - columnName = ((Identifier) (comparisonExpression.getLeft())).getValue(); - } else { - columnName = ((SymbolReference) (comparisonExpression.getLeft())).getName(); - } - } - if (table.getColumnSchema(columnName).getColumnCategory().equals(TsTableColumnCategory.ID)) { - return new DeviceIdFilter(indexMap.get(columnName), value); - } else { - return new DeviceAttributeFilter(columnName, value); - } - } - - private Map getIdColumnIndex(TsTable table) { - Map map = new HashMap<>(); - List columnSchemaList = table.getColumnList(); - int idIndex = 0; - for (TsTableColumnSchema columnSchema : columnSchemaList) { - if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - map.put(columnSchema.getColumnName(), idIndex); - idIndex++; - } - } - return map; - } } From 80c6b3f9f0db58ef50418ede6dbce5ba15e00292 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 28 Apr 2024 11:38:35 +0800 Subject: [PATCH 14/36] refactor schemafilter to device pattern --- .../source/TableDeviceSchemaSource.java | 74 ++++++---------- .../plan/analyze/AnalyzeVisitor.java | 58 +++---------- ...ConvertSchemaPredicateToFilterVisitor.java | 13 ++- .../schema/TableModelSchemaFetcher.java | 15 +--- .../filter/impl/DeviceFilterToPathUtil.java | 85 +++++++++++++++++++ 5 files changed, 132 insertions(+), 113 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java index 529997611421..b09e38e040bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -1,11 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.execution.operator.schema.source; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -21,15 +38,9 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Binary; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; - public class TableDeviceSchemaSource implements ISchemaSource { private String database; @@ -137,46 +148,11 @@ public void close() throws Exception { } private List getDevicePatternList() { - int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3; - String[] nodes = new String[length]; - Arrays.fill(nodes, "*"); - nodes[0] = PATH_ROOT; - nodes[1] = database; - nodes[2] = tableName; - Map> orValueMap = new HashMap<>(); - for (SchemaFilter schemaFilter : idDeterminedFilterList) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - OrFilter orFilter = (OrFilter) schemaFilter; - if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) - && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - orValueMap - .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) - .add(deviceIdFilter.getValue()); - } - } - } - - PartialPath path = new PartialPath(nodes); - List pathList = new ArrayList<>(); - pathList.add(path); - for (Map.Entry> entry : orValueMap.entrySet()) { - for (int i = 0, size = pathList.size(); i < size; i++) { - for (String value : entry.getValue()) { - nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); - nodes[entry.getKey() + 3] = value; - path = new PartialPath(nodes); - pathList.add(path); - } - } - } - - return pathList; + return DeviceFilterToPathUtil.convertToDevicePattern( + database, + tableName, + DataNodeTableCache.getInstance().getTable(database, tableName), + idDeterminedFilterList); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index bd94bf2b0fd2..b685882230cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -33,11 +33,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -186,7 +182,6 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND; import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME; @@ -3717,49 +3712,16 @@ public Analysis visitShowTableDevices( String database = statement.getDatabase(); String tableName = statement.getTableName(); - List columnSchemaList = - DataNodeTableCache.getInstance().getTable(database, tableName).getColumnList(); - - int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3 + 1; - String[] nodes = new String[length]; - Arrays.fill(nodes, "*"); - nodes[0] = PATH_ROOT; - nodes[1] = database; - nodes[2] = tableName; - nodes[nodes.length - 1] = ONE_LEVEL_PATH_WILDCARD; - Map> orValueMap = new HashMap<>(); - for (SchemaFilter schemaFilter : statement.getIdDeterminedFilterList()) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - OrFilter orFilter = (OrFilter) schemaFilter; - if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) - && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - orValueMap - .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) - .add(deviceIdFilter.getValue()); - } - } - } + + List devicePatternList = + DeviceFilterToPathUtil.convertToDevicePattern( + database, + tableName, + DataNodeTableCache.getInstance().getTable(database, tableName), + statement.getIdDeterminedFilterList()); PathPatternTree patternTree = new PathPatternTree(); - PartialPath path = new PartialPath(nodes); - patternTree.appendFullPath(path); - List pathList = new ArrayList<>(); - pathList.add(path); - for (Map.Entry> entry : orValueMap.entrySet()) { - for (int i = 0, size = pathList.size(); i < size; i++) { - for (String value : entry.getValue()) { - nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); - nodes[entry.getKey() + 3] = value; - path = new PartialPath(nodes); - pathList.add(path); - patternTree.appendFullPath(path); - } - } + for (PartialPath devicePattern : devicePatternList) { + patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); } SchemaPartition partition = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java index 1e9f116d3e26..5c8fa1e6bb22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java @@ -43,7 +43,6 @@ import org.apache.iotdb.db.relational.sql.tree.StringLiteral; import org.apache.iotdb.db.relational.sql.tree.SymbolReference; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -109,6 +108,7 @@ protected SchemaFilter visitComparisonExpression(ComparisonExpression node, Cont .equals(TsTableColumnCategory.ID)) { return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), value); } else { + context.hasAttribute = true; return new DeviceAttributeFilter(columnName, value); } } @@ -143,8 +143,7 @@ public static class Context { private final TsTable table; private final Map idColumeIndexMap; - private final List idDeterminedFilters = new ArrayList<>(); - private final List idFuzzyFilters = new ArrayList<>(); + private boolean hasAttribute = false; public Context(TsTable table) { this.table = table; @@ -163,5 +162,13 @@ private Map getIdColumnIndex(TsTable table) { } return map; } + + public boolean hasAttribute() { + return hasAttribute; + } + + public void reset() { + hasAttribute = false; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 625575819790..239212f461ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -21,8 +21,6 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -184,7 +182,6 @@ public List fetchDeviceSchema( } finally { coordinator.cleanupQueryExecution(queryId, null, t); } - System.out.println(deviceEntryList); return deviceEntryList; } @@ -199,8 +196,9 @@ private Pair, List> transformExpression( if (expression == null) { continue; } + context.reset(); SchemaFilter schemaFilter = expression.accept(visitor, context); - if (hasAttribute(schemaFilter)) { + if (context.hasAttribute()) { idFuzzyFilters.add(schemaFilter); } else { idDeterminedFilters.add(schemaFilter); @@ -208,13 +206,4 @@ private Pair, List> transformExpression( } return new Pair<>(idDeterminedFilters, idFuzzyFilters); } - - private boolean hasAttribute(SchemaFilter schemaFilter) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - return hasAttribute(((OrFilter) schemaFilter).getLeft()) - || hasAttribute(((OrFilter) schemaFilter).getRight()); - } - - return schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE); - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java new file mode 100644 index 000000000000..ff674942055e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.table.TsTable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class DeviceFilterToPathUtil { + + private DeviceFilterToPathUtil() { + // do nothing + } + + public static List convertToDevicePattern( + String database, String tableName, TsTable table, List idDeterminedFilterList) { + int length = table.getIdNums() + 3; + String[] nodes = new String[length]; + Arrays.fill(nodes, ONE_LEVEL_PATH_WILDCARD); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + Map> orValueMap = new HashMap<>(); + for (SchemaFilter schemaFilter : idDeterminedFilterList) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) + && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + orValueMap + .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) + .add(deviceIdFilter.getValue()); + } + } + } + + PartialPath path = new PartialPath(nodes); + List pathList = new ArrayList<>(); + pathList.add(path); + for (Map.Entry> entry : orValueMap.entrySet()) { + for (int i = 0, size = pathList.size(); i < size; i++) { + for (String value : entry.getValue()) { + nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); + nodes[entry.getKey() + 3] = value; + path = new PartialPath(nodes); + pathList.add(path); + } + } + } + + return pathList; + } +} From 1c407ba84a6940922391b312157ec8b64dc69f50 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 28 Apr 2024 12:02:03 +0800 Subject: [PATCH 15/36] implement ser deser --- .../common/header/ColumnHeader.java | 12 +++ .../plan/planner/plan/node/PlanNodeType.java | 4 + .../metedata/read/TableDeviceScanNode.java | 79 ++++++++++++++++++- .../utils/filter/DeviceFilterVisitor.java | 9 ++- .../commons/schema/filter/SchemaFilter.java | 9 +++ .../filter/impl/DeviceAttributeFilter.java | 17 +++- .../schema/filter/impl/DeviceIdFilter.java | 17 +++- 7 files changed, 139 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java index a2589c6bb19f..2ae8d8ca234a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java @@ -22,6 +22,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; @@ -72,6 +74,16 @@ public void serialize(ByteBuffer byteBuffer) { dataType.serializeTo(byteBuffer); } + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(columnName, stream); + ReadWriteIOUtils.write(dataType.ordinal(), stream); + ReadWriteIOUtils.write(hasAlias(), stream); + if (hasAlias()) { + ReadWriteIOUtils.write(alias, stream); + } + dataType.serializeTo(stream); + } + public static ColumnHeader deserialize(ByteBuffer byteBuffer) { String columnName = ReadWriteIOUtils.readString(byteBuffer); TSDataType dataType = TSDataType.values()[ReadWriteIOUtils.readInt(byteBuffer)]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 009e27c6a42e..24a45af766d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -213,6 +214,7 @@ public enum PlanNodeType { PIPE_OPERATE_SCHEMA_QUEUE_REFERENCE((short) 91), CREATE_TABLE_DEVICE((short) 92), + TABLE_DEVICE_SCAN((short) 93), ; public static final int BYTES = Short.BYTES; @@ -446,6 +448,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return PipeOperateSchemaQueueNode.deserialize(buffer); case 92: return CreateTableDeviceNode.deserialize(buffer); + case 93: + return TableDeviceScanNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java index 3605963c6bc6..15570dfe9fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java @@ -5,11 +5,15 @@ import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -103,10 +107,81 @@ public List getOutputColumnNames() { } @Override - protected void serializeAttributes(ByteBuffer byteBuffer) {} + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_DEVICE_SCAN.serialize(byteBuffer); + ReadWriteIOUtils.write(database, byteBuffer); + ReadWriteIOUtils.write(tableName, byteBuffer); + + ReadWriteIOUtils.write(idDeterminedFilterList.size(), byteBuffer); + for (SchemaFilter schemaFilter : idDeterminedFilterList) { + SchemaFilter.serialize(schemaFilter, byteBuffer); + } + + ReadWriteIOUtils.write(idFuzzyFilterList.size(), byteBuffer); + for (SchemaFilter schemaFilter : idFuzzyFilterList) { + SchemaFilter.serialize(schemaFilter, byteBuffer); + } + + ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(byteBuffer); + } + } @Override - protected void serializeAttributes(DataOutputStream stream) throws IOException {} + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_DEVICE_SCAN.serialize(stream); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + ReadWriteIOUtils.write(idDeterminedFilterList.size(), stream); + for (SchemaFilter schemaFilter : idDeterminedFilterList) { + SchemaFilter.serialize(schemaFilter, stream); + } + + ReadWriteIOUtils.write(idFuzzyFilterList.size(), stream); + for (SchemaFilter schemaFilter : idFuzzyFilterList) { + SchemaFilter.serialize(schemaFilter, stream); + } + + ReadWriteIOUtils.write(columnHeaderList.size(), stream); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(stream); + } + } + + public static TableDeviceScanNode deserialize(ByteBuffer buffer) { + String database = ReadWriteIOUtils.readString(buffer); + String tableName = ReadWriteIOUtils.readString(buffer); + + int size = ReadWriteIOUtils.readInt(buffer); + List idDeterminedFilterList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + idDeterminedFilterList.add(SchemaFilter.deserialize(buffer)); + } + + size = ReadWriteIOUtils.readInt(buffer); + List idFuzzyFilterList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + idFuzzyFilterList.add(SchemaFilter.deserialize(buffer)); + } + + size = ReadWriteIOUtils.readInt(buffer); + List columnHeaderList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnHeaderList.add(ColumnHeader.deserialize(buffer)); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableDeviceScanNode( + planNodeId, + database, + tableName, + idDeterminedFilterList, + idFuzzyFilterList, + columnHeaderList, + null); + } @Override public R accept(PlanVisitor visitor, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java index 0555aecf5ea4..070b2b40c176 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java @@ -64,11 +64,16 @@ public boolean visitTemplateFilter(TemplateFilter templateFilter, IDeviceSchemaI @Override public boolean visitDeviceIdFilter(DeviceIdFilter filter, IDeviceSchemaInfo info) { - return info.getPartialPath().getNodes()[filter.getIndex() + 3].equals(filter.getValue()); + String[] nodes = info.getPartialPath().getNodes(); + if (nodes.length < filter.getIndex() + 3) { + return false; + } else { + return nodes[filter.getIndex() + 3].equals(filter.getValue()); + } } @Override public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, IDeviceSchemaInfo info) { - return info.getAttributeValue(filter.getKey()).equals(filter.getValue()); + return filter.getValue().equals(info.getAttributeValue(filter.getKey())); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java index a9e5fea0b73c..ad3c4679a394 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java @@ -21,6 +21,9 @@ import org.apache.iotdb.commons.schema.filter.impl.AndFilter; import org.apache.iotdb.commons.schema.filter.impl.DataTypeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.filter.impl.PathContainsFilter; import org.apache.iotdb.commons.schema.filter.impl.TagFilter; import org.apache.iotdb.commons.schema.filter.impl.TemplateFilter; @@ -73,6 +76,12 @@ public static SchemaFilter deserialize(ByteBuffer byteBuffer) { return new AndFilter(byteBuffer); case TEMPLATE_FILTER: return new TemplateFilter(byteBuffer); + case OR: + return new OrFilter(byteBuffer); + case DEVICE_ID: + return new DeviceIdFilter(byteBuffer); + case DEVICE_ATTRIBUTE: + return new DeviceAttributeFilter(byteBuffer); default: throw new IllegalArgumentException("Unsupported schema filter type: " + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java index 09736bf5b190..d85b437e9c42 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceAttributeFilter.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilterType; import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -37,6 +39,11 @@ public DeviceAttributeFilter(String key, String value) { this.value = value; } + public DeviceAttributeFilter(ByteBuffer byteBuffer) { + this.key = ReadWriteIOUtils.readString(byteBuffer); + this.value = ReadWriteIOUtils.readString(byteBuffer); + } + public String getKey() { return key; } @@ -56,8 +63,14 @@ public SchemaFilterType getSchemaFilterType() { } @Override - public void serialize(ByteBuffer byteBuffer) {} + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(key, byteBuffer); + ReadWriteIOUtils.write(value, byteBuffer); + } @Override - public void serialize(DataOutputStream stream) throws IOException {} + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(key, stream); + ReadWriteIOUtils.write(value, stream); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java index eda64d90a8d2..149766bdb95e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilterType; import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -37,6 +39,11 @@ public DeviceIdFilter(int index, String value) { this.value = value; } + public DeviceIdFilter(ByteBuffer byteBuffer) { + this.index = ReadWriteIOUtils.readInt(byteBuffer); + this.value = ReadWriteIOUtils.readString(byteBuffer); + } + public int getIndex() { return index; } @@ -56,8 +63,14 @@ public SchemaFilterType getSchemaFilterType() { } @Override - public void serialize(ByteBuffer byteBuffer) {} + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(index, byteBuffer); + ReadWriteIOUtils.write(value, byteBuffer); + } @Override - public void serialize(DataOutputStream stream) throws IOException {} + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(index, stream); + ReadWriteIOUtils.write(value, stream); + } } From 07f3b01933335e05c61159eb6d45b2bba6900ffb Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 28 Apr 2024 12:20:37 +0800 Subject: [PATCH 16/36] rollback config --- .../java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 3177bf0c0fd0..3875bf090806 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -101,7 +101,7 @@ public class ConfigNodeConfig { * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 1; + private int defaultDataRegionGroupNumPerDatabase = 2; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; From 5a9a23ed8343539efc7bdf530ad2b86b3f329824 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 28 Apr 2024 12:22:33 +0800 Subject: [PATCH 17/36] delete show device --- .../db/relational/sql/parser/AstBuilder.java | 7 -- .../db/relational/sql/tree/ShowDevice.java | 67 ------------------- 2 files changed, 74 deletions(-) delete mode 100644 iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java diff --git a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java index 27044db61461..68c08e6fff8c 100644 --- a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/parser/AstBuilder.java @@ -100,7 +100,6 @@ import org.apache.iotdb.db.relational.sql.tree.SelectItem; import org.apache.iotdb.db.relational.sql.tree.SetProperties; import org.apache.iotdb.db.relational.sql.tree.ShowDB; -import org.apache.iotdb.db.relational.sql.tree.ShowDevice; import org.apache.iotdb.db.relational.sql.tree.ShowIndex; import org.apache.iotdb.db.relational.sql.tree.ShowTables; import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; @@ -380,12 +379,6 @@ public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementCont return super.visitLoadTsFileStatement(ctx); } - @Override - public Node visitShowDevicesStatement(RelationalSqlParser.ShowDevicesStatementContext ctx) { - // todo parse where clause - return new ShowDevice(getLocation(ctx)); - } - @Override public Node visitCountDevicesStatement(RelationalSqlParser.CountDevicesStatementContext ctx) { return super.visitCountDevicesStatement(ctx); diff --git a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java b/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java deleted file mode 100644 index 4427238fe9af..000000000000 --- a/iotdb-core/relational-parser/src/main/java/org/apache/iotdb/db/relational/sql/tree/ShowDevice.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.relational.sql.tree; - -import com.google.common.collect.ImmutableList; - -import javax.annotation.Nullable; - -import java.util.List; -import java.util.Objects; - -public class ShowDevice extends Statement { - - private Expression whereClause; - - public ShowDevice(@Nullable NodeLocation location) { - super(location); - } - - public void setWhereClause(Expression whereClause) { - this.whereClause = whereClause; - } - - public Expression getWhereClause() { - return whereClause; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ShowDevice)) return false; - ShowDevice that = (ShowDevice) o; - return Objects.equals(whereClause, that.whereClause); - } - - @Override - public String toString() { - return "ShowDevice{" + "whereClause=" + whereClause + '}'; - } -} From 8557956cdf058a2c438294768b9b2210e5eac659 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 5 May 2024 15:09:39 +0800 Subject: [PATCH 18/36] set up TableDeviceSchemaCache --- .../schema/cache/CacheMemoryControlUtil.java | 9 + .../schema/cache/TableDeviceCacheEntry.java | 58 ++++++ .../analyzer/schema/cache/TableDeviceId.java | 58 ++++++ .../schema/cache/TableDeviceSchemaCache.java | 165 ++++++++++++++++++ .../analyzer/schema/cache/TableId.java | 62 +++++++ 5 files changed, 352 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java new file mode 100644 index 000000000000..b37f57a9574a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/CacheMemoryControlUtil.java @@ -0,0 +1,9 @@ +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +public class CacheMemoryControlUtil { + + public static int estimateStringSize(String string) { + // each char takes 2B in Java + return string == null ? 0 : 32 + 2 * string.length(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java new file mode 100644 index 000000000000..18c9355cc368 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableDeviceCacheEntry { + + private final Map attributeMap; + + public TableDeviceCacheEntry() { + attributeMap = new ConcurrentHashMap<>(); + } + + public TableDeviceCacheEntry(Map attributeMap) { + this.attributeMap = new ConcurrentHashMap<>(attributeMap); + } + + public void putAttribute(String key, String value) { + attributeMap.put(key, value); + } + + public String getAttribute(String key) { + return attributeMap.get(key); + } + + public Map getAttributeMap() { + return attributeMap; + } + + public int estimateSize() { + int size = 8; + for (Map.Entry entry : attributeMap.entrySet()) { + size += estimateStringSize(entry.getKey()) + estimateStringSize(entry.getValue()); + } + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java new file mode 100644 index 000000000000..b6b2cf88db32 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.Arrays; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableDeviceId { + + private final String[] idValues; + + public TableDeviceId(String[] idValues) { + this.idValues = idValues; + } + + public String getIdValue(int index) { + return idValues[index]; + } + + public int estimateSize() { + int size = 8 + 8 + 8 + 4; // object header + reference + String[] header + String.length + for (String node : idValues) { + size += estimateStringSize(node); + } + return size; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceId)) return false; + TableDeviceId that = (TableDeviceId) o; + return Arrays.equals(idValues, that.idValues); + } + + @Override + public int hashCode() { + return Arrays.hashCode(idValues); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java new file mode 100644 index 000000000000..fa6554b83ed6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableDeviceSchemaCache { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final IDualKeyCache dualKeyCache; + + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); + + public TableDeviceSchemaCache() { + DualKeyCacheBuilder dualKeyCacheBuilder = + new DualKeyCacheBuilder<>(); + dualKeyCache = + dualKeyCacheBuilder + .cacheEvictionPolicy( + DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) + .memoryCapacity(config.getAllocateMemoryForSchemaCache()) + .firstKeySizeComputer(TableId::estimateSize) + .secondKeySizeComputer(TableDeviceId::estimateSize) + .valueSizeComputer(TableDeviceCacheEntry::estimateSize) + .build(); + } + + public Map getDeviceAttribute( + String database, String tableName, String[] deviceId) { + readWriteLock.readLock().lock(); + try { + return dualKeyCache + .get(new TableId(database, tableName), new TableDeviceId(deviceId)) + .getAttributeMap(); + } finally { + readWriteLock.readLock().unlock(); + } + } + + public void put( + String database, String tableName, String[] deviceId, Map attributeMap) { + readWriteLock.readLock().lock(); + try { + dualKeyCache.put( + new TableId(database, tableName), + new TableDeviceId(deviceId), + new TableDeviceCacheEntry(attributeMap)); + } finally { + readWriteLock.readLock().unlock(); + } + } + + public void addAttribute( + String database, String tableName, String[] deviceId, Map attributeMap) { + readWriteLock.readLock().lock(); + try { + dualKeyCache.update( + new IDualKeyCacheUpdating() { + @Override + public TableId getFirstKey() { + return new TableId(database, tableName); + } + + @Override + public TableDeviceId[] getSecondKeyList() { + return new TableDeviceId[] {new TableDeviceId(deviceId)}; + } + + @Override + public int updateValue(int index, TableDeviceCacheEntry value) { + if (index > 0) { + return 0; + } + Map existingMap = value.getAttributeMap(); + AtomicInteger updatedSize = new AtomicInteger(0); + for (Map.Entry entry : attributeMap.entrySet()) { + existingMap.compute( + entry.getKey(), + (k, v) -> { + if (v == null) { + updatedSize.getAndAdd( + estimateStringSize(entry.getKey()) + + estimateStringSize(entry.getValue())); + } else { + updatedSize.getAndAdd( + estimateStringSize(entry.getValue()) - estimateStringSize(v)); + } + return entry.getValue(); + }); + } + return updatedSize.get(); + } + }); + } finally { + readWriteLock.readLock().unlock(); + } + } + + public void removeAttribute( + String database, String tableName, String[] deviceId, List attributeNameList) { + readWriteLock.readLock().lock(); + try { + dualKeyCache.update( + new IDualKeyCacheUpdating() { + @Override + public TableId getFirstKey() { + return new TableId(database, tableName); + } + + @Override + public TableDeviceId[] getSecondKeyList() { + return new TableDeviceId[] {new TableDeviceId(deviceId)}; + } + + @Override + public int updateValue(int index, TableDeviceCacheEntry value) { + if (index > 0) { + return 0; + } + Map existingMap = value.getAttributeMap(); + int updatedSize = 0; + for (String key : attributeNameList) { + String existingValue = existingMap.remove(key); + if (existingValue != null) { + updatedSize -= (estimateStringSize(key) + estimateStringSize(existingValue)); + } + } + return updatedSize; + } + }); + } finally { + readWriteLock.readLock().unlock(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java new file mode 100644 index 000000000000..f858a591bfb5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableId.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; + +import java.util.Objects; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; + +public class TableId { + + private final String database; + + private final String tableName; + + public TableId(String database, String tableName) { + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public int estimateSize() { + return 8 + 8 + 8 + estimateStringSize(database) + estimateStringSize(tableName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableId)) return false; + TableId tableId = (TableId) o; + return Objects.equals(database, tableId.database) + && Objects.equals(tableName, tableId.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName); + } +} From c1dac7f259ed8730124fda359687c7eee95d1a0f Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 5 May 2024 16:58:52 +0800 Subject: [PATCH 19/36] set up validate schema --- .../confignode/conf/ConfigNodeConfig.java | 2 +- .../plan/analyze/AnalyzeVisitor.java | 4 +- .../plan/planner/LogicalPlanVisitor.java | 4 +- .../schema/TableModelSchemaFetcher.java | 211 +++++++++++++++++- .../schema/TableModelSchemaValidator.java | 3 + .../schema/cache/TableDeviceSchemaCache.java | 88 -------- .../plan/statement/StatementVisitor.java | 4 +- .../CreateTableDeviceStatement.java | 2 +- .../table/FetchTableDevicesStatement.java | 40 ++++ .../ShowTableDevicesStatement.java | 3 +- .../filter/impl/DeviceFilterToPathUtil.java | 1 + 11 files changed, 260 insertions(+), 102 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/{internal => table}/CreateTableDeviceStatement.java (97%) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/{metadata => table}/ShowTableDevicesStatement.java (93%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 3875bf090806..3177bf0c0fd0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -101,7 +101,7 @@ public class ConfigNodeConfig { * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 2; + private int defaultDataRegionGroupNumPerDatabase = 1; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index b685882230cf..736d7feaf255 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -107,7 +107,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -128,7 +127,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement; @@ -145,6 +143,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 04e119ccc8a0..1d4d0dda0402 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -72,7 +72,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -88,7 +87,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowChildNodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowChildPathsStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDevicesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement; @@ -98,6 +96,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 239212f461ce..6bd38253c88b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -35,9 +35,12 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertSchemaPredicateToFilterVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceId; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; -import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.relational.sql.tree.Expression; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; @@ -61,6 +64,10 @@ public class TableModelSchemaFetcher { private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private final Coordinator coordinator = Coordinator.getInstance(); + + private final TableDeviceSchemaCache cache = new TableDeviceSchemaCache(); + private static class TableModelSchemaFetcherHolder { private static final TableModelSchemaFetcher INSTANCE = new TableModelSchemaFetcher(); } @@ -75,13 +82,202 @@ private TableModelSchemaFetcher() { public void validateDeviceSchema( ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) { + ValidateResult cacheValidateResult = validateDeviceSchemaInCache(schemaValidation); + ValidateResult fetchValidateResult = + fetchAndValidateDeviceSchema(schemaValidation, cacheValidateResult, context); + autoCreateDeviceSchema(schemaValidation, fetchValidateResult, context); + } + + private ValidateResult validateDeviceSchemaInCache( + ITableDeviceSchemaValidation schemaValidation) { + ValidateResult result = new ValidateResult(); + String database = schemaValidation.getDatabase(); + String tableName = schemaValidation.getTableName(); + List deviceIdList = schemaValidation.getDeviceIdList(); + List attributeKeyList = schemaValidation.getAttributeColumnNameList(); + List> attributeValueList = schemaValidation.getAttributeValue(); + + for (int i = 0, size = deviceIdList.size(); i < size; i++) { + Map attributeMap = + cache.getDeviceAttribute(database, tableName, deviceIdList.get(i)); + if (attributeMap == null) { + result.missingDeviceIndexList.add(i); + continue; + } + List deviceAttributeValueList = attributeValueList.get(i); + for (int j = 0; j < attributeKeyList.size(); j++) { + String value = attributeMap.get(attributeKeyList.get(j)); + if (value == null) { + result.attributeMissingInCacheDeviceIndexList.add(i); + break; + } else if (!value.equals(deviceAttributeValueList.get(j))) { + result.attributeUpdateDeviceIndexList.add(i); + break; + } + } + } + return result; + } + + private ValidateResult fetchAndValidateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, + ValidateResult previousValidateResult, + MPPQueryContext context) { + List targetDeviceList = + new ArrayList<>( + previousValidateResult.missingDeviceIndexList.size() + + previousValidateResult.attributeMissingInCacheDeviceIndexList.size()); + for (int index : previousValidateResult.missingDeviceIndexList) { + targetDeviceList.add(schemaValidation.getDeviceIdList().get(index)); + } + for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) { + targetDeviceList.add(schemaValidation.getDeviceIdList().get(index)); + } + + Map> fetchedDeviceSchema = + fetchMissingDeviceSchema( + new FetchTableDevicesStatement( + schemaValidation.getDatabase(), schemaValidation.getTableName(), targetDeviceList), + context); + + ValidateResult result = new ValidateResult(); + for (int index : previousValidateResult.missingDeviceIndexList) { + String[] deviceId = schemaValidation.getDeviceIdList().get(index); + Map attributeMap = fetchedDeviceSchema.get(new TableDeviceId(deviceId)); + if (attributeMap == null) { + result.missingDeviceIndexList.add(index); + } else { + for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { + String key = schemaValidation.getAttributeColumnNameList().get(j); + String value = attributeMap.get(key); + if (value == null + || !value.equals(schemaValidation.getAttributeValue().get(index).get(j))) { + result.attributeUpdateDeviceIndexList.add(index); + break; + } + } + } + } + + for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) { + String[] deviceId = schemaValidation.getDeviceIdList().get(index); + Map attributeMap = fetchedDeviceSchema.get(new TableDeviceId(deviceId)); + if (attributeMap == null) { + throw new IllegalStateException("Device shall exist but not exist."); + } else { + for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { + String key = schemaValidation.getAttributeColumnNameList().get(j); + String value = attributeMap.get(key); + if (value == null + || !value.equals(schemaValidation.getAttributeValue().get(index).get(j))) { + result.attributeUpdateDeviceIndexList.add(index); + break; + } + } + } + } + + result.attributeUpdateDeviceIndexList.addAll( + previousValidateResult.attributeUpdateDeviceIndexList); + + return result; + } + + private Map> fetchMissingDeviceSchema( + FetchTableDevicesStatement statement, MPPQueryContext context) { + long queryId = SessionManager.getInstance().requestQueryId(); + Throwable t = null; + + String database = statement.getDatabase(); + String table = statement.getTableName(); + TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + + ExecutionResult executionResult = + Coordinator.getInstance() + .executeForTreeModel( + null, + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + config.getQueryTimeoutThreshold()); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + + List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + Map> fetchedDeviceSchema = new HashMap<>(); + + try { + while (coordinator.getQueryExecution(queryId).hasNextResult()) { + Optional tsBlock; + try { + tsBlock = coordinator.getQueryExecution(queryId).getBatchResult(); + } catch (IoTDBException e) { + t = e; + throw new RuntimeException("Fetch Table Device Schema failed. ", e); + } + if (!tsBlock.isPresent() || tsBlock.get().isEmpty()) { + break; + } + Column[] columns = tsBlock.get().getValueColumns(); + for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { + String[] nodes = new String[idLength]; + int idIndex = 0; + Map attributeMap = new HashMap<>(); + for (int j = 0; j < columnHeaderList.size(); j++) { + TsTableColumnSchema columnSchema = + tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + nodes[idIndex] = columns[j].getBinary(i).toString(); + idIndex++; + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } + } + fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap); + } + } + } catch (Throwable throwable) { + t = throwable; + throw throwable; + } finally { + coordinator.cleanupQueryExecution(queryId, null, t); + } + return fetchedDeviceSchema; + } + + private void autoCreateDeviceSchema( + ITableDeviceSchemaValidation schemaValidation, + ValidateResult previousValidateResult, + MPPQueryContext context) { + List deviceIdList = + new ArrayList<>( + previousValidateResult.missingDeviceIndexList.size() + + previousValidateResult.attributeUpdateDeviceIndexList.size()); + List> attributeValueList = new ArrayList<>(deviceIdList.size()); + for (int index : previousValidateResult.missingDeviceIndexList) { + deviceIdList.add(schemaValidation.getDeviceIdList().get(index)); + attributeValueList.add(schemaValidation.getAttributeValue().get(index)); + } + for (int index : previousValidateResult.attributeUpdateDeviceIndexList) { + deviceIdList.add(schemaValidation.getDeviceIdList().get(index)); + attributeValueList.add(schemaValidation.getAttributeValue().get(index)); + } + CreateTableDeviceStatement statement = new CreateTableDeviceStatement( schemaValidation.getDatabase(), schemaValidation.getTableName(), - schemaValidation.getDeviceIdList(), + deviceIdList, schemaValidation.getAttributeColumnNameList(), - schemaValidation.getAttributeValue()); + attributeValueList); ExecutionResult executionResult = Coordinator.getInstance() .executeForTreeModel( @@ -101,6 +297,12 @@ public void validateDeviceSchema( } } + private static class ValidateResult { + final List missingDeviceIndexList = new ArrayList<>(); + final List attributeMissingInCacheDeviceIndexList = new ArrayList<>(); + final List attributeUpdateDeviceIndexList = new ArrayList<>(); + } + public List fetchDeviceSchema( String database, String table, @@ -108,7 +310,6 @@ public List fetchDeviceSchema( List attributeColumns) { List deviceEntryList = new ArrayList<>(); - Coordinator coordinator = Coordinator.getInstance(); long queryId = SessionManager.getInstance().requestQueryId(); Throwable t = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java new file mode 100644 index 000000000000..cdbee6b2a7cc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaValidator.java @@ -0,0 +1,3 @@ +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +public class TableModelSchemaValidator {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java index fa6554b83ed6..02a57e7b2d8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -22,17 +22,12 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.CacheMemoryControlUtil.estimateStringSize; - public class TableDeviceSchemaCache { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -79,87 +74,4 @@ public void put( readWriteLock.readLock().unlock(); } } - - public void addAttribute( - String database, String tableName, String[] deviceId, Map attributeMap) { - readWriteLock.readLock().lock(); - try { - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public TableId getFirstKey() { - return new TableId(database, tableName); - } - - @Override - public TableDeviceId[] getSecondKeyList() { - return new TableDeviceId[] {new TableDeviceId(deviceId)}; - } - - @Override - public int updateValue(int index, TableDeviceCacheEntry value) { - if (index > 0) { - return 0; - } - Map existingMap = value.getAttributeMap(); - AtomicInteger updatedSize = new AtomicInteger(0); - for (Map.Entry entry : attributeMap.entrySet()) { - existingMap.compute( - entry.getKey(), - (k, v) -> { - if (v == null) { - updatedSize.getAndAdd( - estimateStringSize(entry.getKey()) - + estimateStringSize(entry.getValue())); - } else { - updatedSize.getAndAdd( - estimateStringSize(entry.getValue()) - estimateStringSize(v)); - } - return entry.getValue(); - }); - } - return updatedSize.get(); - } - }); - } finally { - readWriteLock.readLock().unlock(); - } - } - - public void removeAttribute( - String database, String tableName, String[] deviceId, List attributeNameList) { - readWriteLock.readLock().lock(); - try { - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public TableId getFirstKey() { - return new TableId(database, tableName); - } - - @Override - public TableDeviceId[] getSecondKeyList() { - return new TableDeviceId[] {new TableDeviceId(deviceId)}; - } - - @Override - public int updateValue(int index, TableDeviceCacheEntry value) { - if (index > 0) { - return 0; - } - Map existingMap = value.getAttributeMap(); - int updatedSize = 0; - for (String key : attributeNameList) { - String existingValue = existingMap.remove(key); - if (existingValue != null) { - updatedSize -= (estimateStringSize(key) + estimateStringSize(existingValue)); - } - } - return updatedSize; - } - }); - } finally { - readWriteLock.readLock().unlock(); - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index be7f36de3fb0..fc3f5f915abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -29,7 +29,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; @@ -71,7 +70,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowFunctionsStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; @@ -124,6 +122,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.SetThrottleQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; /** * This class provides a visitor of {@link StatementNode}, which can be extended to create a visitor diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java similarity index 97% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java index 11b0b0a0a0dd..d16b016f3400 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/CreateTableDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/CreateTableDeviceStatement.java @@ -1,4 +1,4 @@ -package org.apache.iotdb.db.queryengine.plan.statement.internal; +package org.apache.iotdb.db.queryengine.plan.statement.table; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.plan.statement.Statement; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java new file mode 100644 index 000000000000..24acf8b32cb3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java @@ -0,0 +1,40 @@ +package org.apache.iotdb.db.queryengine.plan.statement.table; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; + +import java.util.Collections; +import java.util.List; + +public class FetchTableDevicesStatement extends Statement { + + private final String database; + + private final String tableName; + + private final List deviceIdList; + + public FetchTableDevicesStatement( + String database, String tableName, List deviceIdList) { + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java similarity index 93% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java index 0cb634830cd7..fec8eec1b27a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTableDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java @@ -17,10 +17,11 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.statement.metadata; +package org.apache.iotdb.db.queryengine.plan.statement.table; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; import java.util.List; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java index ff674942055e..6b937ce33a43 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java @@ -56,6 +56,7 @@ public static List convertToDevicePattern( OrFilter orFilter = (OrFilter) schemaFilter; if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + // todo nested orFilter DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); From 9c47034465a8cbfb1e86a98f733aa412c92f86b3 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 5 May 2024 17:54:38 +0800 Subject: [PATCH 20/36] implement schema fetch and auto create --- .../schema/source/SchemaSourceFactory.java | 8 + .../schema/source/TableDeviceFetchSource.java | 116 ++++++++++ .../plan/analyze/AnalyzeVisitor.java | 28 +++ .../plan/planner/LogicalPlanBuilder.java | 19 +- .../plan/planner/LogicalPlanVisitor.java | 39 +++- .../plan/planner/OperatorTreeGenerator.java | 23 +- .../distribution/ExchangeNodeAdder.java | 8 +- .../plan/planner/plan/node/PlanNodeType.java | 7 +- .../plan/planner/plan/node/PlanVisitor.java | 7 +- .../read/table/TableDeviceFetchNode.java | 213 ++++++++++++++++++ .../read/{ => table}/TableDeviceScanNode.java | 22 +- .../schema/TableModelSchemaFetcher.java | 25 +- .../analyzer/schema/cache/TableDeviceId.java | 4 + .../schema/cache/TableDeviceSchemaCache.java | 6 +- .../plan/statement/StatementVisitor.java | 5 + .../table/FetchTableDevicesStatement.java | 25 ++ .../schemaregion/ISchemaRegion.java | 3 + .../impl/SchemaRegionMemoryImpl.java | 86 +++++++ .../impl/SchemaRegionPBTreeImpl.java | 7 + .../filter/impl/DeviceFilterToPathUtil.java | 15 ++ 20 files changed, 645 insertions(+), 21 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/{ => table}/TableDeviceScanNode.java (87%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 9e3028b62f00..49ba2ba58c7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -109,4 +109,12 @@ public static ISchemaSource getTableDeviceSchemaSource( return new TableDeviceSchemaSource( database, tableName, idDeterminedFilterList, idFuzzyFilterList, columnHeaderList); } + + public static ISchemaSource getTableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + return new TableDeviceFetchSource(database, tableName, deviceIdList, columnHeaderList); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java new file mode 100644 index 000000000000..f91397d5e239 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.List; + +public class TableDeviceFetchSource implements ISchemaSource { + + private String database; + + private String tableName; + + private List deviceIdList; + + private List columnHeaderList; + + public TableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + this.columnHeaderList = columnHeaderList; + } + + @Override + public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { + try { + return schemaRegion.getDeviceReader( + DeviceFilterToPathUtil.convertToDevicePath(database, tableName, deviceIdList)); + } catch (MetadataException e) { + throw new SchemaExecutionException(e); + } + } + + @Override + public List getInfoQueryColumnHeaders() { + return columnHeaderList; + } + + @Override + public void transformToTsBlockColumns( + IDeviceSchemaInfo schemaInfo, TsBlockBuilder builder, String database) { + builder.getTimeColumnBuilder().writeLong(0L); + int resultIndex = 0; + int idIndex = 0; + PartialPath devicePath = schemaInfo.getPartialPath(); + TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); + TsTableColumnSchema columnSchema; + for (ColumnHeader columnHeader : columnHeaderList) { + columnSchema = table.getColumnSchema(columnHeader.getColumnName()); + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary(devicePath.getNodes()[idIndex + 3], TSFileConfig.STRING_CHARSET)); + idIndex++; + } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } + resultIndex++; + } + builder.declarePosition(); + } + + @Override + public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { + return false; + } + + @Override + public long getSchemaStatistic(ISchemaRegion schemaRegion) { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 736d7feaf255..f29b73576503 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -144,6 +144,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; @@ -3732,4 +3733,31 @@ public Analysis visitShowTableDevices( return analysis; } + + @Override + public Analysis visitFetchTableDevices( + FetchTableDevicesStatement statement, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + Analysis analysis = new Analysis(); + analysis.setStatement(statement); + + String database = statement.getDatabase(); + String tableName = statement.getTableName(); + + List devicePatternList = + DeviceFilterToPathUtil.convertToDevicePath( + database, tableName, statement.getDeviceIdList()); + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath devicePattern : devicePatternList) { + patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); + } + + SchemaPartition partition = + partitionFetcher.getOrCreateSchemaPartition( + patternTree, context.getSession().getUserName()); + + analysis.setSchemaPartitionInfo(partition); + + return analysis; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 10b92fe59850..67b952fe1c7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -55,9 +55,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.DeviceViewIntoNode; @@ -1664,4 +1665,20 @@ public LogicalPlanBuilder planTableDeviceSource( null); return this; } + + public LogicalPlanBuilder planTableDeviceFetchSource( + String database, + String tableName, + List deviceIdList, + List columnHeaderList) { + this.root = + new TableDeviceFetchNode( + context.getQueryId().genPlanNodeId(), + database, + tableName, + deviceIdList, + columnHeaderList, + null); + return this; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 1d4d0dda0402..75b9aca64119 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -97,6 +97,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.Template; @@ -1036,10 +1037,27 @@ public PlanNode visitShowTableDevices( ShowTableDevicesStatement statement, MPPQueryContext context) { LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + List columnHeaderList = + getColumnHeaderList(statement.getDatabase(), statement.getTableName()); + + analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true)); + + planBuilder = + planBuilder + .planTableDeviceSource( + statement.getDatabase(), + statement.getTableName(), + statement.getIdDeterminedFilterList(), + statement.getIdFuzzyFilterList(), + columnHeaderList) + .planSchemaQueryMerge(false); + + return planBuilder.getRoot(); + } + + private List getColumnHeaderList(String database, String tableName) { List columnSchemaList = - DataNodeTableCache.getInstance() - .getTable(statement.getDatabase(), statement.getTableName()) - .getColumnList(); + DataNodeTableCache.getInstance().getTable(database, tableName).getColumnList(); List columnHeaderList = new ArrayList<>(columnSchemaList.size()); for (TsTableColumnSchema columnSchema : columnSchemaList) { @@ -1049,16 +1067,25 @@ public PlanNode visitShowTableDevices( new ColumnHeader(columnSchema.getColumnName(), columnSchema.getDataType())); } } + return columnHeaderList; + } + + @Override + public PlanNode visitFetchTableDevices( + FetchTableDevicesStatement statement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + + List columnHeaderList = + getColumnHeaderList(statement.getDatabase(), statement.getTableName()); analysis.setRespDatasetHeader(new DatasetHeader(columnHeaderList, true)); planBuilder = planBuilder - .planTableDeviceSource( + .planTableDeviceFetchSource( statement.getDatabase(), statement.getTableName(), - statement.getIdDeterminedFilterList(), - statement.getIdFuzzyFilterList(), + statement.getDeviceIdList(), columnHeaderList) .planSchemaQueryMerge(false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index d6d1874ceed2..3ceb7c5d8f27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -173,9 +173,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; @@ -3288,4 +3289,24 @@ public Operator visitTableDeviceScan( node.getIdFuzzyFilterList(), node.getColumnHeaderList())); } + + @Override + public Operator visitTableDeviceFetch( + TableDeviceFetchNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaQueryScanOperator.class.getSimpleName()); + return new SchemaQueryScanOperator<>( + node.getPlanNodeId(), + operatorContext, + SchemaSourceFactory.getTableDeviceFetchSource( + node.getDatabase(), + node.getTableName(), + node.getDeviceIdList(), + node.getColumnHeaderList())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index 0c19a738a5d8..3c7f24a19da3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -33,7 +33,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.DeviceMergeNode; @@ -160,6 +161,11 @@ public PlanNode visitTableDeviceScan(TableDeviceScanNode node, NodeGroupContext return processNoChildSourceNode(node, context); } + @Override + public PlanNode visitTableDeviceFetch(TableDeviceFetchNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + @Override public PlanNode visitSchemaFetchScan(SchemaFetchScanNode node, NodeGroupContext context) { return processNoChildSourceNode(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 24a45af766d5..8e6ef9e93832 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -34,9 +34,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; @@ -215,7 +216,7 @@ public enum PlanNodeType { CREATE_TABLE_DEVICE((short) 92), TABLE_DEVICE_SCAN((short) 93), - ; + TABLE_DEVICE_FETCH((short) 94); public static final int BYTES = Short.BYTES; @@ -450,6 +451,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return CreateTableDeviceNode.deserialize(buffer); case 93: return TableDeviceScanNode.deserialize(buffer); + case 94: + return TableDeviceFetchNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index a417be3f9b70..0aa1c0a6c3d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -31,9 +31,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table.TableDeviceScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.BatchActivateTemplateNode; @@ -563,4 +564,8 @@ public R visitCreateTableDevice(CreateTableDeviceNode node, C context) { public R visitTableDeviceScan(TableDeviceScanNode node, C context) { return visitPlan(node, context); } + + public R visitTableDeviceFetch(TableDeviceFetchNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java new file mode 100644 index 000000000000..1a517c95ebae --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceFetchNode.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class TableDeviceFetchNode extends SchemaQueryScanNode { + + private String database; + + private String tableName; + + private List deviceIdList; + + private List columnHeaderList; + + private TRegionReplicaSet schemaRegionReplicaSet; + + public TableDeviceFetchNode(PlanNodeId id) { + super(id); + } + + public TableDeviceFetchNode( + PlanNodeId id, + String database, + String tableName, + List deviceIdList, + List columnHeaderList, + TRegionReplicaSet regionReplicaSet) { + super(id); + this.database = database; + this.tableName = tableName; + this.deviceIdList = deviceIdList; + this.columnHeaderList = columnHeaderList; + this.schemaRegionReplicaSet = regionReplicaSet; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getDeviceIdList() { + return deviceIdList; + } + + public List getColumnHeaderList() { + return columnHeaderList; + } + + @Override + public void open() throws Exception {} + + @Override + public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { + this.schemaRegionReplicaSet = regionReplicaSet; + } + + @Override + public void close() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public PlanNode clone() { + return new TableDeviceFetchNode( + getPlanNodeId(), + database, + tableName, + deviceIdList, + columnHeaderList, + schemaRegionReplicaSet); + } + + @Override + public List getOutputColumnNames() { + return columnHeaderList.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_DEVICE_FETCH.serialize(byteBuffer); + ReadWriteIOUtils.write(database, byteBuffer); + ReadWriteIOUtils.write(tableName, byteBuffer); + + ReadWriteIOUtils.write(deviceIdList.size(), byteBuffer); + for (String[] deviceId : deviceIdList) { + ReadWriteIOUtils.write(deviceId.length, byteBuffer); + for (String idValue : deviceId) { + ReadWriteIOUtils.write(idValue, byteBuffer); + } + } + + ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(byteBuffer); + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_DEVICE_FETCH.serialize(stream); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + ReadWriteIOUtils.write(deviceIdList.size(), stream); + for (String[] deviceId : deviceIdList) { + ReadWriteIOUtils.write(deviceId.length, stream); + for (String idValue : deviceId) { + ReadWriteIOUtils.write(idValue, stream); + } + } + + ReadWriteIOUtils.write(columnHeaderList.size(), stream); + for (ColumnHeader columnHeader : columnHeaderList) { + columnHeader.serialize(stream); + } + } + + public static TableDeviceFetchNode deserialize(ByteBuffer buffer) { + String database = ReadWriteIOUtils.readString(buffer); + String tableName = ReadWriteIOUtils.readString(buffer); + + int size = ReadWriteIOUtils.readInt(buffer); + List deviceIdList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + int length = ReadWriteIOUtils.readInt(buffer); + String[] nodes = new String[length]; + for (int j = 0; j < length; j++) { + nodes[j] = ReadWriteIOUtils.readString(buffer); + } + deviceIdList.add(nodes); + } + + size = ReadWriteIOUtils.readInt(buffer); + List columnHeaderList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnHeaderList.add(ColumnHeader.deserialize(buffer)); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableDeviceFetchNode( + planNodeId, database, tableName, deviceIdList, columnHeaderList, null); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitTableDeviceFetch(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TableDeviceFetchNode)) return false; + if (!super.equals(o)) return false; + TableDeviceFetchNode that = (TableDeviceFetchNode) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(deviceIdList, that.deviceIdList) + && Objects.equals(columnHeaderList, that.columnHeaderList) + && Objects.equals(schemaRegionReplicaSet, that.schemaRegionReplicaSet); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + database, + tableName, + deviceIdList, + columnHeaderList, + schemaRegionReplicaSet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java similarity index 87% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java index 15570dfe9fd2..5439b089f5a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java @@ -1,4 +1,23 @@ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; @@ -7,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 6bd38253c88b..59c3b79c0ac3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -82,10 +82,17 @@ private TableModelSchemaFetcher() { public void validateDeviceSchema( ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) { - ValidateResult cacheValidateResult = validateDeviceSchemaInCache(schemaValidation); - ValidateResult fetchValidateResult = - fetchAndValidateDeviceSchema(schemaValidation, cacheValidateResult, context); - autoCreateDeviceSchema(schemaValidation, fetchValidateResult, context); + ValidateResult validateResult = validateDeviceSchemaInCache(schemaValidation); + + if (!validateResult.missingDeviceIndexList.isEmpty() + || !validateResult.attributeMissingInCacheDeviceIndexList.isEmpty()) { + validateResult = fetchAndValidateDeviceSchema(schemaValidation, validateResult, context); + } + + if (!validateResult.missingDeviceIndexList.isEmpty() + || !validateResult.attributeUpdateDeviceIndexList.isEmpty()) { + autoCreateDeviceSchema(schemaValidation, validateResult, context); + } } private ValidateResult validateDeviceSchemaInCache( @@ -140,6 +147,14 @@ private ValidateResult fetchAndValidateDeviceSchema( schemaValidation.getDatabase(), schemaValidation.getTableName(), targetDeviceList), context); + for (Map.Entry> entry : fetchedDeviceSchema.entrySet()) { + cache.put( + schemaValidation.getDatabase(), + schemaValidation.getTableName(), + entry.getKey().getIdValues(), + entry.getValue()); + } + ValidateResult result = new ValidateResult(); for (int index : previousValidateResult.missingDeviceIndexList) { String[] deviceId = schemaValidation.getDeviceIdList().get(index); @@ -195,7 +210,7 @@ private Map> fetchMissingDeviceSchema( ExecutionResult executionResult = Coordinator.getInstance() .executeForTreeModel( - null, + statement, queryId, SessionManager.getInstance() .getSessionInfo(SessionManager.getInstance().getCurrSession()), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java index b6b2cf88db32..a40bdf54821d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceId.java @@ -35,6 +35,10 @@ public String getIdValue(int index) { return idValues[index]; } + public String[] getIdValues() { + return idValues; + } + public int estimateSize() { int size = 8 + 8 + 8 + 4; // object header + reference + String[] header + String.length for (String node : idValues) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java index 02a57e7b2d8a..c73125b4a281 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -54,9 +54,9 @@ public Map getDeviceAttribute( String database, String tableName, String[] deviceId) { readWriteLock.readLock().lock(); try { - return dualKeyCache - .get(new TableId(database, tableName), new TableDeviceId(deviceId)) - .getAttributeMap(); + TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(database, tableName), new TableDeviceId(deviceId)); + return entry == null ? null : entry.getAttributeMap(); } finally { readWriteLock.readLock().unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index fc3f5f915abf..1473eda7b7f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -123,6 +123,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; /** @@ -616,4 +617,8 @@ public R visitCreateTableDevice( public R visitShowTableDevices(ShowTableDevicesStatement statement, C context) { return visitStatement(statement, context); } + + public R visitFetchTableDevices(FetchTableDevicesStatement statement, C context) { + return visitStatement(statement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java index 24acf8b32cb3..e1ea9582ffa4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/FetchTableDevicesStatement.java @@ -1,7 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.statement.table; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import java.util.Collections; import java.util.List; @@ -37,4 +57,9 @@ public List getDeviceIdList() { public List getPaths() { return Collections.emptyList(); } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitFetchTableDevices(this, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index a8f814d4a9ca..a3d8f0dfd3f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -317,5 +317,8 @@ ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan) ISchemaReader getTableDeviceReader(ShowTableDevicesPlan showTableDevicesPlan) throws MetadataException; + + ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException; // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 3aae464def17..6428f647c473 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; @@ -35,6 +36,7 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.metadata.SeriesOverflowException; @@ -59,6 +61,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.logfile.visitor.SchemaRegionPlanSerializer; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MTreeBelowSGMemoryImpl; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowNodesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; @@ -66,6 +69,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowDevicesResult; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.schemaregion.tag.TagManager; import org.apache.iotdb.db.schemaengine.schemaregion.utils.filter.FilterContainsVisitor; @@ -90,6 +94,7 @@ import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SchemaUtils; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Pair; @@ -101,8 +106,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -1286,6 +1293,85 @@ public ISchemaReader getTableDeviceReader( (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); } + @Override + public ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException { + return new ISchemaReader() { + + Iterator devicePathIterator = devicePathList.listIterator(); + + IDeviceSchemaInfo next = null; + + Throwable t = null; + + @Override + public boolean isSuccess() { + return t == null; + } + + @Override + public Throwable getFailure() { + return t; + } + + @Override + public ListenableFuture isBlocked() { + return NOT_BLOCKED; + } + + @Override + public boolean hasNext() { + if (next == null) { + tryGetNext(); + } + return next != null; + } + + @Override + public IDeviceSchemaInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + IDeviceSchemaInfo result = next; + next = null; + return result; + } + + private void tryGetNext() { + while (devicePathIterator.hasNext()) { + try { + IMemMNode node = mtree.getNodeByPath(devicePathIterator.next()); + if (!node.isDevice()) { + continue; + } + IDeviceMNode deviceNode = node.getAsDeviceMNode(); + ShowDevicesResult result = + new ShowDevicesResult( + deviceNode.getFullPath(), + deviceNode.isAlignedNullable(), + deviceNode.getSchemaTemplateId()); + result.setAttributeProvider( + k -> + deviceAttributeStore.getAttribute( + ((TableDeviceInfo) deviceNode.getDeviceInfo()) + .getAttributePointer(), + k)); + next = result; + break; + } catch (PathNotExistException e) { + continue; + } catch (Throwable e) { + t = e; + return; + } + } + } + + @Override + public void close() throws Exception {} + }; + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 6cd1fcd1f937..89cdba5b2ee3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1361,6 +1361,13 @@ public ISchemaReader getTableDeviceReader( ShowTableDevicesPlan showTableDevicesPlan) { throw new UnsupportedOperationException(); } + + @Override + public ISchemaReader getDeviceReader(List devicePathList) + throws MetadataException { + throw new UnsupportedOperationException(); + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java index 6b937ce33a43..f77219cac328 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java @@ -83,4 +83,19 @@ public static List convertToDevicePattern( return pathList; } + + public static List convertToDevicePath( + String database, String tableName, List deviceIdList) { + List devicePathList = new ArrayList<>(deviceIdList.size()); + String[] nodes; + for (String[] idValues : deviceIdList) { + nodes = new String[idValues.length + 3]; + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + System.arraycopy(idValues, 0, nodes, 3, idValues.length); + devicePathList.add(new PartialPath(nodes)); + } + return devicePathList; + } } From 3b27baa7c6d79759ff226ee686eb61fc59a27dcc Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Thu, 9 May 2024 20:09:01 +0800 Subject: [PATCH 21/36] implement query using device cache --- .../schema/source/SchemaSourceFactory.java | 6 +- .../source/TableDeviceSchemaSource.java | 13 +- .../plan/analyze/AnalyzeVisitor.java | 1 - .../plan/planner/LogicalPlanBuilder.java | 6 +- .../plan/planner/LogicalPlanVisitor.java | 2 +- .../plan/planner/OperatorTreeGenerator.java | 2 +- .../read/table/TableDeviceScanNode.java | 62 +++---- .../schema/DeviceInCacheFilterVisitor.java | 66 +++++++ .../schema/TableModelSchemaFetcher.java | 172 +++++++++++++++--- .../table/ShowTableDevicesStatement.java | 18 +- .../read/req/impl/ShowTableDevicesPlan.java | 4 +- .../filter/impl/DeviceFilterToPathUtil.java | 74 ++++++++ .../schema/filter/impl/DeviceIdFilter.java | 3 + 13 files changed, 347 insertions(+), 82 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 49ba2ba58c7e..d3e596c46011 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -103,11 +103,11 @@ public static ISchemaSource getLogicalViewSchemaSource( public static ISchemaSource getTableDeviceSchemaSource( String database, String tableName, - List idDeterminedFilterList, - List idFuzzyFilterList, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, List columnHeaderList) { return new TableDeviceSchemaSource( - database, tableName, idDeterminedFilterList, idFuzzyFilterList, columnHeaderList); + database, tableName, idDeterminedFilterList, idFuzzyFilter, columnHeaderList); } public static ISchemaSource getTableDeviceFetchSource( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java index b09e38e040bb..728e556c0a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -47,22 +47,22 @@ public class TableDeviceSchemaSource implements ISchemaSource private String tableName; - private List idDeterminedFilterList; + private List> idDeterminedFilterList; - private List idFuzzyFilterList; + private SchemaFilter idFuzzyFilter; private List columnHeaderList; public TableDeviceSchemaSource( String database, String tableName, - List idDeterminedFilterList, - List idFuzzyFilterList, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, List columnHeaderList) { this.database = database; this.tableName = tableName; this.idDeterminedFilterList = idDeterminedFilterList; - this.idFuzzyFilterList = idFuzzyFilterList; + this.idFuzzyFilter = idFuzzyFilter; this.columnHeaderList = columnHeaderList; } @@ -116,7 +116,7 @@ public boolean hasNext() { while (index < devicePatternList.size()) { deviceReader = schemaRegion.getTableDeviceReader( - new ShowTableDevicesPlan(devicePatternList.get(index), idFuzzyFilterList)); + new ShowTableDevicesPlan(devicePatternList.get(index), idFuzzyFilter)); index++; if (deviceReader.hasNext()) { return true; @@ -150,7 +150,6 @@ public void close() throws Exception { private List getDevicePatternList() { return DeviceFilterToPathUtil.convertToDevicePattern( database, - tableName, DataNodeTableCache.getInstance().getTable(database, tableName), idDeterminedFilterList); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index f29b73576503..8448ca045827 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -3717,7 +3717,6 @@ public Analysis visitShowTableDevices( List devicePatternList = DeviceFilterToPathUtil.convertToDevicePattern( database, - tableName, DataNodeTableCache.getInstance().getTable(database, tableName), statement.getIdDeterminedFilterList()); PathPatternTree patternTree = new PathPatternTree(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 67b952fe1c7b..6c7e80c260f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -1651,8 +1651,8 @@ public LogicalPlanBuilder planEndTimeColumnInject( public LogicalPlanBuilder planTableDeviceSource( String database, String tableName, - List idDeterminedFilterList, - List idFuzzyFilterList, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, List columnHeaderList) { this.root = new TableDeviceScanNode( @@ -1660,7 +1660,7 @@ public LogicalPlanBuilder planTableDeviceSource( database, tableName, idDeterminedFilterList, - idFuzzyFilterList, + idFuzzyFilter, columnHeaderList, null); return this; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 75b9aca64119..f1aa9c07725e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -1048,7 +1048,7 @@ public PlanNode visitShowTableDevices( statement.getDatabase(), statement.getTableName(), statement.getIdDeterminedFilterList(), - statement.getIdFuzzyFilterList(), + statement.getIdFuzzyFilter(), columnHeaderList) .planSchemaQueryMerge(false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 3ceb7c5d8f27..8f0af6b1f39e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -3286,7 +3286,7 @@ public Operator visitTableDeviceScan( node.getDatabase(), node.getTableName(), node.getIdDeterminedFilterList(), - node.getIdFuzzyFilterList(), + node.getIdFuzzyFilter(), node.getColumnHeaderList())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java index 5439b089f5a5..072ad10cf751 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java @@ -44,9 +44,9 @@ public class TableDeviceScanNode extends SchemaQueryScanNode { private String tableName; - private List idDeterminedFilterList; + private List> idDeterminedFilterList; - private List idFuzzyFilterList; + private SchemaFilter idFuzzyFilter; private List columnHeaderList; @@ -60,15 +60,15 @@ public TableDeviceScanNode( PlanNodeId id, String database, String tableName, - List idDeterminedFilterList, - List idFuzzyFilterList, + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter, List columnHeaderList, TRegionReplicaSet regionReplicaSet) { super(id); this.database = database; this.tableName = tableName; this.idDeterminedFilterList = idDeterminedFilterList; - this.idFuzzyFilterList = idFuzzyFilterList; + this.idFuzzyFilter = idFuzzyFilter; this.columnHeaderList = columnHeaderList; this.schemaRegionReplicaSet = regionReplicaSet; } @@ -81,12 +81,12 @@ public String getTableName() { return tableName; } - public List getIdDeterminedFilterList() { + public List> getIdDeterminedFilterList() { return idDeterminedFilterList; } - public List getIdFuzzyFilterList() { - return idFuzzyFilterList; + public SchemaFilter getIdFuzzyFilter() { + return idFuzzyFilter; } public List getColumnHeaderList() { @@ -116,7 +116,7 @@ public PlanNode clone() { database, tableName, idDeterminedFilterList, - idFuzzyFilterList, + idFuzzyFilter, columnHeaderList, schemaRegionReplicaSet); } @@ -133,14 +133,14 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(tableName, byteBuffer); ReadWriteIOUtils.write(idDeterminedFilterList.size(), byteBuffer); - for (SchemaFilter schemaFilter : idDeterminedFilterList) { - SchemaFilter.serialize(schemaFilter, byteBuffer); + for (List filterList : idDeterminedFilterList) { + ReadWriteIOUtils.write(filterList.size(), byteBuffer); + for (SchemaFilter schemaFilter : filterList) { + SchemaFilter.serialize(schemaFilter, byteBuffer); + } } - ReadWriteIOUtils.write(idFuzzyFilterList.size(), byteBuffer); - for (SchemaFilter schemaFilter : idFuzzyFilterList) { - SchemaFilter.serialize(schemaFilter, byteBuffer); - } + SchemaFilter.serialize(idFuzzyFilter, byteBuffer); ReadWriteIOUtils.write(columnHeaderList.size(), byteBuffer); for (ColumnHeader columnHeader : columnHeaderList) { @@ -155,14 +155,14 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(tableName, stream); ReadWriteIOUtils.write(idDeterminedFilterList.size(), stream); - for (SchemaFilter schemaFilter : idDeterminedFilterList) { - SchemaFilter.serialize(schemaFilter, stream); + for (List filterList : idDeterminedFilterList) { + ReadWriteIOUtils.write(filterList.size(), stream); + for (SchemaFilter schemaFilter : filterList) { + SchemaFilter.serialize(schemaFilter, stream); + } } - ReadWriteIOUtils.write(idFuzzyFilterList.size(), stream); - for (SchemaFilter schemaFilter : idFuzzyFilterList) { - SchemaFilter.serialize(schemaFilter, stream); - } + SchemaFilter.serialize(idFuzzyFilter, stream); ReadWriteIOUtils.write(columnHeaderList.size(), stream); for (ColumnHeader columnHeader : columnHeaderList) { @@ -175,16 +175,16 @@ public static TableDeviceScanNode deserialize(ByteBuffer buffer) { String tableName = ReadWriteIOUtils.readString(buffer); int size = ReadWriteIOUtils.readInt(buffer); - List idDeterminedFilterList = new ArrayList<>(size); + List> idDeterminedFilterList = new ArrayList<>(size); for (int i = 0; i < size; i++) { - idDeterminedFilterList.add(SchemaFilter.deserialize(buffer)); + int singleSize = ReadWriteIOUtils.readInt(buffer); + idDeterminedFilterList.add(new ArrayList<>(singleSize)); + for (int k = 0; k < singleSize; k++) { + idDeterminedFilterList.get(i).add(SchemaFilter.deserialize(buffer)); + } } - size = ReadWriteIOUtils.readInt(buffer); - List idFuzzyFilterList = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - idFuzzyFilterList.add(SchemaFilter.deserialize(buffer)); - } + SchemaFilter idFuzzyFilter = SchemaFilter.deserialize(buffer); size = ReadWriteIOUtils.readInt(buffer); List columnHeaderList = new ArrayList<>(size); @@ -198,7 +198,7 @@ public static TableDeviceScanNode deserialize(ByteBuffer buffer) { database, tableName, idDeterminedFilterList, - idFuzzyFilterList, + idFuzzyFilter, columnHeaderList, null); } @@ -217,7 +217,7 @@ public boolean equals(Object o) { return Objects.equals(database, that.database) && Objects.equals(tableName, that.tableName) && Objects.equals(idDeterminedFilterList, that.idDeterminedFilterList) - && Objects.equals(idFuzzyFilterList, that.idFuzzyFilterList) + && Objects.equals(idFuzzyFilter, that.idFuzzyFilter) && Objects.equals(columnHeaderList, that.columnHeaderList) && Objects.equals(schemaRegionReplicaSet, that.schemaRegionReplicaSet); } @@ -229,7 +229,7 @@ public int hashCode() { database, tableName, idDeterminedFilterList, - idFuzzyFilterList, + idFuzzyFilter, columnHeaderList, schemaRegionReplicaSet); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java new file mode 100644 index 000000000000..56b9b040c6e9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/DeviceInCacheFilterVisitor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; + +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor; +import org.apache.iotdb.commons.schema.filter.impl.DeviceAttributeFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DeviceInCacheFilterVisitor extends SchemaFilterVisitor { + + private final Map attributeIndexMap = new HashMap<>(); + + DeviceInCacheFilterVisitor(List attributeColumns) { + for (int i = 0; i < attributeColumns.size(); i++) { + attributeIndexMap.put(attributeColumns.get(i), i); + } + } + + @Override + protected boolean visitNode(SchemaFilter filter, DeviceEntry deviceEntry) { + return false; + } + + @Override + public boolean visitDeviceIdFilter(DeviceIdFilter filter, DeviceEntry deviceEntry) { + IDeviceID deviceID = deviceEntry.getDeviceID(); + // the first segment is "db.table", skip it + if (deviceID.segmentNum() < filter.getIndex() + 1) { + return false; + } else { + return deviceID.segment(filter.getIndex() + 1).equals(filter.getValue()); + } + } + + @Override + public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, DeviceEntry deviceEntry) { + return filter + .getValue() + .equals(deviceEntry.getAttributeColumnValues().get(attributeIndexMap.get(filter.getKey()))); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 59c3b79c0ac3..a31624698453 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -21,6 +21,9 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.AndFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -52,10 +55,12 @@ import org.apache.tsfile.utils.Pair; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; @@ -325,16 +330,141 @@ public List fetchDeviceSchema( List attributeColumns) { List deviceEntryList = new ArrayList<>(); - long queryId = SessionManager.getInstance().requestQueryId(); - Throwable t = null; - TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); Pair, List> filters = transformExpression(expressionList, tableInstance); List idFilters = filters.getLeft(); List attributeFilters = filters.getRight(); + DeviceInCacheFilterVisitor filterVisitor = new DeviceInCacheFilterVisitor(attributeColumns); + SchemaFilter attributeFilter = getAttributeFilter(attributeFilters); + + List> idPatternList = + DeviceFilterToPathUtil.convertSchemaFilterToOrConcatList(idFilters); + List> idFilterListForFetch = new ArrayList<>(); + boolean cacheFetchedDevice = true; + for (int i = 0; i < idPatternList.size(); i++) { + SchemaFilterCheckResult checkResult = + tryGetInCache( + deviceEntryList, + database, + tableInstance, + idPatternList.get(i), + o -> attributeFilter == null || filterVisitor.process(attributeFilter, o), + attributeColumns); + if (checkResult.needFetch) { + idFilterListForFetch.add(idPatternList.get(i)); + if (!checkResult.isIdDetermined) { + cacheFetchedDevice = false; + } + } + } + + if (!idFilterListForFetch.isEmpty()) { + fetchMissingDeviceSchemaForQuery( + database, + tableInstance, + attributeColumns, + idFilterListForFetch, + attributeFilter, + deviceEntryList, + cacheFetchedDevice); + } + + return deviceEntryList; + } + + private Pair, List> transformExpression( + List expressionList, TsTable table) { + List idDeterminedFilters = new ArrayList<>(); + List idFuzzyFilters = new ArrayList<>(); + ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor(); + ConvertSchemaPredicateToFilterVisitor.Context context = + new ConvertSchemaPredicateToFilterVisitor.Context(table); + for (Expression expression : expressionList) { + if (expression == null) { + continue; + } + context.reset(); + SchemaFilter schemaFilter = expression.accept(visitor, context); + if (context.hasAttribute()) { + idFuzzyFilters.add(schemaFilter); + } else { + idDeterminedFilters.add(schemaFilter); + } + } + return new Pair<>(idDeterminedFilters, idFuzzyFilters); + } + + // return whether this condition shall be used for remote fetch + private SchemaFilterCheckResult tryGetInCache( + List deviceEntryList, + String database, + TsTable tableInstance, + List idFilters, + Predicate check, + List attributeColumns) { + String[] idValues = new String[tableInstance.getIdNums()]; + for (SchemaFilter schemaFilter : idFilters) { + DeviceIdFilter idFilter = (DeviceIdFilter) schemaFilter; + if (idValues[idFilter.getIndex()] == null) { + idValues[idFilter.getIndex()] = idFilter.getValue(); + } else { + // conflict filter + return new SchemaFilterCheckResult(false, false); + } + } + if (idFilters.size() < idValues.length) { + return new SchemaFilterCheckResult(true, false); + } + Map attributeMap = + cache.getDeviceAttribute(database, tableInstance.getTableName(), idValues); + if (attributeMap == null) { + return new SchemaFilterCheckResult(true, true); + } + List attributeValues = new ArrayList<>(attributeColumns.size()); + for (String attributeKey : attributeColumns) { + String value = attributeMap.get(attributeKey); + if (value == null) { + return new SchemaFilterCheckResult(true, true); + } else { + attributeValues.add(value); + } + } + String[] deviceIdNodes = new String[idValues.length + 1]; + deviceIdNodes[0] = database + PATH_SEPARATOR + tableInstance.getTableName(); + System.arraycopy(idValues, 0, deviceIdNodes, 1, idValues.length); + DeviceEntry deviceEntry = + new DeviceEntry(new StringArrayDeviceID(deviceIdNodes), attributeValues); + if (check.test(deviceEntry)) { + deviceEntryList.add(deviceEntry); + } + return new SchemaFilterCheckResult(false, true); + } + + private static class SchemaFilterCheckResult { + boolean needFetch; + boolean isIdDetermined; + + SchemaFilterCheckResult(boolean needFetch, boolean isIdDetermined) { + this.needFetch = needFetch; + this.isIdDetermined = isIdDetermined; + } + } + + private void fetchMissingDeviceSchemaForQuery( + String database, + TsTable tableInstance, + List attributeColumns, + List> idPatternList, + SchemaFilter attributeFilter, + List deviceEntryList, + boolean cacheFetchedDevice) { + + String table = tableInstance.getTableName(); + + long queryId = SessionManager.getInstance().requestQueryId(); ShowTableDevicesStatement statement = - new ShowTableDevicesStatement(database, table, idFilters, attributeFilters); + new ShowTableDevicesStatement(database, table, idPatternList, attributeFilter); ExecutionResult executionResult = Coordinator.getInstance() .executeForTreeModel( @@ -357,6 +487,7 @@ public List fetchDeviceSchema( int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); Map attributeMap; + Throwable t = null; try { while (coordinator.getQueryExecution(queryId).hasNextResult()) { Optional tsBlock; @@ -390,6 +521,9 @@ public List fetchDeviceSchema( new DeviceEntry( deviceID, attributeColumns.stream().map(attributeMap::get).collect(Collectors.toList()))); + if (cacheFetchedDevice) { + cache.put(database, table, Arrays.copyOfRange(nodes, 1, nodes.length), attributeMap); + } } } } catch (Throwable throwable) { @@ -398,28 +532,18 @@ public List fetchDeviceSchema( } finally { coordinator.cleanupQueryExecution(queryId, null, t); } - return deviceEntryList; } - private Pair, List> transformExpression( - List expressionList, TsTable table) { - List idDeterminedFilters = new ArrayList<>(); - List idFuzzyFilters = new ArrayList<>(); - ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor(); - ConvertSchemaPredicateToFilterVisitor.Context context = - new ConvertSchemaPredicateToFilterVisitor.Context(table); - for (Expression expression : expressionList) { - if (expression == null) { - continue; - } - context.reset(); - SchemaFilter schemaFilter = expression.accept(visitor, context); - if (context.hasAttribute()) { - idFuzzyFilters.add(schemaFilter); - } else { - idDeterminedFilters.add(schemaFilter); - } + private SchemaFilter getAttributeFilter(List filterList) { + if (filterList.isEmpty()) { + return null; } - return new Pair<>(idDeterminedFilters, idFuzzyFilters); + AndFilter andFilter; + SchemaFilter latestFilter = filterList.get(0); + for (int i = 1; i < filterList.size(); i++) { + andFilter = new AndFilter(latestFilter, filterList.get(i)); + latestFilter = andFilter; + } + return latestFilter; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java index fec8eec1b27a..ec3f9ffb266b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/table/ShowTableDevicesStatement.java @@ -31,20 +31,20 @@ public class ShowTableDevicesStatement extends ShowStatement { private final String tableName; - private final List idDeterminedFilterList; + private final List> idDeterminedFilterList; - private final List idFuzzyFilterList; + private final SchemaFilter idFuzzyFilter; public ShowTableDevicesStatement( String database, String tableName, - List idDeterminedFilterList, - List idFuzzyFilterList) { + List> idDeterminedFilterList, + SchemaFilter idFuzzyFilter) { super(); this.database = database; this.tableName = tableName; this.idDeterminedFilterList = idDeterminedFilterList; - this.idFuzzyFilterList = idFuzzyFilterList; + this.idFuzzyFilter = idFuzzyFilter; } public String getDatabase() { @@ -55,12 +55,12 @@ public String getTableName() { return tableName; } - public List getIdDeterminedFilterList() { - return idDeterminedFilterList; + public SchemaFilter getIdFuzzyFilter() { + return idFuzzyFilter; } - public List getIdFuzzyFilterList() { - return idFuzzyFilterList; + public List> getIdDeterminedFilterList() { + return idDeterminedFilterList; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java index d3af79e093d8..e579d235f6bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java @@ -12,9 +12,9 @@ public class ShowTableDevicesPlan { private SchemaFilter attributeFilter; - public ShowTableDevicesPlan(PartialPath devicePattern, List attributeFilterList) { + public ShowTableDevicesPlan(PartialPath devicePattern, SchemaFilter attributeFilter) { this.devicePattern = devicePattern; - this.attributeFilter = getAttributeFilter(attributeFilterList); + this.attributeFilter = attributeFilter; } private SchemaFilter getAttributeFilter(List filterList) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java index f77219cac328..0ba46b7a1a0d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; @@ -84,6 +85,32 @@ public static List convertToDevicePattern( return pathList; } + // if the element in idDeterminedFilterList isEmpty, the corresponding pattern is + // root.db.table.*.*.. + public static List convertToDevicePattern( + String database, TsTable table, List> idDeterminedFilterList) { + List pathList = new ArrayList<>(); + int length = table.getIdNums() + 3; + for (List idFilterList : idDeterminedFilterList) { + String[] nodes = new String[length]; + Arrays.fill(nodes, ONE_LEVEL_PATH_WILDCARD); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = table.getTableName(); + for (SchemaFilter schemaFilter : idFilterList) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else { + throw new IllegalStateException("Input single filter must be DeviceIdFilter"); + } + } + pathList.add(new PartialPath(nodes)); + } + + return pathList; + } + public static List convertToDevicePath( String database, String tableName, List deviceIdList) { List devicePathList = new ArrayList<>(deviceIdList.size()); @@ -98,4 +125,51 @@ public static List convertToDevicePath( } return devicePathList; } + + // input and-concat filter list + // return or concat filter list, inner which all filter is and concat + public static List> convertSchemaFilterToOrConcatList( + List schemaFilterList) { + List> orConcatList = + schemaFilterList.stream() + .map(DeviceFilterToPathUtil::convertOneSchemaFilterToOrConcat) + .collect(Collectors.toList()); + int orSize = orConcatList.size(); + int finalResultSize = 1; + for (List filterList : orConcatList) { + finalResultSize *= filterList.size(); + } + List> result = new ArrayList<>(finalResultSize); + int[] indexes = new int[orSize]; + while (finalResultSize > 0) { + List oneCase = new ArrayList<>(orConcatList.size()); + for (int j = 0; j < orSize; j++) { + oneCase.add(orConcatList.get(j).get(indexes[j])); + } + result.add(oneCase); + for (int k = orSize - 1; k >= 0; k--) { + indexes[k]++; + if (indexes[k] < orConcatList.get(k).size()) { + break; + } + indexes[k] = 0; + } + finalResultSize--; + } + return result; + } + + private static List convertOneSchemaFilterToOrConcat(SchemaFilter schemaFilter) { + List result = new ArrayList<>(); + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + result.addAll(convertOneSchemaFilterToOrConcat(orFilter.getLeft())); + result.addAll(convertOneSchemaFilterToOrConcat(orFilter.getRight())); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.AND)) { + throw new IllegalStateException("Input filter shall not be AND operation"); + } else { + result.add(schemaFilter); + } + return result; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java index 149766bdb95e..6ba5cd71f54e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceIdFilter.java @@ -30,6 +30,9 @@ public class DeviceIdFilter extends SchemaFilter { + // id column index + // when used in partialPath, the index of node in path shall be [this.index + 3] + // since a partialPath start with {root, db, table} private final int index; private final String value; From b0a536b3f5f3695075ad2eab8eb62fdc1485849c Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Fri, 10 May 2024 09:08:42 +0800 Subject: [PATCH 22/36] improve code smell --- .../schema/source/TableDeviceFetchSource.java | 4 +- .../source/TableDeviceSchemaSource.java | 4 +- .../plan/analyze/AnalyzeVisitor.java | 7 +-- .../schema/TableModelSchemaFetcher.java | 8 +-- ...rToPathUtil.java => DeviceFilterUtil.java} | 56 ++----------------- 5 files changed, 17 insertions(+), 62 deletions(-) rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/{DeviceFilterToPathUtil.java => DeviceFilterUtil.java} (68%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java index f91397d5e239..a7eb1b6afb26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -63,7 +63,7 @@ public TableDeviceFetchSource( public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { try { return schemaRegion.getDeviceReader( - DeviceFilterToPathUtil.convertToDevicePath(database, tableName, deviceIdList)); + DeviceFilterUtil.convertToDevicePath(database, tableName, deviceIdList)); } catch (MetadataException e) { throw new SchemaExecutionException(e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java index 728e556c0a20..f94f19e126b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -148,7 +148,7 @@ public void close() throws Exception { } private List getDevicePatternList() { - return DeviceFilterToPathUtil.convertToDevicePattern( + return DeviceFilterUtil.convertToDevicePattern( database, DataNodeTableCache.getInstance().getTable(database, tableName), idDeterminedFilterList); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 8448ca045827..b151b177cea8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -33,7 +33,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -3715,7 +3715,7 @@ public Analysis visitShowTableDevices( String tableName = statement.getTableName(); List devicePatternList = - DeviceFilterToPathUtil.convertToDevicePattern( + DeviceFilterUtil.convertToDevicePattern( database, DataNodeTableCache.getInstance().getTable(database, tableName), statement.getIdDeterminedFilterList()); @@ -3744,8 +3744,7 @@ public Analysis visitFetchTableDevices( String tableName = statement.getTableName(); List devicePatternList = - DeviceFilterToPathUtil.convertToDevicePath( - database, tableName, statement.getDeviceIdList()); + DeviceFilterUtil.convertToDevicePath(database, tableName, statement.getDeviceIdList()); PathPatternTree patternTree = new PathPatternTree(); for (PartialPath devicePattern : devicePatternList) { patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index a31624698453..f9c74de61ea2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.impl.AndFilter; -import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; @@ -339,12 +339,12 @@ public List fetchDeviceSchema( SchemaFilter attributeFilter = getAttributeFilter(attributeFilters); List> idPatternList = - DeviceFilterToPathUtil.convertSchemaFilterToOrConcatList(idFilters); + DeviceFilterUtil.convertSchemaFilterToOrConcatList(idFilters); List> idFilterListForFetch = new ArrayList<>(); boolean cacheFetchedDevice = true; for (int i = 0; i < idPatternList.size(); i++) { SchemaFilterCheckResult checkResult = - tryGetInCache( + checkIdFilterAndTryGetDeviceInCache( deviceEntryList, database, tableInstance, @@ -396,7 +396,7 @@ private Pair, List> transformExpression( } // return whether this condition shall be used for remote fetch - private SchemaFilterCheckResult tryGetInCache( + private SchemaFilterCheckResult checkIdFilterAndTryGetDeviceInCache( List deviceEntryList, String database, TsTable tableInstance, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java similarity index 68% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java index 0ba46b7a1a0d..c65ccf2579d8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterUtil.java @@ -26,67 +26,21 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; -public class DeviceFilterToPathUtil { +public class DeviceFilterUtil { - private DeviceFilterToPathUtil() { + private DeviceFilterUtil() { // do nothing } - public static List convertToDevicePattern( - String database, String tableName, TsTable table, List idDeterminedFilterList) { - int length = table.getIdNums() + 3; - String[] nodes = new String[length]; - Arrays.fill(nodes, ONE_LEVEL_PATH_WILDCARD); - nodes[0] = PATH_ROOT; - nodes[1] = database; - nodes[2] = tableName; - Map> orValueMap = new HashMap<>(); - for (SchemaFilter schemaFilter : idDeterminedFilterList) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - OrFilter orFilter = (OrFilter) schemaFilter; - if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) - && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - // todo nested orFilter - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - orValueMap - .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) - .add(deviceIdFilter.getValue()); - } - } - } - - PartialPath path = new PartialPath(nodes); - List pathList = new ArrayList<>(); - pathList.add(path); - for (Map.Entry> entry : orValueMap.entrySet()) { - for (int i = 0, size = pathList.size(); i < size; i++) { - for (String value : entry.getValue()) { - nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); - nodes[entry.getKey() + 3] = value; - path = new PartialPath(nodes); - pathList.add(path); - } - } - } - - return pathList; - } - // if the element in idDeterminedFilterList isEmpty, the corresponding pattern is // root.db.table.*.*.. + // e.g. input (db, table[c1, c2], [[]]), return [root.db.table.*.*] public static List convertToDevicePattern( String database, TsTable table, List> idDeterminedFilterList) { List pathList = new ArrayList<>(); @@ -128,11 +82,13 @@ public static List convertToDevicePath( // input and-concat filter list // return or concat filter list, inner which all filter is and concat + // e.g. (a OR b) AND (c OR d) -> (a AND c) OR (a AND d) OR (b AND c) OR (b AND d) + // if input is empty, then return [[]] public static List> convertSchemaFilterToOrConcatList( List schemaFilterList) { List> orConcatList = schemaFilterList.stream() - .map(DeviceFilterToPathUtil::convertOneSchemaFilterToOrConcat) + .map(DeviceFilterUtil::convertOneSchemaFilterToOrConcat) .collect(Collectors.toList()); int orSize = orConcatList.size(); int finalResultSize = 1; From a867ffb8baf7a58d4b29aff3a8b5bf7b52896561 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 12 May 2024 19:08:35 +0800 Subject: [PATCH 23/36] support null device value --- .../schema/source/TableDeviceFetchSource.java | 31 ++-- .../source/TableDeviceSchemaSource.java | 30 +-- .../schema/TableModelSchemaFetcher.java | 24 ++- .../statement/crud/InsertTableStatement.java | 2 +- .../impl/SchemaRegionMemoryImpl.java | 3 +- .../impl/mem/MTreeBelowSGMemoryImpl.java | 5 +- .../impl/mem/mnode/basic/BasicMNode.java | 3 +- .../mnode/container/MemMNodeContainer.java | 3 +- .../container/NullableConcurrentHashMap.java | 175 ++++++++++++++++++ .../impl/mem/mnode/info/BasicMNodeInfo.java | 2 +- .../read/resp/info/IDeviceSchemaInfo.java | 2 + .../resp/info/impl/ShowDevicesResult.java | 19 ++ .../iotdb/commons/path/PartialPath.java | 5 +- .../iotdb/commons/path/PathPatternUtil.java | 3 +- 14 files changed, 270 insertions(+), 37 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java index a7eb1b6afb26..654d3e28f8f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; -import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; @@ -80,24 +79,32 @@ public void transformToTsBlockColumns( builder.getTimeColumnBuilder().writeLong(0L); int resultIndex = 0; int idIndex = 0; - PartialPath devicePath = schemaInfo.getPartialPath(); + String[] pathNodes = schemaInfo.getRawNodes(); TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); TsTableColumnSchema columnSchema; for (ColumnHeader columnHeader : columnHeaderList) { columnSchema = table.getColumnSchema(columnHeader.getColumnName()); if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - builder - .getColumnBuilder(resultIndex) - .writeBinary( - new Binary(devicePath.getNodes()[idIndex + 3], TSFileConfig.STRING_CHARSET)); + if (pathNodes[idIndex + 3] == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary(new Binary(pathNodes[idIndex + 3], TSFileConfig.STRING_CHARSET)); + } idIndex++; } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { - builder - .getColumnBuilder(resultIndex) - .writeBinary( - new Binary( - schemaInfo.getAttributeValue(columnHeader.getColumnName()), - TSFileConfig.STRING_CHARSET)); + String attributeValue = schemaInfo.getAttributeValue(columnHeader.getColumnName()); + if (attributeValue == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } } resultIndex++; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java index f94f19e126b8..b5275b444c7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -165,24 +165,32 @@ public void transformToTsBlockColumns( builder.getTimeColumnBuilder().writeLong(0L); int resultIndex = 0; int idIndex = 0; - PartialPath devicePath = schemaInfo.getPartialPath(); + String[] pathNodes = schemaInfo.getRawNodes(); TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); TsTableColumnSchema columnSchema; for (ColumnHeader columnHeader : columnHeaderList) { columnSchema = table.getColumnSchema(columnHeader.getColumnName()); if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - builder - .getColumnBuilder(resultIndex) - .writeBinary( - new Binary(devicePath.getNodes()[idIndex + 3], TSFileConfig.STRING_CHARSET)); + if (pathNodes[idIndex + 3] == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary(new Binary(pathNodes[idIndex + 3], TSFileConfig.STRING_CHARSET)); + } idIndex++; } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { - builder - .getColumnBuilder(resultIndex) - .writeBinary( - new Binary( - schemaInfo.getAttributeValue(columnHeader.getColumnName()), - TSFileConfig.STRING_CHARSET)); + String attributeValue = schemaInfo.getAttributeValue(columnHeader.getColumnName()); + if (attributeValue == null) { + builder.getColumnBuilder(resultIndex).appendNull(); + } else { + builder + .getColumnBuilder(resultIndex) + .writeBinary( + new Binary( + schemaInfo.getAttributeValue(columnHeader.getColumnName()), + TSFileConfig.STRING_CHARSET)); + } } resultIndex++; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index f9c74de61ea2..6620a3a897c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -255,10 +255,18 @@ private Map> fetchMissingDeviceSchema( TsTableColumnSchema columnSchema = tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - nodes[idIndex] = columns[j].getBinary(i).toString(); + if (columns[j].isNull(i)) { + nodes[idIndex] = null; + } else { + nodes[idIndex] = columns[j].getBinary(i).toString(); + } idIndex++; } else { - attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + if (columns[j].isNull(i)) { + attributeMap.put(columnSchema.getColumnName(), null); + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } } } fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap); @@ -510,10 +518,18 @@ private void fetchMissingDeviceSchemaForQuery( TsTableColumnSchema columnSchema = tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - nodes[idIndex + 1] = columns[j].getBinary(i).toString(); + if (columns[j].isNull(i)) { + nodes[idIndex + 1] = null; + } else { + nodes[idIndex + 1] = columns[j].getBinary(i).toString(); + } idIndex++; } else { - attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + if (columns[j].isNull(i)) { + attributeMap.put(columnSchema.getColumnName(), null); + } else { + attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + } } } IDeviceID deviceID = new StringArrayDeviceID(nodes); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index a884e8d69dad..8e9c8fc17c38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -130,7 +130,7 @@ private InsertRowStatement parseInsert(Insert insert) { TsTableColumnCategory category = table.getColumnList().get(i).getColumnCategory(); if (category.equals(TsTableColumnCategory.ID)) { String id = idColumnMap.get(table.getColumnList().get(i).getColumnName()); - deviceIds[3 + idIndex] = id == null ? "" : id; + deviceIds[3 + idIndex] = id; idIndex++; } else if (category.equals(TsTableColumnCategory.MEASUREMENT)) { String measurement = table.getColumnList().get(i).getColumnName(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 6428f647c473..7e1174aa764c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1349,7 +1349,8 @@ private void tryGetNext() { new ShowDevicesResult( deviceNode.getFullPath(), deviceNode.isAlignedNullable(), - deviceNode.getSchemaTemplateId()); + deviceNode.getSchemaTemplateId(), + deviceNode.getPartialPath().getNodes()); result.setAttributeProvider( k -> deviceAttributeStore.getAttribute( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index d78b8325a370..297a3fb49f97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1042,7 +1042,10 @@ protected IDeviceSchemaInfo collectEntity(IDeviceMNode node) { PartialPath device = getPartialPathFromRootToNode(node.getAsMNode()); ShowDevicesResult result = new ShowDevicesResult( - device.getFullPath(), node.isAlignedNullable(), node.getSchemaTemplateId()); + device.getFullPath(), + node.isAlignedNullable(), + node.getSchemaTemplateId(), + node.getPartialPath().getNodes()); result.setAttributeProvider( k -> attributeProvider.apply( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java index ccacef6c7ebe..ea1ff409ecbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java @@ -84,7 +84,8 @@ public String getFullPath() { } String concatFullPath() { - StringBuilder builder = new StringBuilder(getName()); + StringBuilder builder = new StringBuilder(); + builder.insert(0, getName()); IMemMNode curr = this; while (curr.getParent() != null) { curr = curr.getParent(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java index 44c16126ea87..6a84975ce6ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java @@ -28,11 +28,10 @@ import java.util.Collection; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import static java.util.Collections.emptySet; -public class MemMNodeContainer extends ConcurrentHashMap +public class MemMNodeContainer extends NullableConcurrentHashMap implements IMNodeContainer { private static final IMNodeContainer EMPTY_CONTAINER = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java new file mode 100644 index 000000000000..6683d94d6a20 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.container; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +// The value in this map shall not be null. +// Therefore, when using compute method, use v==null to judge if there's existing value. +public class NullableConcurrentHashMap implements Map { + + private final Map, V> map = new ConcurrentHashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(Optional.ofNullable(key)); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(value); + } + + @Override + public V get(Object key) { + return map.get(Optional.ofNullable(key)); + } + + @Override + public V put(K key, V value) { + return map.put(Optional.ofNullable(key), value); + } + + @Override + public V remove(Object key) { + return map.remove(Optional.ofNullable(key)); + } + + @Override + public void putAll(Map m) { + m.forEach((k, v) -> map.put(Optional.ofNullable(k), v)); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Set keySet() { + return map.keySet().stream().map(k -> k.orElse(null)).collect(Collectors.toSet()); + } + + @Override + public Collection values() { + return map.values(); + } + + @Override + public Set> entrySet() { + return map.entrySet().stream() + .map( + o -> + new Entry() { + @Override + public K getKey() { + return o.getKey().orElse(null); + } + + @Override + public V getValue() { + return o.getValue(); + } + + @Override + public V setValue(V value) { + return o.setValue(value); + } + }) + .collect(Collectors.toSet()); + } + + @Override + public V getOrDefault(Object key, V defaultValue) { + return map.getOrDefault(Optional.ofNullable(key), defaultValue); + } + + @Override + public void forEach(BiConsumer action) { + map.forEach((k, v) -> action.accept(k.orElse(null), v)); + } + + @Override + public void replaceAll(BiFunction function) { + map.replaceAll((k, v) -> function.apply(k.orElse(null), v)); + } + + @Override + public V putIfAbsent(K key, V value) { + return map.putIfAbsent(Optional.ofNullable(key), value); + } + + @Override + public boolean remove(Object key, Object value) { + return map.remove(Optional.ofNullable(key), value); + } + + @Override + public boolean replace(K key, V oldValue, V newValue) { + return map.replace(Optional.ofNullable(key), oldValue, newValue); + } + + @Override + public V replace(K key, V value) { + return map.replace(Optional.ofNullable(key), value); + } + + @Override + public V computeIfAbsent(K key, Function mappingFunction) { + return map.computeIfAbsent( + Optional.ofNullable(key), k -> mappingFunction.apply(k.orElse(null))); + } + + @Override + public V computeIfPresent( + K key, BiFunction remappingFunction) { + return map.computeIfPresent( + Optional.ofNullable(key), (k, v) -> remappingFunction.apply(k.orElse(null), v)); + } + + @Override + public V compute(K key, BiFunction remappingFunction) { + return map.compute( + Optional.ofNullable(key), (k, v) -> remappingFunction.apply(k.orElse(null), v)); + } + + @Override + public V merge(K key, V value, BiFunction remappingFunction) { + return map.merge(Optional.ofNullable(key), value, remappingFunction); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java index 36f92bfda56b..97d13ee6757d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/BasicMNodeInfo.java @@ -37,6 +37,6 @@ public void setName(String name) { public int estimateSize() { // object header, 8B // name reference, name length and name hash code, 8 + 4 + 4 = 16B - return 8 + 16 + 2 * name.length(); + return 8 + 16 + 2 * (name == null ? 0 : name.length()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java index 634f7d654615..a144e10c9c15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java @@ -26,4 +26,6 @@ public interface IDeviceSchemaInfo extends ISchemaInfo { int getTemplateId(); String getAttributeValue(String attributeKey); + + String[] getRawNodes(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java index 8f4c5b95a6bd..fda9cf1f300e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import java.util.Objects; @@ -29,12 +30,21 @@ public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchema private Function attributeProvider; + private String[] rawNodes = null; + public ShowDevicesResult(String name, Boolean isAligned, int templateId) { super(name); this.isAligned = isAligned; this.templateId = templateId; } + public ShowDevicesResult(String name, Boolean isAligned, int templateId, String[] rawNodes) { + super(name); + this.isAligned = isAligned; + this.templateId = templateId; + this.rawNodes = rawNodes; + } + public Boolean isAligned() { return isAligned; } @@ -52,6 +62,15 @@ public String getAttributeValue(String attributeKey) { return attributeProvider.apply(attributeKey); } + public String[] getRawNodes() { + return rawNodes; + } + + @Override + public PartialPath getPartialPath() { + return new PartialPath(rawNodes); + } + @Override public String toString() { return "ShowDevicesResult{" diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 2ffeca565963..af42e170a796 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; @@ -698,7 +699,7 @@ public boolean equals(Object obj) { return false; } else { for (int i = 0; i < this.nodes.length; i++) { - if (!nodes[i].equals(otherNodes[i])) { + if (!Objects.equals(nodes[i], otherNodes[i])) { return false; } } @@ -715,7 +716,7 @@ public boolean equals(String obj) { public int hashCode() { int h = 0; for (String node : nodes) { - h += 31 * h + node.hashCode(); + h += 31 * h + Objects.hashCode(node); } return h; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java index a8ba920813b6..6ee736453500 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternUtil.java @@ -33,7 +33,8 @@ private PathPatternUtil() {} * patternNode that can match batch explicit node names. e.g. *, e.g. *, **, d*, *d*. */ public static boolean hasWildcard(String node) { - return node.startsWith(ONE_LEVEL_PATH_WILDCARD) || node.endsWith(ONE_LEVEL_PATH_WILDCARD); + return node != null + && (node.startsWith(ONE_LEVEL_PATH_WILDCARD) || node.endsWith(ONE_LEVEL_PATH_WILDCARD)); } public static boolean isMultiLevelMatchWildcard(String node) { From 1b5f92328f5ba76e85044abdec760baf92252e91 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 19 May 2024 10:20:11 +0800 Subject: [PATCH 24/36] implement is null --- .../ConvertSchemaPredicateToFilterVisitor.java | 12 +++++++++++- .../predicate/PredicatePushIntoIndexScanChecker.java | 2 +- .../analyzer/schema/cache/TableDeviceCacheEntry.java | 7 ++----- ...ashMap.java => KeyNullableConcurrentHashMap.java} | 2 +- .../impl/mem/mnode/container/MemMNodeContainer.java | 2 +- .../utils/filter/DeviceFilterVisitor.java | 4 +++- .../apache/iotdb/commons/path/fa/nfa/SimpleNFA.java | 5 ++++- .../commons/schema/tree/AbstractTreeVisitor.java | 4 +++- 8 files changed, 26 insertions(+), 12 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/{NullableConcurrentHashMap.java => KeyNullableConcurrentHashMap.java} (98%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java index 5c8fa1e6bb22..d4811ae4ad8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java @@ -57,7 +57,17 @@ protected SchemaFilter visitInPredicate(InPredicate node, Context context) { @Override protected SchemaFilter visitIsNullPredicate(IsNullPredicate node, Context context) { - return visitExpression(node, context); + String columnName = ((SymbolReference) node.getValue()).getName(); + if (context + .table + .getColumnSchema(columnName) + .getColumnCategory() + .equals(TsTableColumnCategory.ID)) { + return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), null); + } else { + context.hasAttribute = true; + return new DeviceAttributeFilter(columnName, null); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java index 3f7488205df5..cd2d90974473 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java @@ -60,7 +60,7 @@ protected Boolean visitInPredicate(InPredicate node, Void context) { @Override protected Boolean visitIsNullPredicate(IsNullPredicate node, Void context) { - return Boolean.FALSE; + return isIdOrAttributeColumn(node.getValue()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java index 18c9355cc368..7d60e91111c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceCacheEntry.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -33,11 +34,7 @@ public TableDeviceCacheEntry() { } public TableDeviceCacheEntry(Map attributeMap) { - this.attributeMap = new ConcurrentHashMap<>(attributeMap); - } - - public void putAttribute(String key, String value) { - attributeMap.put(key, value); + this.attributeMap = new HashMap<>(attributeMap); } public String getAttribute(String key) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java similarity index 98% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java index 6683d94d6a20..d5bfaa4717a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/NullableConcurrentHashMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/KeyNullableConcurrentHashMap.java @@ -31,7 +31,7 @@ // The value in this map shall not be null. // Therefore, when using compute method, use v==null to judge if there's existing value. -public class NullableConcurrentHashMap implements Map { +public class KeyNullableConcurrentHashMap implements Map { private final Map, V> map = new ConcurrentHashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java index 6a84975ce6ad..97de504f4b5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/container/MemMNodeContainer.java @@ -31,7 +31,7 @@ import static java.util.Collections.emptySet; -public class MemMNodeContainer extends NullableConcurrentHashMap +public class MemMNodeContainer extends KeyNullableConcurrentHashMap implements IMNodeContainer { private static final IMNodeContainer EMPTY_CONTAINER = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java index 070b2b40c176..f80dfba0c2a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/filter/DeviceFilterVisitor.java @@ -28,6 +28,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; +import java.util.Objects; + public class DeviceFilterVisitor extends SchemaFilterVisitor { @Override public boolean visitNode(SchemaFilter filter, IDeviceSchemaInfo info) { @@ -74,6 +76,6 @@ public boolean visitDeviceIdFilter(DeviceIdFilter filter, IDeviceSchemaInfo info @Override public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, IDeviceSchemaInfo info) { - return filter.getValue().equals(info.getAttributeValue(filter.getKey())); + return Objects.equals(filter.getValue(), info.getAttributeValue(filter.getKey())); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java index 698f27b37c85..4a68861af1d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/fa/nfa/SimpleNFA.java @@ -29,6 +29,7 @@ import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.regex.Pattern; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; @@ -123,6 +124,8 @@ private SinglePathPatternNode getNextNode(SinglePathPatternNode currentNode) { if (patternNodes[nextIndex] == null) { if (nextIndex == rawNodes.length) { patternNodes[nextIndex] = new PrefixMatchNode(nextIndex, currentNode.getTracebackNode()); + } else if (rawNodes[nextIndex] == null) { + patternNodes[nextIndex] = new NameMatchNode(nextIndex, currentNode.getTracebackNode()); } else if (rawNodes[nextIndex].equals(MULTI_LEVEL_PATH_WILDCARD)) { patternNodes[nextIndex] = new MultiLevelWildcardMatchNode(nextIndex); } else if (rawNodes[nextIndex].equals(ONE_LEVEL_PATH_WILDCARD)) { @@ -439,7 +442,7 @@ private NameMatchNode(int patternIndex, SinglePathPatternNode tracebackNode) { @Override public boolean isMatch(String event) { - return rawNodes[patternIndex].equals(event); + return Objects.equals(rawNodes[patternIndex], event); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java index caa8019ef871..606d1b00dafd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java @@ -121,7 +121,9 @@ protected AbstractTreeVisitor( boolean usingDFA = false; // Use DFA if there are ** and no regex node in pathPattern for (String pathNode : pathPattern.getNodes()) { - if (IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(pathNode)) { + if (pathNode == null) { + continue; + } else if (IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD.equals(pathNode)) { // ** node usingDFA = true; } else if (pathNode.length() > 1 && PathPatternUtil.hasWildcard(pathNode)) { From 987f06f03381c56f72003fc130887c8ba687537a Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 19 May 2024 16:09:32 +0800 Subject: [PATCH 25/36] set up table validate --- .../plan/analyze/AnalyzeVisitor.java | 22 +++- .../schema/TableModelSchemaFetcher.java | 106 ++++++++++++++++++ .../statement/crud/InsertTableStatement.java | 55 +++++++-- 3 files changed, 168 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index b151b177cea8..65bcbc18b5a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -3659,9 +3659,27 @@ public Analysis visitInsertTable( InsertTableStatement insertTableStatement, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); Analysis analysis = new Analysis(); + + try { + insertTableStatement.setTableSchema( + TableModelSchemaFetcher.getInstance() + .validateTableHeaderSchema( + insertTableStatement.getDatabase(), + insertTableStatement.getTableSchema(), + context)); + } catch (SemanticException e) { + analysis.setFinishQueryAfterAnalyze(true); + if (e.getCause() instanceof IoTDBException) { + IoTDBException exception = (IoTDBException) e.getCause(); + analysis.setFailStatus( + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage())); + } else { + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); + } + } + InsertRowStatement insertRowStatement = insertTableStatement.getInsertRowStatement(); - final long startTime = System.nanoTime(); try { TableModelSchemaFetcher.getInstance().validateDeviceSchema(insertTableStatement, context); } catch (SemanticException e) { @@ -3673,8 +3691,6 @@ public Analysis visitInsertTable( } else { analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.METADATA_ERROR, e.getMessage())); } - } finally { - PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost(System.nanoTime() - startTime); } if (analysis.isFinishQueryAfterAnalyze()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 6620a3a897c7..762fdb7ba375 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -40,7 +42,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertSchemaPredicateToFilterVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceId; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.statement.table.CreateTableDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.FetchTableDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.table.ShowTableDevicesStatement; @@ -52,6 +56,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.read.common.type.UnknownType; import org.apache.tsfile.utils.Pair; import java.util.ArrayList; @@ -85,6 +91,106 @@ private TableModelSchemaFetcher() { // do nothing } + // This method return all the existing column schemas in the target table. + // When table or column is missing, this method will execute auto creation. + // When using SQL, the columnSchemaList could be null and there won't be any validation. + // All input column schemas will be validated and auto created when necessary. + // When the input dataType or category of one column is null, the column cannot be auto created. + public TableSchema validateTableHeaderSchema( + String database, TableSchema tableSchema, MPPQueryContext context) { + List inputColumnList = tableSchema.getColumns(); + TsTable table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); + List missingColumnList = new ArrayList<>(); + List resultColumnList = new ArrayList<>(); + + // first round validate, check existing schema + if (table == null) { + if (inputColumnList == null) { + throw new SemanticException("Unknown column names. Cannot auto create table."); + } + // check arguments for table auto creation + for (ColumnSchema columnSchema : inputColumnList) { + if (columnSchema.getColumnCategory() == null) { + throw new SemanticException("Unknown column category. Cannot auto create table."); + } + if (columnSchema.getType() == null) { + throw new IllegalArgumentException("Unknown column data type. Cannot auto create table."); + } + missingColumnList.add(columnSchema); + } + } else if (inputColumnList == null) { + // SQL insert without columnName, nothing to check + } else { + for (int i = 0; i < inputColumnList.size(); i++) { + ColumnSchema columnSchema = inputColumnList.get(i); + TsTableColumnSchema existingColumn = table.getColumnSchema(columnSchema.getName()); + if (existingColumn == null) { + // check arguments for column auto creation + if (columnSchema.getColumnCategory() == null) { + throw new IllegalArgumentException( + "Unknown column category. Cannot auto create column."); + } + if (columnSchema.getType() == null) { + throw new IllegalArgumentException( + "Unknown column data type. Cannot auto create column."); + } + missingColumnList.add(columnSchema); + } else { + // check and validate column data type and category + if (!columnSchema.getType().equals(UnknownType.UNKNOWN) + && !TypeFactory.getType(existingColumn.getDataType()) + .equals(columnSchema.getType())) { + throw new SemanticException( + String.format("Wrong data type at column %s.", columnSchema.getName())); + } + if (columnSchema.getColumnCategory() != null + && !existingColumn.getColumnCategory().equals(columnSchema.getColumnCategory())) { + throw new SemanticException( + String.format("Wrong category at column %s.", columnSchema.getName())); + } + } + } + } + + // auto create missing table or columns + if (table == null) { + autoCreateTable(database, tableSchema, context); + table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); + } else if (inputColumnList == null) { + // do nothing + } else { + if (!missingColumnList.isEmpty()) { + autoCreateColumn(database, tableSchema.getTableName(), missingColumnList, context); + } + } + table + .getColumnList() + .forEach( + o -> + resultColumnList.add( + new ColumnSchema( + o.getColumnName(), + TypeFactory.getType(o.getDataType()), + false, + o.getColumnCategory()))); + return new TableSchema(tableSchema.getTableName(), resultColumnList); + } + + public void autoCreateTable(String database, TableSchema tableSchema, MPPQueryContext context) { + throw new SemanticException(new TableNotExistsException(database, tableSchema.getTableName())); + } + + private void autoCreateColumn( + String database, + String tableName, + List columnSchemaList, + MPPQueryContext context) { + throw new SemanticException( + String.format( + "Unknown columns %s", + columnSchemaList.stream().map(ColumnSchema::getName).collect(Collectors.toList()))); + } + public void validateDeviceSchema( ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) { ValidateResult validateResult = validateDeviceSchemaInCache(schemaValidation); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index 8e9c8fc17c38..a08f0e3545e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -27,6 +27,8 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.ITableDeviceSchemaValidation; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.relational.sql.tree.Expression; @@ -39,6 +41,7 @@ import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.UnknownType; import org.apache.tsfile.write.schema.MeasurementSchema; import java.util.ArrayList; @@ -53,9 +56,9 @@ public class InsertTableStatement extends Statement implements ITableDeviceSchemaValidation { - private final String database; + private final Insert insert; - private final String table; + private final String database; private List deviceIdList; @@ -63,15 +66,17 @@ public class InsertTableStatement extends Statement implements ITableDeviceSchem private List> attributeValueList; - private final InsertRowStatement insertRowStatement; + private InsertRowStatement insertRowStatement; + + private TableSchema tableSchema; public InsertTableStatement(IClientSession clientSession, Insert insert) { - this.database = parseDatabase(clientSession, insert); - this.table = parseTable(insert); - insertRowStatement = parseInsert(insert); + this.insert = insert; + this.database = parseDatabase(clientSession); + this.tableSchema = parseTable(); } - private String parseDatabase(IClientSession clientSession, Insert insert) { + private String parseDatabase(IClientSession clientSession) { String database = clientSession.getDatabaseName(); if (database == null) { database = insert.getTable().getName().getPrefix().get().getSuffix(); @@ -79,13 +84,28 @@ private String parseDatabase(IClientSession clientSession, Insert insert) { return database; } - private String parseTable(Insert insert) { - return insert.getTable().getName().getSuffix().toString(); + private TableSchema parseTable() { + String tableName = insert.getTable().getName().getSuffix().toString(); + List columnSchemaList = new ArrayList<>(); + boolean hasColumn = insert.getColumns().isPresent(); + if (!hasColumn) { + return new TableSchema(tableName, null); + } + int size = insert.getColumns().get().size(); + List columnNameList = insert.getColumns().get(); + for (int i = 0; i < size; i++) { + String columnName = columnNameList.get(i).getValue(); + if (columnName.equalsIgnoreCase("time")) { + continue; + } + columnSchemaList.add(new ColumnSchema(columnName, UnknownType.UNKNOWN, false, null)); + } + return new TableSchema(tableName, columnSchemaList); } - private InsertRowStatement parseInsert(Insert insert) { + private InsertRowStatement parseInsert() { InsertRowStatement insertStatement = new InsertRowStatement(); - String tableName = table; + String tableName = tableSchema.getTableName(); TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName); List values = ((Row) (((Values) (insert.getQuery().getQueryBody())).getRows().get(0))).getItems(); @@ -173,7 +193,18 @@ private InsertRowStatement parseInsert(Insert insert) { return insertStatement; } + public TableSchema getTableSchema() { + return tableSchema; + } + + public void setTableSchema(TableSchema tableSchema) { + this.tableSchema = tableSchema; + } + public InsertRowStatement getInsertRowStatement() { + if (insertRowStatement == null) { + insertRowStatement = parseInsert(); + } return insertRowStatement; } @@ -194,7 +225,7 @@ public String getDatabase() { @Override public String getTableName() { - return table; + return tableSchema.getTableName(); } @Override From 66fc5f46d704c085a86b0348bdfbf5fc53989595 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Sun, 19 May 2024 22:31:14 +0800 Subject: [PATCH 26/36] implement drop table --- .../consensus/request/ConfigPhysicalPlan.java | 12 + .../request/ConfigPhysicalPlanType.java | 3 + .../write/table/CommitDropTablePlan.java | 67 +++ .../request/write/table/PreDropTablePlan.java | 67 +++ .../write/table/RollbackDropTablePlan.java | 67 +++ .../confignode/manager/ConfigManager.java | 10 + .../confignode/manager/ProcedureManager.java | 44 ++ .../manager/schema/ClusterSchemaManager.java | 5 + .../executor/ConfigPlanExecutor.java | 9 + .../persistence/schema/ClusterSchemaInfo.java | 57 +++ .../persistence/schema/ConfigMTree.java | 49 +++ .../impl/schema/table/DropTableProcedure.java | 415 ++++++++++++++++++ .../state/schema/DropTableState.java | 27 ++ .../procedure/store/ProcedureFactory.java | 6 + .../procedure/store/ProcedureType.java | 1 + .../thrift/ConfigNodeRPCServiceProcessor.java | 6 + .../schemaregion/SchemaExecutionVisitor.java | 12 + .../db/protocol/client/ConfigNodeClient.java | 7 + .../impl/DataNodeInternalRPCServiceImpl.java | 73 +++ .../config/TableConfigTaskVisitor.java | 12 +- .../executor/ClusterConfigTaskExecutor.java | 36 ++ .../config/executor/IConfigTaskExecutor.java | 2 + .../metadata/relational/DropTableTask.java | 47 ++ .../plan/planner/plan/node/PlanNodeType.java | 6 +- .../plan/planner/plan/node/PlanVisitor.java | 5 + .../metedata/write/DeleteTableDeviceNode.java | 107 +++++ .../schemaregion/ISchemaRegion.java | 2 + .../impl/SchemaRegionMemoryImpl.java | 5 + .../impl/SchemaRegionPBTreeImpl.java | 5 + .../impl/mem/MTreeBelowSGMemoryImpl.java | 4 + .../table/DataNodeTableCache.java | 34 ++ .../db/schemaengine/table/ITableCache.java | 4 + .../schema/table/TsTableInternalRPCType.java | 12 +- .../src/main/thrift/confignode.thrift | 11 + 34 files changed, 1226 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 1ddd33aca483..7a8da7e962b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -116,8 +116,11 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.sync.ShowPipePlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -408,6 +411,15 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case CommitCreateTable: plan = new CommitCreateTablePlan(); break; + case PreDropTable: + plan = new PreDropTablePlan(); + break; + case RollbackPreDropTable: + plan = new RollbackDropTablePlan(); + break; + case CommitDropTable: + plan = new CommitDropTablePlan(); + break; case GetNodePathsPartition: plan = new GetNodePathsPartitionPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 9b476e12d295..61e3c609722d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -151,6 +151,9 @@ public enum ConfigPhysicalPlanType { PreCreateTable((short) 850), RollbackCreateTable((short) 851), CommitCreateTable((short) 852), + PreDropTable((short) 853), + RollbackPreDropTable((short) 854), + CommitDropTable((short) 855), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java new file mode 100644 index 000000000000..c8e6690718af --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class CommitDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public CommitDropTablePlan() { + super(ConfigPhysicalPlanType.CommitDropTable); + } + + public CommitDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.CommitDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java new file mode 100644 index 000000000000..e8ed113ece5b --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class PreDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public PreDropTablePlan() { + super(ConfigPhysicalPlanType.PreDropTable); + } + + public PreDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.PreDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java new file mode 100644 index 000000000000..9e8e150880e4 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackDropTablePlan.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class RollbackDropTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + public RollbackDropTablePlan() { + super(ConfigPhysicalPlanType.RollbackPreDropTable); + } + + public RollbackDropTablePlan(String database, String tableName) { + super(ConfigPhysicalPlanType.RollbackPreDropTable); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index da2d97cd4d35..f4778bb501da 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -141,6 +141,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -2285,4 +2286,13 @@ public TSStatus createTable(ByteBuffer tableInfo) { return status; } } + + public TSStatus dropTable(TDropTableReq req) { + TSStatus status = confirmLeader(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return procedureManager.dropTable(req.getDatabase(), req.getTableName(), req.getQueryId()); + } else { + return status; + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 1e2e675eda10..c0d4ac7c280a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -77,6 +77,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure; @@ -1259,6 +1260,49 @@ public TSStatus createTable(String database, TsTable table) { } } + public TSStatus dropTable(String database, String tableName, String queryId) { + long procedureId = -1; + synchronized (this) { + boolean hasOverlappedTask = false; + ProcedureType type; + DropTableProcedure dropTableProcedure; + for (Procedure procedure : executor.getProcedures().values()) { + type = ProcedureFactory.getProcedureType(procedure); + if (type == null || !type.equals(ProcedureType.DROP_TABLE_PROCEDURE)) { + continue; + } + dropTableProcedure = (DropTableProcedure) procedure; + if (queryId.equals(dropTableProcedure.getQueryId())) { + procedureId = dropTableProcedure.getProcId(); + break; + } + if (database.equals(dropTableProcedure.getDatabase()) + && tableName.equals(dropTableProcedure.getTableName())) { + hasOverlappedTask = true; + break; + } + } + + if (procedureId == -1) { + if (hasOverlappedTask) { + return RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task dropping table with same name."); + } + procedureId = + this.executor.submitProcedure(new DropTableProcedure(database, tableName, queryId)); + } + } + List procedureStatus = new ArrayList<>(); + boolean isSucceed = + waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus); + if (isSucceed) { + return StatusUtils.OK; + } else { + return procedureStatus.get(0); + } + } + // ====================================================== /* GET-SET Region diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 15ea10a16358..6e8587a54bd6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; @@ -1170,6 +1171,10 @@ public void updateSchemaQuotaConfiguration(long seriesThreshold, long deviceThre schemaQuotaStatistics.setSeriesThreshold(seriesThreshold); } + public TsTable getTable(String database, String tableName) { + return clusterSchemaInfo.getTsTable(database, tableName); + } + public void clearSchemaQuotaCache() { schemaQuotaStatistics.clear(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index f09e7c7c6dc8..e909429970eb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -98,8 +98,11 @@ import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.runtime.TopicHandleMetaChangePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -453,6 +456,12 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.rollbackCreateTable((RollbackCreateTablePlan) physicalPlan); case CommitCreateTable: return clusterSchemaInfo.commitCreateTable((CommitCreateTablePlan) physicalPlan); + case PreDropTable: + return clusterSchemaInfo.preDropTable((PreDropTablePlan) physicalPlan); + case RollbackPreDropTable: + return clusterSchemaInfo.rollbackDropTable((RollbackDropTablePlan) physicalPlan); + case CommitDropTable: + return clusterSchemaInfo.dropTable((CommitDropTablePlan) physicalPlan); case CreatePipeV2: return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 8a92c73fb14b..d921dc5990ce 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -46,8 +46,11 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -1112,6 +1115,60 @@ public Map> getAllPreCreateTables() { } } + public TsTable getTsTable(String database, String tableName) { + databaseReadWriteLock.readLock().lock(); + try { + return mTree.getTable(new PartialPath(new String[] {ROOT, database}), tableName); + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + throw new RuntimeException(e); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + + public TSStatus preDropTable(PreDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.preDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus rollbackDropTable(RollbackDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.rollbackDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus dropTable(CommitDropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.commitDropTable( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + // endregion @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index d031aa984209..6ff82dd49fc8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -705,6 +705,55 @@ public Map> getAllPreCreateTables() throws MetadataExcepti return result; } + public TsTable getTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + return tableNode.getTable(); + } + + public void preDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.USING)) { + throw new IllegalStateException(); + } + tableNode.setStatus(TableNodeStatus.PRE_DELETE); + } + + public void rollbackDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.PRE_DELETE)) { + throw new IllegalStateException(); + } + tableNode.setStatus(TableNodeStatus.USING); + } + + public void commitDropTable(PartialPath database, String tableName) throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.PRE_DELETE)) { + throw new IllegalStateException(); + } + databaseNode.deleteChild(tableName); + } + // endregion // region Serialization and Deserialization diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java new file mode 100644 index 000000000000..1d3ddb81c30a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; +import org.apache.iotdb.confignode.client.DataNodeRequestType; +import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; +import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; +import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; + +public class DropTableProcedure + extends StateMachineProcedure { + + private static final Logger LOGGER = LoggerFactory.getLogger(DropTableProcedure.class); + + private String database; + + private String tableName; + + private String queryId; + + public DropTableProcedure() {} + + public DropTableProcedure(String database, String tableName, String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, DropTableState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case PRE_DROP: + LOGGER.info("Pre drop table {}.{}", database, tableName); + preDropTable(env); + break; + case INVALIDATE_CACHE: + LOGGER.info("Invalidate cache of table {}.{}", database, tableName); + invalidateCache(env); + break; + case DELETE_DATA: + LOGGER.info("Delete data of table {}.{}", database, tableName); + deleteTableData(env); + break; + case DROP_TABLE: + LOGGER.info("Commit drop table {}.{}", database, tableName); + dropTable(env); + return Flow.NO_MORE_STATE; + default: + setFailure(new ProcedureException("Unrecognized DropTableState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "DropTable-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + private void preDropTable(ConfigNodeProcedureEnv env) { + PreDropTablePlan plan = new PreDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setNextState(DropTableState.INVALIDATE_CACHE); + } else { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + private void invalidateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = new TUpdateTableReq(); + req.setType(TsTableInternalRPCType.INVALIDATE_CACHE.getOperationType()); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } catch (IOException ignored) { + // won't happen + } + req.setTableInfo(stream.toByteArray()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to invalidate cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Invalidate table cache failed"))); + return; + } + } + setNextState(DropTableState.DELETE_DATA); + } + + private void deleteTableData(ConfigNodeProcedureEnv env) { + PartialPath tableTsPath = new PartialPath(new String[] {"root", database, tableName, "**"}); + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendFullPath(tableTsPath); + patternTree.constructTree(); + + deleteDataInDataRegions(env, patternTree); + deleteSchemaInSchemaRegions(env, patternTree); + setNextState(DropTableState.DROP_TABLE); + } + + private void deleteDataInDataRegions(ConfigNodeProcedureEnv env, PathPatternTree patternTree) { + Map targetDataRegionGroup = + env.getConfigManager().getRelatedDataRegionGroup(patternTree); + if (targetDataRegionGroup.isEmpty()) { + return; + } + DropTableRegionTaskExecutor deleteDataTask = + new DropTableRegionTaskExecutor<>( + "delete data", + env, + targetDataRegionGroup, + true, + DataNodeRequestType.UPDATE_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TUpdateTableReq( + TsTableInternalRPCType.DELETE_DATA_IN_DATA_REGION.getOperationType(), + serializeDeleteDataReq(database, tableName, targetDataRegionGroup)))); + deleteDataTask.execute(); + } + + private void deleteSchemaInSchemaRegions( + ConfigNodeProcedureEnv env, PathPatternTree patternTree) { + Map targetSchemaRegionGroup = + env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); + if (targetSchemaRegionGroup.isEmpty()) { + return; + } + DropTableRegionTaskExecutor deleteDataTask = + new DropTableRegionTaskExecutor<>( + "delete schema", + env, + targetSchemaRegionGroup, + false, + DataNodeRequestType.UPDATE_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TUpdateTableReq( + TsTableInternalRPCType.DELETE_SCHEMA_IN_SCHEMA_REGION.getOperationType(), + serializeDeleteDataReq(database, tableName, targetSchemaRegionGroup)))); + deleteDataTask.execute(); + } + + private ByteBuffer serializeDeleteDataReq( + String database, + String tableName, + Map targetRegionMap) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(targetRegionMap.size(), stream); + for (TConsensusGroupId id : targetRegionMap.keySet()) { + ReadWriteIOUtils.write(id.getId(), stream); + } + } catch (IOException ignored) { + // won't happen + } + return ByteBuffer.wrap(stream.toByteArray()); + } + + private void dropTable(ConfigNodeProcedureEnv env) { + CommitDropTablePlan plan = new CommitDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + @Override + protected void rollbackState(ConfigNodeProcedureEnv env, DropTableState state) + throws IOException, InterruptedException, ProcedureException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case PRE_DROP: + rollbackPreDrop(env); + break; + case INVALIDATE_CACHE: + rollbackInvalidateCache(env); + break; + } + } finally { + LOGGER.info( + "Rollback DropTable-{} costs {}ms.", state, (System.currentTimeMillis() - startTime)); + } + } + + private void rollbackInvalidateCache(ConfigNodeProcedureEnv env) { + TsTable table = env.getConfigManager().getClusterSchemaManager().getTable(database, tableName); + TSStatus status = env.getConfigManager().getProcedureManager().createTable(database, table); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode()) { + return; + } + LOGGER.warn( + "Error occurred during rollback table cache for DropTable {}.{}: {}", + database, + tableName, + status); + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + + private void rollbackPreDrop(ConfigNodeProcedureEnv env) { + RollbackDropTablePlan plan = new RollbackDropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + @Override + protected DropTableState getState(int stateId) { + return DropTableState.values()[stateId]; + } + + @Override + protected int getStateId(DropTableState dropTableState) { + return dropTableState.ordinal(); + } + + @Override + protected DropTableState getInitialState() { + return DropTableState.PRE_DROP; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getQueryId() { + return queryId; + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.DROP_TABLE_PROCEDURE.getTypeCode()); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DropTableProcedure)) return false; + DropTableProcedure that = (DropTableProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } + + private class DropTableRegionTaskExecutor extends DataNodeRegionTaskExecutor { + + private final String taskName; + + DropTableRegionTaskExecutor( + String taskName, + ConfigNodeProcedureEnv env, + Map targetSchemaRegionGroup, + boolean executeOnAllReplicaset, + DataNodeRequestType dataNodeRequestType, + BiFunction, Q> dataNodeRequestGenerator) { + super( + env, + targetSchemaRegionGroup, + executeOnAllReplicaset, + dataNodeRequestType, + dataNodeRequestGenerator); + this.taskName = taskName; + } + + @Override + protected List processResponseOfOneDataNode( + TDataNodeLocation dataNodeLocation, + List consensusGroupIdList, + TSStatus response) { + List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return failedRegionList; + } + + if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + List subStatus = response.getSubStatus(); + for (int i = 0; i < subStatus.size(); i++) { + if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } + } else { + failedRegionList.addAll(consensusGroupIdList); + } + return failedRegionList; + } + + @Override + protected void onAllReplicasetFailure( + TConsensusGroupId consensusGroupId, Set dataNodeLocationSet) { + setFailure( + new ProcedureException( + new MetadataException( + String.format( + "Drop table %s.%s failed when [%s] because all replicaset of schemaRegion %s failed. %s", + database, tableName, taskName, consensusGroupId.id, dataNodeLocationSet)))); + interruptTask(); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java new file mode 100644 index 000000000000..941139746367 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum DropTableState { + PRE_DROP, + INVALIDATE_CACHE, + DELETE_DATA, + DROP_TABLE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index 394a76a802d7..bc2b68314bfc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -46,6 +46,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure; @@ -181,6 +182,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case CREATE_TABLE_PROCEDURE: procedure = new CreateTableProcedure(); break; + case DROP_TABLE_PROCEDURE: + procedure = new DropTableProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; @@ -308,6 +312,8 @@ public static ProcedureType getProcedureType(Procedure procedure) { return ProcedureType.UNSET_TEMPLATE_PROCEDURE; } else if (procedure instanceof CreateTableProcedure) { return ProcedureType.CREATE_TABLE_PROCEDURE; + } else if (procedure instanceof DropTableProcedure) { + return ProcedureType.DROP_TABLE_PROCEDURE; } else if (procedure instanceof CreatePipePluginProcedure) { return ProcedureType.CREATE_PIPE_PLUGIN_PROCEDURE; } else if (procedure instanceof DropPipePluginProcedure) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 39a92287cdf1..06ba6dd5c9b5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -63,6 +63,7 @@ public enum ProcedureType { SET_TEMPLATE_PROCEDURE((short) 702), CREATE_TABLE_PROCEDURE((short) 750), + DROP_TABLE_PROCEDURE((short) 751), // ProcedureId 800-899 is used by IoTDB-Ml diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index e88ab55d144d..cd9573a0a026 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -108,6 +108,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -1114,4 +1115,9 @@ public TThrottleQuotaResp getThrottleQuota() { public TSStatus createTable(ByteBuffer tableInfo) throws TException { return configManager.createTable(tableInfo); } + + @Override + public TSStatus dropTable(TDropTableReq req) throws TException { + return configManager.dropTable(req); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 42941c251a00..c67cef061019 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -538,6 +539,17 @@ public TSStatus visitCreateTableDevice(CreateTableDeviceNode node, ISchemaRegion } } + @Override + public TSStatus visitDeleteTableDevice(DeleteTableDeviceNode node, ISchemaRegion schemaRegion) { + try { + schemaRegion.deleteTableDevice(node.getTable()); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( PipeEnrichedWritePlanNode node, ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index f31728aa1b86..945e506c0c70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -77,6 +77,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; @@ -1073,6 +1074,12 @@ public TSStatus createTable(ByteBuffer tableInfo) throws TException { () -> client.createTable(tableInfo), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus dropTable(TDropTableReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.dropTable(req), status -> !updateConfigNodeLeader(status)); + } + public static class Factory extends ThriftClientFactory { public Factory( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index b74a7def880e..5d009397979a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.protocol.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; @@ -107,6 +108,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.PreDeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.RollbackPreDeactivateTemplateNode; @@ -267,6 +269,7 @@ import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -493,7 +496,15 @@ public TSStatus invalidateSchemaCache(TInvalidateCacheReq req) { // req.getFullPath() is a database path DataNodeSchemaCache.getInstance().invalidate(req.getFullPath()); ClusterTemplateManager.getInstance().invalid(req.getFullPath()); + if (req.isStorageGroup()) { + PartialPath path = new PartialPath(req.getFullPath()); + String[] nodes = path.getNodes(); + DataNodeTableCache.getInstance().invalidateTable(nodes[1]); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } catch (IllegalPathException e) { + throw new RuntimeException(e); } finally { DataNodeSchemaCache.getInstance().releaseWriteLock(); } @@ -1416,6 +1427,9 @@ public TFetchFragmentInstanceStatisticsResp fetchFragmentInstanceStatistics( @Override public TSStatus updateTable(TUpdateTableReq req) throws TException { + String database; + String tableName; + List regionIdList; switch (TsTableInternalRPCType.getType(req.type)) { case PRE_CREATE: DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.TIMESERIES_VS_TABLE); @@ -1440,6 +1454,48 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo)); break; + case INVALIDATE_CACHE: + DataNodeTableCache.getInstance() + .invalidateTable( + ReadWriteIOUtils.readString(req.tableInfo), + ReadWriteIOUtils.readString(req.tableInfo)); + break; + case DELETE_DATA_IN_DATA_REGION: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); + regionIdList = deserializeRegionList(req.tableInfo, true); + return executeInternalSchemaTask( + regionIdList, + consensusGroupId -> { + RegionWriteExecutor executor = new RegionWriteExecutor(); + return executor + .execute( + new DataRegionId(consensusGroupId.getId()), + new DeleteDataNode( + new PlanNodeId(""), + Collections.singletonList( + new PartialPath( + new String[] { + PATH_ROOT, database, tableName, MULTI_LEVEL_PATH_WILDCARD + })), + Long.MIN_VALUE, + Long.MAX_VALUE)) + .getStatus(); + }); + case DELETE_SCHEMA_IN_SCHEMA_REGION: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); + regionIdList = deserializeRegionList(req.tableInfo, false); + return executeInternalSchemaTask( + regionIdList, + consensusGroupId -> { + RegionWriteExecutor executor = new RegionWriteExecutor(); + return executor + .execute( + new SchemaRegionId(consensusGroupId.getId()), + new DeleteTableDeviceNode(new PlanNodeId(""), tableName)) + .getStatus(); + }); default: LOGGER.warn("Unsupported type {} when updating table", req.type); return RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER); @@ -1447,6 +1503,23 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + private List deserializeRegionList(ByteBuffer byteBuffer, boolean isData) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + List regionIdList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + if (isData) { + regionIdList.add( + new TConsensusGroupId( + TConsensusGroupType.DataRegion, ReadWriteIOUtils.readInt(byteBuffer))); + } else { + regionIdList.add( + new TConsensusGroupId( + TConsensusGroupType.SchemaRegion, ReadWriteIOUtils.readInt(byteBuffer))); + } + } + return regionIdList; + } + private PathPatternTree filterPathPatternTree(PathPatternTree patternTree, String storageGroup) { PathPatternTree filteredPatternTree = new PathPatternTree(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index b005c08c5c69..9cd116f13342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; @@ -151,7 +152,16 @@ private TSDataType getDataType(DataType dataType) { @Override protected IConfigTask visitDropTable(DropTable node, MPPQueryContext context) { - return super.visitDropTable(node, context); + context.setQueryType(QueryType.WRITE); + String database = clientSession.getDatabaseName(); + if (node.getTableName().getPrefix().isPresent()) { + database = node.getTableName().getPrefix().get().toString(); + } + if (database == null) { + throw new SemanticException("unknown database"); + } + return new DropTableTask( + database, node.getTableName().getSuffix(), context.getQueryId().toString()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 534eb794ed87..efa5c89151eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -71,6 +71,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp; @@ -2856,6 +2857,41 @@ public SettableFuture showTables(String database) { return future; } + @Override + public SettableFuture dropTable( + String database, String tableName, String queryId) { + final SettableFuture future = SettableFuture.create(); + try (ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + do { + try { + tsStatus = client.dropTable(new TDropTableReq(database, tableName, queryId)); + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + LOGGER.warn( + "Failed to execute drop table {}.{}, status is {}.", database, tableName, tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + public void handlePipeConfigClientExit(String clientId) { try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index b5f8ce9c54f7..cdd1d45314a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -261,4 +261,6 @@ SettableFuture showThrottleQuota( SettableFuture describeTable(String database, String tableName); SettableFuture showTables(String database); + + SettableFuture dropTable(String database, String tableName, String queryId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java new file mode 100644 index 000000000000..80dc871180d6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class DropTableTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final String queryId; + + public DropTableTask(String database, String tableName, String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.dropTable(database, tableName, queryId); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 8e6ef9e93832..1152d9b0c6e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -47,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -216,7 +217,8 @@ public enum PlanNodeType { CREATE_TABLE_DEVICE((short) 92), TABLE_DEVICE_SCAN((short) 93), - TABLE_DEVICE_FETCH((short) 94); + TABLE_DEVICE_FETCH((short) 94), + DELETE_TABLE_DEVICE((short) 95); public static final int BYTES = Short.BYTES; @@ -453,6 +455,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TableDeviceScanNode.deserialize(buffer); case 94: return TableDeviceFetchNode.deserialize(buffer); + case 95: + return DeleteTableDeviceNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 0aa1c0a6c3d3..03b28177027b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.CreateTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeactivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalBatchActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.InternalCreateMultiTimeSeriesNode; @@ -568,4 +569,8 @@ public R visitTableDeviceScan(TableDeviceScanNode node, C context) { public R visitTableDeviceFetch(TableDeviceFetchNode node, C context) { return visitPlan(node, context); } + + public R visitDeleteTableDevice(DeleteTableDeviceNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java new file mode 100644 index 000000000000..0c54b9bb0a1b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/DeleteTableDeviceNode.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +public class DeleteTableDeviceNode extends PlanNode { + + private final String table; + + public DeleteTableDeviceNode(PlanNodeId id, String table) { + super(id); + this.table = table; + } + + public String getTable() { + return table; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitDeleteTableDevice(this, context); + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return new DeleteTableDeviceNode(getPlanNodeId(), table); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.DELETE_TABLE_DEVICE.serialize(byteBuffer); + ReadWriteIOUtils.write(table, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.DELETE_TABLE_DEVICE.serialize(stream); + ReadWriteIOUtils.write(table, stream); + } + + public static DeleteTableDeviceNode deserialize(ByteBuffer buffer) { + String table = ReadWriteIOUtils.readString(buffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new DeleteTableDeviceNode(planNodeId, table); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeleteTableDeviceNode)) return false; + if (!super.equals(o)) return false; + DeleteTableDeviceNode that = (DeleteTableDeviceNode) o; + return Objects.equals(table, that.table); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), table); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index a3d8f0dfd3f2..2b0e5d85a8d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -302,6 +302,8 @@ void createTableDevice( List> attributeValueList) throws MetadataException; + void deleteTableDevice(String table) throws MetadataException; + // endregion // region Interfaces for SchemaReader diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 7e1174aa764c..ad7eedb237f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1250,6 +1250,11 @@ public void createTableDevice( } } + @Override + public void deleteTableDevice(String table) throws MetadataException { + mtree.deleteTableDevice(table); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 89cdba5b2ee3..d54d4c194fc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1323,6 +1323,11 @@ public void createTableDevice( throw new UnsupportedOperationException(); } + @Override + public void deleteTableDevice(String table) throws MetadataException { + throw new UnsupportedOperationException(); + } + @Override public ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) throws MetadataException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 297a3fb49f97..dec75300cb0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1371,5 +1371,9 @@ public void createTableDevice( } } + public void deleteTableDevice(String tableName) { + storageGroupMNode.deleteChild(tableName); + } + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 4b92c961c801..bce098411a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -146,6 +146,40 @@ public void commitCreateTable(String database, String tableName) { } } + @Override + public void invalidateTable(String database, String tableName) { + readWriteLock.writeLock().lock(); + try { + databaseTableMap.compute( + database, + (db, map) -> { + if (map == null) { + return null; + } + map.remove(tableName); + if (map.isEmpty()) { + return null; + } else { + return map; + } + }); + LOGGER.info("Invalidate table {}.{} successfully", database, tableName); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void invalidateTable(String database) { + readWriteLock.writeLock().lock(); + try { + databaseTableMap.remove(database); + preCreateTableMap.remove(database); + } finally { + readWriteLock.writeLock().unlock(); + } + } + public TsTable getTable(String database, String tableName) { readWriteLock.readLock().lock(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index 16d263004734..47a58118df26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -30,4 +30,8 @@ public interface ITableCache { void rollbackCreateTable(String database, String tableName); void commitCreateTable(String database, String tableName); + + void invalidateTable(String database, String tableName); + + void invalidateTable(String database); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java index 06e08bb41bec..9fcce8781f62 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java @@ -28,7 +28,11 @@ public enum TsTableInternalRPCType { PRE_CREATE((byte) 0), ROLLBACK_CREATE((byte) 1), - COMMIT_CREATE((byte) 2); + COMMIT_CREATE((byte) 2), + + INVALIDATE_CACHE((byte) 3), + DELETE_DATA_IN_DATA_REGION((byte) 4), + DELETE_SCHEMA_IN_SCHEMA_REGION((byte) 5); private final byte operationType; @@ -57,6 +61,12 @@ public static TsTableInternalRPCType getType(byte type) { return ROLLBACK_CREATE; case 2: return COMMIT_CREATE; + case 3: + return INVALIDATE_CACHE; + case 4: + return DELETE_DATA_IN_DATA_REGION; + case 5: + return DELETE_SCHEMA_IN_SCHEMA_REGION; default: throw new IllegalArgumentException("Unknown table update operation type" + type); } diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index b14f67cef03d..0fb148b8737c 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -919,6 +919,15 @@ enum TTestOperation { TEST_SUB_PROCEDURE, } +// ==================================================== +// Table +// ==================================================== +struct TDropTableReq{ + 1: required string database + 2: required string tableName + 3: required string queryId +} + service IConfigNodeRPCService { // ====================================================== @@ -1565,5 +1574,7 @@ service IConfigNodeRPCService { // ====================================================== common.TSStatus createTable(binary tableInfo) + + common.TSStatus dropTable(TDropTableReq req) } From 5564225cb829a852b0d735e4d913753f5dfda48c Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Mon, 20 May 2024 09:33:53 +0800 Subject: [PATCH 27/36] fix OrFilter --- .../org/apache/iotdb/commons/schema/filter/impl/OrFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java index 1f57622719d7..e740c23b2c97 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/OrFilter.java @@ -57,7 +57,7 @@ public boolean accept(SchemaFilterVisitor visitor, C node) { @Override public SchemaFilterType getSchemaFilterType() { - return SchemaFilterType.AND; + return SchemaFilterType.OR; } @Override From 1edcc41bc9d7803b418639aa7405dc3d9e32a13f Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Mon, 20 May 2024 11:55:32 +0800 Subject: [PATCH 28/36] add table column --- .../consensus/request/ConfigPhysicalPlan.java | 4 + .../request/ConfigPhysicalPlanType.java | 1 + .../write/table/AddTableColumnPlan.java | 93 ++++++ .../confignode/manager/ConfigManager.java | 16 + .../confignode/manager/ProcedureManager.java | 54 ++++ .../manager/schema/ClusterSchemaManager.java | 51 ++++ .../executor/ConfigPlanExecutor.java | 3 + .../persistence/schema/ClusterSchemaInfo.java | 24 ++ .../persistence/schema/ConfigMTree.java | 27 ++ .../schema/table/AddTableColumnProcedure.java | 283 ++++++++++++++++++ .../state/schema/AddTableColumnState.java | 25 ++ .../procedure/store/ProcedureFactory.java | 6 + .../procedure/store/ProcedureType.java | 1 + .../thrift/ConfigNodeRPCServiceProcessor.java | 6 + .../db/protocol/client/ConfigNodeClient.java | 7 + .../impl/DataNodeInternalRPCServiceImpl.java | 15 + .../executor/ClusterConfigTaskExecutor.java | 49 +++ .../config/executor/IConfigTaskExecutor.java | 9 + .../relational/AlterTableAddColumnTask.java | 96 ++++++ .../schema/TableModelSchemaFetcher.java | 33 +- .../table/DataNodeTableCache.java | 25 ++ .../db/schemaengine/table/ITableCache.java | 9 + .../schema/table/AlterTableOperationType.java | 43 +++ .../iotdb/commons/schema/table/TsTable.java | 4 + .../schema/table/TsTableInternalRPCType.java | 9 +- .../table/column/TsTableColumnSchemaUtil.java | 37 +++ .../src/main/thrift/confignode.thrift | 10 + 27 files changed, 934 insertions(+), 6 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 7a8da7e962b4..fc36d0b99e48 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -115,6 +115,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan; import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.sync.ShowPipePlanV1; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; @@ -420,6 +421,9 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case CommitDropTable: plan = new CommitDropTablePlan(); break; + case AddTableColumn: + plan = new AddTableColumnPlan(); + break; case GetNodePathsPartition: plan = new GetNodePathsPartitionPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 61e3c609722d..b3b54c34446e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -154,6 +154,7 @@ public enum ConfigPhysicalPlanType { PreDropTable((short) 853), RollbackPreDropTable((short) 854), CommitDropTable((short) 855), + AddTableColumn((short) 856), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java new file mode 100644 index 000000000000..eda7a26af74e --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class AddTableColumnPlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + private List columnSchemaList; + + private boolean isRollback; + + public AddTableColumnPlan() { + super(ConfigPhysicalPlanType.AddTableColumn); + } + + public AddTableColumnPlan( + String database, + String tableName, + List columnSchemaList, + boolean isRollback) { + super(ConfigPhysicalPlanType.AddTableColumn); + this.database = database; + this.tableName = tableName; + this.columnSchemaList = columnSchemaList; + this.isRollback = isRollback; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public List getColumnSchemaList() { + return columnSchemaList; + } + + public boolean isRollback() { + return isRollback; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + TsTableColumnSchemaUtil.serialize(columnSchemaList, stream); + ReadWriteIOUtils.write(isRollback, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + this.columnSchemaList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(buffer); + this.isRollback = ReadWriteIOUtils.readBool(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f4778bb501da..d840690031de 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -44,6 +44,7 @@ import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.AlterTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.service.metric.MetricService; @@ -116,6 +117,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; @@ -2295,4 +2297,18 @@ public TSStatus dropTable(TDropTableReq req) { return status; } } + + public TSStatus alterTable(TAlterTableReq req) { + TSStatus status = confirmLeader(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + switch (AlterTableOperationType.getType(req.operationType)) { + case ADD_COLUMN: + return procedureManager.alterTableAddColumn(req); + default: + throw new IllegalArgumentException(); + } + } else { + return status; + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index c0d4ac7c280a..37c4e4ca4697 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -34,6 +34,8 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.trigger.TriggerInformation; @@ -76,6 +78,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; @@ -99,6 +102,7 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -1303,6 +1307,56 @@ public TSStatus dropTable(String database, String tableName, String queryId) { } } + public TSStatus alterTableAddColumn(TAlterTableReq req) { + String database = req.database; + String tableName = req.tableName; + String queryId = req.queryId; + List columnSchemaList = + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.updateInfo); + + long procedureId = -1; + synchronized (this) { + boolean hasOverlappedTask = false; + ProcedureType type; + AddTableColumnProcedure addTableColumnProcedure; + for (Procedure procedure : executor.getProcedures().values()) { + type = ProcedureFactory.getProcedureType(procedure); + if (type == null || !type.equals(ProcedureType.ADD_TABLE_COLUMN_PROCEDURE)) { + continue; + } + addTableColumnProcedure = (AddTableColumnProcedure) procedure; + if (queryId.equals(addTableColumnProcedure.getQueryId())) { + procedureId = addTableColumnProcedure.getProcId(); + break; + } + if (database.equals(addTableColumnProcedure.getDatabase()) + && tableName.equals(addTableColumnProcedure.getTableName())) { + hasOverlappedTask = true; + break; + } + } + + if (procedureId == -1) { + if (hasOverlappedTask) { + return RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task dropping table with same name."); + } + procedureId = + this.executor.submitProcedure( + new AddTableColumnProcedure(database, tableName, queryId, columnSchemaList)); + } + } + List procedureStatus = new ArrayList<>(); + boolean isSucceed = + waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus); + if (isSucceed) { + return StatusUtils.OK; + } else { + return procedureStatus.get(0); + } + } + // ====================================================== /* GET-SET Region diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 6e8587a54bd6..23f232cb4dfc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -53,6 +54,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; @@ -1175,6 +1177,55 @@ public TsTable getTable(String database, String tableName) { return clusterSchemaInfo.getTsTable(database, tableName); } + public synchronized Pair> addTableColumn( + String database, String tableName, List columnSchemaList) { + Map> currentUsingTable = clusterSchemaInfo.getAllUsingTables(); + TsTable targetTable = null; + for (TsTable table : currentUsingTable.get(database)) { + if (table.getTableName().equals(tableName)) { + targetTable = table; + break; + } + } + + if (targetTable == null) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.TABLE_NOT_EXISTS, + String.format("Table %s.%s not exist", database, tableName)), + null); + } + + List copiedList = new ArrayList<>(); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + if (targetTable.getColumnSchema(columnSchema.getColumnName()) == null) { + copiedList.add(columnSchema); + } + } + + AddTableColumnPlan addTableColumnPlan = + new AddTableColumnPlan(database, tableName, copiedList, false); + try { + return new Pair<>(getConsensusManager().write(addTableColumnPlan), copiedList); + } catch (ConsensusException e) { + LOGGER.warn(e.getMessage(), e); + return new Pair<>( + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()), null); + } + } + + public synchronized TSStatus rollbackAddTableColumn( + String database, String tableName, List columnSchemaList) { + AddTableColumnPlan addTableColumnPlan = + new AddTableColumnPlan(database, tableName, columnSchemaList, true); + try { + return getConsensusManager().write(addTableColumnPlan); + } catch (ConsensusException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + public void clearSchemaQuotaCache() { schemaQuotaStatistics.clear(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index e909429970eb..a226f1c9bf2e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -97,6 +97,7 @@ import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.runtime.TopicHandleMetaChangePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; @@ -462,6 +463,8 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.rollbackDropTable((RollbackDropTablePlan) physicalPlan); case CommitDropTable: return clusterSchemaInfo.dropTable((CommitDropTablePlan) physicalPlan); + case AddTableColumn: + return clusterSchemaInfo.addTableColumn((AddTableColumnPlan) physicalPlan); case CreatePipeV2: return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index d921dc5990ce..eb0fa53e02d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -45,6 +45,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; @@ -1169,6 +1170,29 @@ public TSStatus dropTable(CommitDropTablePlan plan) { } } + public TSStatus addTableColumn(AddTableColumnPlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + if (plan.isRollback()) { + mTree.rollbackAddTableColumn( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), + plan.getTableName(), + plan.getColumnSchemaList()); + } else { + mTree.addTableColumn( + new PartialPath(new String[] {ROOT, plan.getDatabase()}), + plan.getTableName(), + plan.getColumnSchemaList()); + } + return RpcUtils.SUCCESS_STATUS; + } catch (MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + // endregion @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 6ff82dd49fc8..6252fffe10cb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.factory.ConfigMNodeFactory; @@ -754,6 +755,32 @@ public void commitDropTable(PartialPath database, String tableName) throws Metad databaseNode.deleteChild(tableName); } + public void addTableColumn( + PartialPath database, String tableName, List columnSchemaList) + throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + TsTable table = tableNode.getTable(); + columnSchemaList.forEach(table::addColumnSchema); + } + + public void rollbackAddTableColumn( + PartialPath database, String tableName, List columnSchemaList) + throws MetadataException { + IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + TsTable table = tableNode.getTable(); + columnSchemaList.forEach(o -> table.removeColumnSchema(o.getColumnName())); + } + // endregion // region Serialization and Deserialization diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java new file mode 100644 index 000000000000..201e8307dfd8 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.client.DataNodeRequestType; +import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; +import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class AddTableColumnProcedure + extends StateMachineProcedure { + + private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); + + private String database; + + private String tableName; + + private String queryId; + + private List inputColumnList; + + private List actualAddedColumnList; + + public AddTableColumnProcedure() {} + + public AddTableColumnProcedure( + String database, + String tableName, + String queryId, + List inputColumnList) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + this.inputColumnList = inputColumnList; + } + + @Override + protected Flow executeFromState(ConfigNodeProcedureEnv env, AddTableColumnState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case ADD_COLUMN: + LOGGER.info("Add column to table {}.{}", database, tableName); + addColumn(env); + break; + case UPDATE_CACHE: + LOGGER.info("Update cache of table {}.{} when adding column", database, tableName); + updateCache(env); + return Flow.NO_MORE_STATE; + default: + setFailure(new ProcedureException("Unrecognized AddTableColumnState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "AddTableColumn-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + private void addColumn(ConfigNodeProcedureEnv env) { + Pair> result = + env.getConfigManager() + .getClusterSchemaManager() + .addTableColumn(database, tableName, inputColumnList); + TSStatus status = result.getLeft(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + return; + } + actualAddedColumnList = result.getRight(); + setNextState(AddTableColumnState.UPDATE_CACHE); + } + + private void updateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = + new TUpdateTableReq( + TsTableInternalRPCType.ADD_COLUMN.getOperationType(), getCacheRequestInfo()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to update cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Update table cache failed"))); + return; + } + } + } + + @Override + protected void rollbackState(ConfigNodeProcedureEnv env, AddTableColumnState state) + throws IOException, InterruptedException, ProcedureException { + long startTime = System.currentTimeMillis(); + try { + switch (state) { + case ADD_COLUMN: + rollbackAddColumn(env); + break; + case UPDATE_CACHE: + rollbackUpdateCache(env); + break; + } + } finally { + LOGGER.info( + "Rollback DropTable-{} costs {}ms.", state, (System.currentTimeMillis() - startTime)); + } + } + + private void rollbackAddColumn(ConfigNodeProcedureEnv env) { + if (actualAddedColumnList == null) { + return; + } + TSStatus status = + env.getConfigManager() + .getClusterSchemaManager() + .rollbackAddTableColumn(database, tableName, actualAddedColumnList); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + + private void rollbackUpdateCache(ConfigNodeProcedureEnv env) { + Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + + TUpdateTableReq req = + new TUpdateTableReq( + TsTableInternalRPCType.ROLLBACK_ADD_COLUMN.getOperationType(), getCacheRequestInfo()); + + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.UPDATE_TABLE, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + Map statusMap = clientHandler.getResponseMap(); + for (TSStatus status : statusMap.values()) { + // all dataNodes must clear the related schema cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to rollback cache of table {}.{}", database, tableName); + setFailure(new ProcedureException(new MetadataException("Rollback table cache failed"))); + return; + } + } + } + + private ByteBuffer getCacheRequestInfo() { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + + TsTableColumnSchemaUtil.serialize(actualAddedColumnList, stream); + } catch (IOException ignored) { + // won't happen + } + return ByteBuffer.wrap(stream.toByteArray()); + } + + @Override + protected AddTableColumnState getState(int stateId) { + return AddTableColumnState.values()[stateId]; + } + + @Override + protected int getStateId(AddTableColumnState state) { + return state.ordinal(); + } + + @Override + protected AddTableColumnState getInitialState() { + return AddTableColumnState.ADD_COLUMN; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getQueryId() { + return queryId; + } + + public List getInputColumnList() { + return inputColumnList; + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.ADD_TABLE_COLUMN_PROCEDURE.getTypeCode()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + + TsTableColumnSchemaUtil.serialize(inputColumnList, stream); + TsTableColumnSchemaUtil.serialize(actualAddedColumnList, stream); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + + this.inputColumnList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(byteBuffer); + this.actualAddedColumnList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(byteBuffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AddTableColumnProcedure)) return false; + AddTableColumnProcedure that = (AddTableColumnProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java new file mode 100644 index 000000000000..d819e73aa5d0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AddTableColumnState.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum AddTableColumnState { + ADD_COLUMN, + UPDATE_CACHE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index bc2b68314bfc..99dd350d05a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -45,6 +45,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; @@ -185,6 +186,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case DROP_TABLE_PROCEDURE: procedure = new DropTableProcedure(); break; + case ADD_TABLE_COLUMN_PROCEDURE: + procedure = new AddTableColumnProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; @@ -314,6 +318,8 @@ public static ProcedureType getProcedureType(Procedure procedure) { return ProcedureType.CREATE_TABLE_PROCEDURE; } else if (procedure instanceof DropTableProcedure) { return ProcedureType.DROP_TABLE_PROCEDURE; + } else if (procedure instanceof AddTableColumnProcedure) { + return ProcedureType.ADD_TABLE_COLUMN_PROCEDURE; } else if (procedure instanceof CreatePipePluginProcedure) { return ProcedureType.CREATE_PIPE_PLUGIN_PROCEDURE; } else if (procedure instanceof DropPipePluginProcedure) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 06ba6dd5c9b5..f2c4cea6089a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -64,6 +64,7 @@ public enum ProcedureType { CREATE_TABLE_PROCEDURE((short) 750), DROP_TABLE_PROCEDURE((short) 751), + ADD_TABLE_COLUMN_PROCEDURE((short) 752), // ProcedureId 800-899 is used by IoTDB-Ml diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index cd9573a0a026..deeda0835579 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -69,6 +69,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -1120,4 +1121,9 @@ public TSStatus createTable(ByteBuffer tableInfo) throws TException { public TSStatus dropTable(TDropTableReq req) throws TException { return configManager.dropTable(req); } + + @Override + public TSStatus alterTable(TAlterTableReq req) throws TException { + return configManager.alterTable(req); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 945e506c0c70..e20b26f615ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -38,6 +38,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -1080,6 +1081,12 @@ public TSStatus dropTable(TDropTableReq req) throws TException { () -> client.dropTable(req), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus alterTable(TAlterTableReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.alterTable(req), status -> !updateConfigNodeLeader(status)); + } + public static class Factory extends ThriftClientFactory { public Factory( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 5d009397979a..31bd9b7fe87a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -49,6 +49,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -1496,6 +1497,20 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { new DeleteTableDeviceNode(new PlanNodeId(""), tableName)) .getStatus(); }); + case ADD_COLUMN: + DataNodeTableCache.getInstance() + .addTableColumn( + ReadWriteIOUtils.readString(req.tableInfo), + ReadWriteIOUtils.readString(req.tableInfo), + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.tableInfo)); + break; + case ROLLBACK_ADD_COLUMN: + DataNodeTableCache.getInstance() + .rollbackAddColumn( + ReadWriteIOUtils.readString(req.tableInfo), + ReadWriteIOUtils.readString(req.tableInfo), + TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.tableInfo)); + break; default: LOGGER.warn("Unsupported type {} when updating table", req.type); return RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index efa5c89151eb..732ee3c8f92d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -42,8 +42,11 @@ import org.apache.iotdb.commons.pipe.plugin.service.PipePluginClassLoader; import org.apache.iotdb.commons.pipe.plugin.service.PipePluginExecutableManager; import org.apache.iotdb.commons.pipe.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.schema.table.AlterTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.trigger.service.TriggerExecutableManager; @@ -53,6 +56,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -2892,6 +2896,51 @@ public SettableFuture dropTable( return future; } + @Override + public SettableFuture alterTableAddColumn( + String database, + String tableName, + List columnSchemaList, + String queryId) { + final SettableFuture future = SettableFuture.create(); + try (ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + TAlterTableReq req = new TAlterTableReq(); + req.setDatabase(database); + req.setTableName(tableName); + req.setQueryId(queryId); + req.setOperationType(AlterTableOperationType.ADD_COLUMN.getTypeValue()); + req.setUpdateInfo(TsTableColumnSchemaUtil.serialize(columnSchemaList)); + + do { + try { + tsStatus = client.alterTable(req); + } catch (TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + LOGGER.warn( + "Failed to add column to table {}.{}, status is {}.", database, tableName, tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + public void handlePipeConfigClientExit(String clientId) { try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index cdd1d45314a3..7406331d2d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.db.protocol.session.IClientSession; @@ -84,6 +85,8 @@ import com.google.common.util.concurrent.SettableFuture; +import java.util.List; + public interface IConfigTaskExecutor { SettableFuture setDatabase(DatabaseSchemaStatement databaseSchemaStatement); @@ -263,4 +266,10 @@ SettableFuture showThrottleQuota( SettableFuture showTables(String database); SettableFuture dropTable(String database, String tableName, String queryId); + + SettableFuture alterTableAddColumn( + String database, + String tableName, + List columnSchemaList, + String queryId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java new file mode 100644 index 000000000000..febd54cc8a7a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.IdColumnSchema; +import org.apache.iotdb.commons.schema.table.column.MeasurementColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.BinaryType; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class AlterTableAddColumnTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final List columnList; + + private final String queryId; + + public AlterTableAddColumnTask( + String database, String tableName, List columnList, String queryId) { + this.database = database; + this.tableName = tableName; + this.columnList = parseInputColumnSchema(columnList); + this.queryId = queryId; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterTableAddColumn(database, tableName, columnList, queryId); + } + + private List parseInputColumnSchema(List inputColumnList) { + List columnSchemaList = new ArrayList<>(inputColumnList.size()); + for (ColumnSchema inputColumn : inputColumnList) { + switch (inputColumn.getColumnCategory()) { + case ID: + if (!inputColumn.getType().equals(BinaryType.TEXT)) { + throw new SemanticException("Id column only support data type TEXT."); + } + columnSchemaList.add(new IdColumnSchema(inputColumn.getName(), TSDataType.TEXT)); + break; + case ATTRIBUTE: + if (!inputColumn.getType().equals(BinaryType.TEXT)) { + throw new SemanticException("Attribute column only support data type TEXT."); + } + columnSchemaList.add(new AttributeColumnSchema(inputColumn.getName(), TSDataType.TEXT)); + break; + case MEASUREMENT: + TSDataType dataType = InternalTypeManager.getTSDataType(inputColumn.getType()); + columnSchemaList.add( + new MeasurementColumnSchema( + inputColumn.getName(), + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor())); + break; + } + } + return columnSchemaList; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 762fdb7ba375..6e2108c2ea4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -39,6 +39,9 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertSchemaPredicateToFilterVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceId; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; @@ -52,6 +55,7 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; @@ -59,6 +63,8 @@ import org.apache.tsfile.read.common.type.TypeFactory; import org.apache.tsfile.read.common.type.UnknownType; import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -66,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -73,12 +80,17 @@ public class TableModelSchemaFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(TableModelSchemaFetcher.class); + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private final Coordinator coordinator = Coordinator.getInstance(); private final TableDeviceSchemaCache cache = new TableDeviceSchemaCache(); + private final ClusterConfigTaskExecutor configTaskExecutor = + ClusterConfigTaskExecutor.getInstance(); + private static class TableModelSchemaFetcherHolder { private static final TableModelSchemaFetcher INSTANCE = new TableModelSchemaFetcher(); } @@ -183,12 +195,23 @@ public void autoCreateTable(String database, TableSchema tableSchema, MPPQueryCo private void autoCreateColumn( String database, String tableName, - List columnSchemaList, + List inputColumnList, MPPQueryContext context) { - throw new SemanticException( - String.format( - "Unknown columns %s", - columnSchemaList.stream().map(ColumnSchema::getName).collect(Collectors.toList()))); + AlterTableAddColumnTask task = + new AlterTableAddColumnTask( + database, tableName, inputColumnList, context.getQueryId().getId()); + try { + ListenableFuture future = task.execute(configTaskExecutor); + ConfigTaskResult result = future.get(); + if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + "Auto add table column failed.", result.getStatusCode().getStatusCode())); + } + } catch (ExecutionException | InterruptedException e) { + LOGGER.warn("Auto add table column failed.", e); + throw new RuntimeException(e); + } } public void validateDeviceSchema( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index bce098411a20..24f592fd0be4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -180,6 +181,30 @@ public void invalidateTable(String database) { } } + @Override + public void addTableColumn( + String database, String tableName, List columnSchemaList) { + readWriteLock.writeLock().lock(); + try { + TsTable table = databaseTableMap.get(database).get(tableName); + columnSchemaList.forEach(table::addColumnSchema); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void rollbackAddColumn( + String database, String tableName, List columnSchemaList) { + readWriteLock.writeLock().lock(); + try { + TsTable table = databaseTableMap.get(database).get(tableName); + columnSchemaList.forEach(o -> table.removeColumnSchema(o.getColumnName())); + } finally { + readWriteLock.writeLock().unlock(); + } + } + public TsTable getTable(String database, String tableName) { readWriteLock.readLock().lock(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index 47a58118df26..9efeeb940161 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -20,6 +20,9 @@ package org.apache.iotdb.db.schemaengine.table; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; + +import java.util.List; public interface ITableCache { @@ -34,4 +37,10 @@ public interface ITableCache { void invalidateTable(String database, String tableName); void invalidateTable(String database); + + void addTableColumn( + String database, String tableName, List columnSchemaList); + + void rollbackAddColumn( + String database, String tableName, List columnSchemaList); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java new file mode 100644 index 000000000000..fe9d8b7e4faf --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +public enum AlterTableOperationType { + ADD_COLUMN((byte) 0); + + private final byte type; + + AlterTableOperationType(byte type) { + this.type = type; + } + + public byte getTypeValue() { + return type; + } + + public static AlterTableOperationType getType(byte value) { + switch (value) { + case 0: + return ADD_COLUMN; + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index bc8dab7fbc07..dd0d4662cc70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -75,6 +75,10 @@ public void addColumnSchema(TsTableColumnSchema columnSchema) { } } + public void removeColumnSchema(String columnName) { + columnSchemaMap.remove(columnName); + } + public int getColumnNum() { return columnSchemaMap.size(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java index 9fcce8781f62..fb080a266c33 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCType.java @@ -32,7 +32,10 @@ public enum TsTableInternalRPCType { INVALIDATE_CACHE((byte) 3), DELETE_DATA_IN_DATA_REGION((byte) 4), - DELETE_SCHEMA_IN_SCHEMA_REGION((byte) 5); + DELETE_SCHEMA_IN_SCHEMA_REGION((byte) 5), + + ADD_COLUMN((byte) 6), + ROLLBACK_ADD_COLUMN((byte) 7); private final byte operationType; @@ -67,6 +70,10 @@ public static TsTableInternalRPCType getType(byte type) { return DELETE_DATA_IN_DATA_REGION; case 5: return DELETE_SCHEMA_IN_SCHEMA_REGION; + case 6: + return ADD_COLUMN; + case 7: + return ROLLBACK_ADD_COLUMN; default: throw new IllegalArgumentException("Unknown table update operation type" + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java index 4ebb8a25231f..eac67b2b3b28 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java @@ -21,10 +21,13 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class TsTableColumnSchemaUtil { @@ -77,4 +80,38 @@ private static TsTableColumnSchema deserialize( throw new IllegalArgumentException(); } } + + public static byte[] serialize(List columnSchemaList) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + serialize(columnSchemaList, stream); + } catch (IOException ignored) { + // won't happen + } + return stream.toByteArray(); + } + + public static void serialize(List columnSchemaList, OutputStream stream) + throws IOException { + if (columnSchemaList == null) { + ReadWriteIOUtils.write(-1, stream); + return; + } + ReadWriteIOUtils.write(columnSchemaList.size(), stream); + for (TsTableColumnSchema columnSchema : columnSchemaList) { + serialize(columnSchema, stream); + } + } + + public static List deserializeColumnSchemaList(ByteBuffer buffer) { + int size = ReadWriteIOUtils.readInt(buffer); + if (size == -1) { + return null; + } + List columnSchemaList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + columnSchemaList.add(deserialize(buffer)); + } + return columnSchemaList; + } } diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 0fb148b8737c..3a15aab022e6 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -928,6 +928,14 @@ struct TDropTableReq{ 3: required string queryId } +struct TAlterTableReq{ + 1: required string database + 2: required string tableName + 3: required string queryId + 4: required byte operationType + 5: required binary updateInfo +} + service IConfigNodeRPCService { // ====================================================== @@ -1576,5 +1584,7 @@ service IConfigNodeRPCService { common.TSStatus createTable(binary tableInfo) common.TSStatus dropTable(TDropTableReq req) + + common.TSStatus alterTable(TAlterTableReq req) } From 861c4b383ce2d4a8c7e5bfe1c45c4d3bf17d0601 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Tue, 21 May 2024 12:22:11 +0800 Subject: [PATCH 29/36] fix serialize bug during device fetch --- .../apache/iotdb/db/queryengine/common/header/ColumnHeader.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java index 2ae8d8ca234a..dff3becd51b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeader.java @@ -71,7 +71,6 @@ public void serialize(ByteBuffer byteBuffer) { if (hasAlias()) { ReadWriteIOUtils.write(alias, byteBuffer); } - dataType.serializeTo(byteBuffer); } public void serialize(DataOutputStream stream) throws IOException { @@ -81,7 +80,6 @@ public void serialize(DataOutputStream stream) throws IOException { if (hasAlias()) { ReadWriteIOUtils.write(alias, stream); } - dataType.serializeTo(stream); } public static ColumnHeader deserialize(ByteBuffer byteBuffer) { From 4b9cd18faccb62882ad0e49936d9016d12b0babb Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 22 May 2024 10:59:55 +0800 Subject: [PATCH 30/36] insert null --- .../plan/statement/crud/InsertTableStatement.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index a08f0e3545e6..09211bca3637 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.relational.sql.tree.Identifier; import org.apache.iotdb.db.relational.sql.tree.Insert; import org.apache.iotdb.db.relational.sql.tree.LongLiteral; +import org.apache.iotdb.db.relational.sql.tree.NullLiteral; import org.apache.iotdb.db.relational.sql.tree.Row; import org.apache.iotdb.db.relational.sql.tree.Values; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -121,11 +122,17 @@ private InsertRowStatement parseInsert() { hasColumn ? columnNameList.get(i).getValue() : table.getColumnList().get(i).getColumnName(); + if (columnName.equalsIgnoreCase("time")){ + columnName = "Time"; + } TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); if (columnSchema == null) { throw new SemanticException(String.format("Unknown Column %s", columnName)); } TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); + if (values.get(i) instanceof NullLiteral){ + continue; + } if (category.equals(TsTableColumnCategory.ID)) { idColumnMap.put(columnName, ((Identifier) values.get(i)).getValue()); } else if (category.equals(TsTableColumnCategory.ATTRIBUTE)) { From 0daae576286f4a4bb40a3c8bc5c3e44c1e226196 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 22 May 2024 11:33:06 +0800 Subject: [PATCH 31/36] fix --- .../analyzer/schema/TableModelSchemaFetcher.java | 7 ++++++- .../plan/relational/metadata/DeviceEntry.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 6e2108c2ea4b..0819ccb5f6df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -69,9 +69,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -507,7 +509,10 @@ public List fetchDeviceSchema( cacheFetchedDevice); } - return deviceEntryList; + Set set = new HashSet<>(); + deviceEntryList.forEach(set::add); + + return new ArrayList<>(set); } private Pair, List> transformExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java index 396352ab78da..6a46cb52301e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class DeviceEntry { @@ -73,6 +74,19 @@ public static DeviceEntry deserialize(ByteBuffer byteBuffer) { return new DeviceEntry(iDeviceID, attributeColumnValues); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeviceEntry)) return false; + DeviceEntry that = (DeviceEntry) o; + return Objects.equals(deviceID, that.deviceID) && Objects.equals(attributeColumnValues, that.attributeColumnValues); + } + + @Override + public int hashCode() { + return Objects.hash(deviceID, attributeColumnValues); + } + @Override public String toString() { return "DeviceEntry{" From 4de5ed1080b9363d050802080a05d08ed99a0deb Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 22 May 2024 12:02:14 +0800 Subject: [PATCH 32/36] use linkedHashSet for deduplicate --- .../relational/analyzer/schema/TableModelSchemaFetcher.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 0819ccb5f6df..be011951a003 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -70,6 +70,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -509,9 +510,7 @@ public List fetchDeviceSchema( cacheFetchedDevice); } - Set set = new HashSet<>(); - deviceEntryList.forEach(set::add); - + Set set = new LinkedHashSet<>(deviceEntryList); return new ArrayList<>(set); } From 83541b6104e6bf196a7031cc7b1de4b6394ef3fe Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 22 May 2024 15:21:15 +0800 Subject: [PATCH 33/36] fix invalidate cache during droping table --- .../thrift/impl/DataNodeInternalRPCServiceImpl.java | 9 ++++++--- .../analyzer/schema/TableModelSchemaFetcher.java | 4 ++++ .../analyzer/schema/cache/TableDeviceSchemaCache.java | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 31bd9b7fe87a..5535cbc3c7a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -121,6 +121,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -1456,10 +1458,11 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { ReadWriteIOUtils.readString(req.tableInfo)); break; case INVALIDATE_CACHE: + database = ReadWriteIOUtils.readString(req.tableInfo); + tableName = ReadWriteIOUtils.readString(req.tableInfo); DataNodeTableCache.getInstance() - .invalidateTable( - ReadWriteIOUtils.readString(req.tableInfo), - ReadWriteIOUtils.readString(req.tableInfo)); + .invalidateTable(database, tableName); + TableModelSchemaFetcher.getInstance().invalidateDeviceCache(database, tableName); break; case DELETE_DATA_IN_DATA_REGION: database = ReadWriteIOUtils.readString(req.tableInfo); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index be011951a003..8700f069b84a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -106,6 +106,10 @@ private TableModelSchemaFetcher() { // do nothing } + public void invalidateDeviceCache(String database, String tableName){ + cache.invalidate(database, tableName); + } + // This method return all the existing column schemas in the target table. // When table or column is missing, this method will execute auto creation. // When using SQL, the columnSchemaList could be null and there won't be any validation. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java index c73125b4a281..3cff9377a381 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -74,4 +74,13 @@ public void put( readWriteLock.readLock().unlock(); } } + + public void invalidate(String database, String tableName){ + readWriteLock.writeLock().lock(); + try { + dualKeyCache.invalidateAll(); + }finally { + readWriteLock.writeLock().unlock(); + } + } } From 586b419242a023dc761de23e8ba9125db150e5d9 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Wed, 22 May 2024 15:22:36 +0800 Subject: [PATCH 34/36] spotless apply --- .../protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java | 4 +--- .../relational/analyzer/schema/TableModelSchemaFetcher.java | 3 +-- .../analyzer/schema/cache/TableDeviceSchemaCache.java | 4 ++-- .../db/queryengine/plan/relational/metadata/DeviceEntry.java | 3 ++- .../queryengine/plan/statement/crud/InsertTableStatement.java | 4 ++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 5535cbc3c7a9..383ad26d28b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -122,7 +122,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.TableModelSchemaFetcher; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -1460,8 +1459,7 @@ public TSStatus updateTable(TUpdateTableReq req) throws TException { case INVALIDATE_CACHE: database = ReadWriteIOUtils.readString(req.tableInfo); tableName = ReadWriteIOUtils.readString(req.tableInfo); - DataNodeTableCache.getInstance() - .invalidateTable(database, tableName); + DataNodeTableCache.getInstance().invalidateTable(database, tableName); TableModelSchemaFetcher.getInstance().invalidateDeviceCache(database, tableName); break; case DELETE_DATA_IN_DATA_REGION: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 8700f069b84a..784bcc743eee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -69,7 +69,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -106,7 +105,7 @@ private TableModelSchemaFetcher() { // do nothing } - public void invalidateDeviceCache(String database, String tableName){ + public void invalidateDeviceCache(String database, String tableName) { cache.invalidate(database, tableName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java index 3cff9377a381..4c17dbfadff5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/cache/TableDeviceSchemaCache.java @@ -75,11 +75,11 @@ public void put( } } - public void invalidate(String database, String tableName){ + public void invalidate(String database, String tableName) { readWriteLock.writeLock().lock(); try { dualKeyCache.invalidateAll(); - }finally { + } finally { readWriteLock.writeLock().unlock(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java index 6a46cb52301e..09bb03daf885 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java @@ -79,7 +79,8 @@ public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof DeviceEntry)) return false; DeviceEntry that = (DeviceEntry) o; - return Objects.equals(deviceID, that.deviceID) && Objects.equals(attributeColumnValues, that.attributeColumnValues); + return Objects.equals(deviceID, that.deviceID) + && Objects.equals(attributeColumnValues, that.attributeColumnValues); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java index 09211bca3637..4fe5fac42ab9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTableStatement.java @@ -122,7 +122,7 @@ private InsertRowStatement parseInsert() { hasColumn ? columnNameList.get(i).getValue() : table.getColumnList().get(i).getColumnName(); - if (columnName.equalsIgnoreCase("time")){ + if (columnName.equalsIgnoreCase("time")) { columnName = "Time"; } TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); @@ -130,7 +130,7 @@ private InsertRowStatement parseInsert() { throw new SemanticException(String.format("Unknown Column %s", columnName)); } TsTableColumnCategory category = table.getColumnSchema(columnName).getColumnCategory(); - if (values.get(i) instanceof NullLiteral){ + if (values.get(i) instanceof NullLiteral) { continue; } if (category.equals(TsTableColumnCategory.ID)) { From da315b9e833c7153b41c728b9f950c2f99c05cd7 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Mon, 27 May 2024 16:04:25 +0800 Subject: [PATCH 35/36] fix multi region --- .../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- .../db/queryengine/plan/analyze/AnalyzeVisitor.java | 4 +--- .../node/metedata/read/table/TableDeviceScanNode.java | 11 +++++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index a164c07fe36b..3f8821714ed4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -87,7 +87,7 @@ public class ConfigNodeConfig { * SchemaRegionGroups for each Database. When set schema_region_group_extension_policy=AUTO, this * parameter is the default minimal number of SchemaRegionGroups for each Database. */ - private int defaultSchemaRegionGroupNumPerDatabase = 1; + private int defaultSchemaRegionGroupNumPerDatabase = 2; /** The maximum number of SchemaRegions expected to be managed by each DataNode. */ private double schemaRegionPerDataNode = schemaReplicationFactor; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 65bcbc18b5a6..911f5a4ded54 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -3740,9 +3740,7 @@ public Analysis visitShowTableDevices( patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); } - SchemaPartition partition = - partitionFetcher.getOrCreateSchemaPartition( - patternTree, context.getSession().getUserName()); + SchemaPartition partition = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(partition); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java index 072ad10cf751..64400d3dd1ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/table/TableDeviceScanNode.java @@ -20,13 +20,16 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.table; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -93,6 +96,14 @@ public List getColumnHeaderList() { return columnHeaderList; } + @Override + public List getPathPatternList() { + return DeviceFilterUtil.convertToDevicePattern( + database, + DataNodeTableCache.getInstance().getTable(database, tableName), + idDeterminedFilterList); + } + @Override public void open() throws Exception {} From a4fc70998f918372ea028cd3666c882486257c69 Mon Sep 17 00:00:00 2001 From: MarcosZyk <1534661820@qq.com> Date: Tue, 11 Jun 2024 09:19:44 +0800 Subject: [PATCH 36/36] fix serialization --- .../apache/iotdb/confignode/persistence/schema/ConfigMTree.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 6252fffe10cb..27b8e263e3a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -804,7 +804,7 @@ private void serializeChildren(IConfigMNode node, OutputStream outputStream) thr if (child.isDatabase()) { serializeDatabaseNode(child.getAsDatabaseMNode(), outputStream); } else if (child instanceof ConfigTableNode) { - serializeTableNode((ConfigTableNode) node, outputStream); + serializeTableNode((ConfigTableNode) child, outputStream); } else { serializeConfigBasicMNode(child, outputStream); }