From bcaaa918bd55555751266cda9ec868bccf4c9e96 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 25 Jul 2022 23:49:48 +0800 Subject: [PATCH 01/37] create statement and analysis --- .../db/mpp/plan/analyze/AnalyzeVisitor.java | 61 ++++++++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 44 +++++- .../planner/plan/node/write/LoadFileNode.java | 77 +++++++++++ .../mpp/plan/statement/StatementVisitor.java | 5 + .../statement/crud/LoadFileStatement.java | 130 ++++++++++++++++++ 5 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 9279c0ae31720..c1beca0e06359 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -18,12 +18,15 @@ */ package org.apache.iotdb.db.mpp.plan.analyze; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.sql.MeasurementNotExistException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; @@ -55,6 +58,7 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.LastPointFetchStatement; @@ -82,6 +86,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ShowSchemaTemplateStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.ShowVersionStatement; +import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.filter.GroupByFilter; import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter; @@ -92,6 +97,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -1050,6 +1057,60 @@ public Analysis visitInsertRowsOfOneDevice( return analysis; } + @Override + public Analysis visitLoadFile(LoadFileStatement loadFileStatement, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + + Map device2MinTime = new HashMap<>(); + Map device2MaxTime = new HashMap<>(); + for (File tsFile : loadFileStatement.getTsFiles()) { + try { + TsFileResource resource = new TsFileResource(tsFile); + FileLoaderUtils.loadOrGenerateResource(resource); + for (String device : resource.getDevices()) { + device2MinTime.put( + device, + Math.min( + device2MinTime.getOrDefault(device, Long.MAX_VALUE), + resource.getStartTime(device))); + device2MaxTime.put( + device, + Math.max( + device2MaxTime.getOrDefault(device, Long.MIN_VALUE), + resource.getEndTime(device))); + } + } catch (IOException e) { + logger.error(String.format("Parse file %s to resource error.", tsFile.getPath()), e); + throw new SemanticException( + String.format("Parse file %s to resource error", tsFile.getPath())); + } + } + + List params = new ArrayList<>(); + for (Map.Entry entry : device2MinTime.entrySet()) { + List timePartitionSlots = new ArrayList<>(); + String device = entry.getKey(); + long endTime = device2MaxTime.get(device); + long interval = StorageEngineV2.getTimePartitionInterval(); + long time = (entry.getValue() / interval) * interval; + for (; time <= endTime; time += interval) { + timePartitionSlots.add(StorageEngineV2.getTimePartitionSlot(time)); + } + + DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); + dataPartitionQueryParam.setDevicePath(device); + dataPartitionQueryParam.setTimePartitionSlotList(timePartitionSlots); + } + + DataPartition dataPartition = partitionFetcher.getOrCreateDataPartition(params); + + Analysis analysis = new Analysis(); + analysis.setStatement(loadFileStatement); + analysis.setDataPartitionInfo(dataPartition); + + return analysis; + } + @Override public Analysis visitShowTimeSeries( ShowTimeSeriesStatement showTimeSeriesStatement, MPPQueryContext context) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index c81604671f15f..c6575c7a59aaa 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -74,6 +74,7 @@ import org.apache.iotdb.db.mpp.plan.statement.component.WhereCondition; import org.apache.iotdb.db.mpp.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.literal.BooleanLiteral; import org.apache.iotdb.db.mpp.plan.statement.literal.DoubleLiteral; @@ -1315,6 +1316,44 @@ private void parseInsertValuesSpec( insertStatement.setValuesList(valuesList); } + // Load File + + @Override + public Statement visitLoadFile(IoTDBSqlParser.LoadFileContext ctx) { + LoadFileStatement loadFileStatement = + new LoadFileStatement(parseStringLiteral(ctx.fileName.getText())); + if (ctx.loadFilesClause() != null) { + parseLoadFiles(loadFileStatement, ctx.loadFilesClause()); + } + return loadFileStatement; + } + + /** + * used for parsing load tsfile, context will be one of "SCHEMA, LEVEL, METADATA", and maybe + * followed by a recursion property statement + * + * @param loadFileStatement the result statement, setting by clause context + * @param ctx context of property statement + */ + private void parseLoadFiles( + LoadFileStatement loadFileStatement, IoTDBSqlParser.LoadFilesClauseContext ctx) { + if (ctx.AUTOREGISTER() != null) { + loadFileStatement.setAutoCreateSchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); + } else if (ctx.SGLEVEL() != null) { + loadFileStatement.setSgLevel(Integer.parseInt(ctx.INTEGER_LITERAL().getText())); + } else if (ctx.VERIFY() != null) { + loadFileStatement.setVerifySchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); + } else { + throw new SQLParserException( + String.format( + "load tsfile format %s error, please input AUTOREGISTER | SGLEVEL | VERIFY.", + ctx.getText())); + } + if (ctx.loadFilesClause() != null) { + parseLoadFiles(loadFileStatement, ctx.loadFilesClause()); + } + } + /** Common Parsers */ // IoTDB Objects ======================================================================== @@ -1943,11 +1982,6 @@ private TimeRange parseTimeRange( } } - /** function for parsing file path used by LOAD statement. */ - public String parseFilePath(String src) { - return src.substring(1, src.length() - 1); - } - // Expression & Predicate ======================================================================== private Expression parseExpression( diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java new file mode 100644 index 0000000000000..09ea74e9fc274 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.planner.plan.node.write; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.mpp.plan.analyze.Analysis; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class LoadFileNode extends WritePlanNode { + + protected LoadFileNode(PlanNodeId id) { + super(id); + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return null; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return null; + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException {} + + @Override + public List splitByPartition(Analysis analysis) { + return null; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java index 3137f958d8809..b62caca8199f9 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.LastPointFetchStatement; @@ -179,6 +180,10 @@ public R visitInsertTablet(InsertTabletStatement insertTabletStatement, C contex return visitStatement(insertTabletStatement, context); } + public R visitLoadFile(LoadFileStatement loadFileStatement, C context) { + return visitStatement(loadFileStatement, context); + } + /** Data Control Language (DCL) */ public R visitAuthor(AuthorStatement authorStatement, C context) { return visitStatement(authorStatement, context); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java new file mode 100644 index 0000000000000..289f0d555022b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.statement.crud; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.mpp.plan.statement.Statement; +import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.utils.FilePathUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class LoadFileStatement extends Statement { + private File file; + private boolean autoCreateSchema; + private int sgLevel; + private boolean verifySchema; + + private List tsFiles; + + public LoadFileStatement(String filePath) { + this.file = new File(filePath); + this.autoCreateSchema = true; + this.sgLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel(); + this.verifySchema = true; + + tsFiles = new ArrayList<>(); + if (file.isFile()) { + tsFiles.add(file); + } else { + findAllTsFile(file); + } + sortTsFiles(tsFiles); + } + + private void findAllTsFile(File file) { + for (File nowFile : file.listFiles()) { + if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { + tsFiles.add(nowFile); + } else if (nowFile.isDirectory()) { + findAllTsFile(nowFile); + } + } + } + + private void sortTsFiles(List files) { + Map file2Timestamp = new HashMap<>(); + Map file2Version = new HashMap<>(); + for (File file : files) { + String[] splitStrings = file.getName().split(FilePathUtils.FILE_NAME_SEPARATOR); + file2Timestamp.put(file, Long.parseLong(splitStrings[0])); + file2Version.put(file, Long.parseLong(splitStrings[1])); + } + + Collections.sort( + files, + (o1, o2) -> { + long timestampDiff = file2Timestamp.get(o1) - file2Timestamp.get(o2); + if (timestampDiff != 0) { + return (int) (timestampDiff); + } + return (int) (file2Version.get(o1) - file2Version.get(o2)); + }); + } + + public void setAutoCreateSchema(boolean autoCreateSchema) { + this.autoCreateSchema = autoCreateSchema; + } + + public void setSgLevel(int sgLevel) { + this.sgLevel = sgLevel; + } + + public void setVerifySchema(boolean verifySchema) { + this.verifySchema = verifySchema; + } + + public List getTsFiles() { + return tsFiles; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitLoadFile(this, context); + } + + @Override + public String toString() { + return "LoadFileStatement{" + + "file=" + + file + + ", autoCreateSchema=" + + autoCreateSchema + + ", sgLevel=" + + sgLevel + + ", verifySchema=" + + verifySchema + + ", tsFiles Size=" + + tsFiles.size() + + '}'; + } +} From 98a22862840491d71cc92daa709c224305ebec1b Mon Sep 17 00:00:00 2001 From: yschengzi Date: Sun, 14 Aug 2022 23:15:34 +0800 Subject: [PATCH 02/37] finish coordinator and scheduler --- .../iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 11 + .../db/mpp/plan/analyze/AnalyzeVisitor.java | 9 +- .../db/mpp/plan/execution/QueryExecution.java | 7 + .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 22 +- .../mpp/plan/planner/LogicalPlanVisitor.java | 7 + .../plan/node/load/LoadSingleTsFileNode.java | 351 ++++++++++++++ .../LoadTsFileNode.java} | 35 +- .../plan/node/load/LoadTsFilePieceNode.java | 429 ++++++++++++++++++ .../load/LoadTsFileDispatcherImpl.java | 37 ++ .../scheduler/load/LoadTsFileScheduler.java | 131 ++++++ .../mpp/plan/statement/StatementVisitor.java | 6 +- ...tatement.java => LoadTsFileStatement.java} | 6 +- .../db/sync/datasource/TsFileOpBlock.java | 2 +- .../write/chunk/AlignedChunkWriterImpl.java | 6 + .../tsfile/write/chunk/ChunkWriterImpl.java | 6 + .../tsfile/write/chunk/IChunkWriter.java | 3 + 17 files changed, 1041 insertions(+), 29 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java rename server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/{write/LoadFileNode.java => load/LoadTsFileNode.java} (62%) create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java rename server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/{LoadFileStatement.java => LoadTsFileStatement.java} (96%) diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index aa946f67b7d5e..5aacec0152e5c 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "test.tsfile"; + String filename = "alignedTablet.tsfile"; if (args.length >= 1) { filename = args[0]; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index cb2583e4d9499..da26cf4e04013 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -42,6 +42,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.fileSystem.FSType; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -726,6 +728,11 @@ public class IoTDBConfig { */ private long partitionInterval = 604800; + /** + * Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. + */ + private long maxPlanNodeSize = 500 * 1048576L; + /** * Level of TimeIndex, which records the start time and end time of TsFileResource. Currently, * DEVICE_TIME_INDEX and FILE_TIME_INDEX are supported, and could not be changed after first set. @@ -1035,6 +1042,10 @@ public void setPartitionInterval(long partitionInterval) { this.partitionInterval = partitionInterval; } + public long getMaxPlanNodeSize() { + return maxPlanNodeSize; + } + public TimeIndexLevel getTimeIndexLevel() { return timeIndexLevel; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index ca8090d7e32d8..8d1497f2a2f71 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -65,7 +65,7 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; -import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.SchemaFetchStatement; @@ -1195,12 +1195,12 @@ public Analysis visitInsertRowsOfOneDevice( } @Override - public Analysis visitLoadFile(LoadFileStatement loadFileStatement, MPPQueryContext context) { + public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); Map device2MinTime = new HashMap<>(); Map device2MaxTime = new HashMap<>(); - for (File tsFile : loadFileStatement.getTsFiles()) { + for (File tsFile : loadTsFileStatement.getTsFiles()) { try { TsFileResource resource = new TsFileResource(tsFile); FileLoaderUtils.loadOrGenerateResource(resource); @@ -1237,12 +1237,13 @@ public Analysis visitLoadFile(LoadFileStatement loadFileStatement, MPPQueryConte DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDevicePath(device); dataPartitionQueryParam.setTimePartitionSlotList(timePartitionSlots); + params.add(dataPartitionQueryParam); } DataPartition dataPartition = partitionFetcher.getOrCreateDataPartition(params); Analysis analysis = new Analysis(); - analysis.setStatement(loadFileStatement); + analysis.setStatement(loadTsFileStatement); analysis.setDataPartitionInfo(dataPartition); return analysis; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index 842352a7b608f..d3eba9a3cab6d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -51,11 +51,13 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeUtil; import org.apache.iotdb.db.mpp.plan.scheduler.ClusterScheduler; import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.mpp.plan.scheduler.StandaloneScheduler; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.read.common.block.TsBlock; @@ -235,6 +237,11 @@ private Analysis analyze( } private void schedule() { + if (rawStatement instanceof LoadTsFileStatement) { + this.scheduler = new LoadTsFileScheduler(distributedPlan, context); + this.scheduler.start(); + return; + } // TODO: (xingtanzjr) initialize the query scheduler according to configuration this.scheduler = config.isClusterMode() diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index 2d89e442ac378..e781de13e4a19 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -75,7 +75,7 @@ import org.apache.iotdb.db.mpp.plan.statement.component.WhereCondition; import org.apache.iotdb.db.mpp.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; -import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.literal.BooleanLiteral; import org.apache.iotdb.db.mpp.plan.statement.literal.DoubleLiteral; @@ -1348,29 +1348,29 @@ private void parseInsertValuesSpec( @Override public Statement visitLoadFile(IoTDBSqlParser.LoadFileContext ctx) { - LoadFileStatement loadFileStatement = - new LoadFileStatement(parseStringLiteral(ctx.fileName.getText())); + LoadTsFileStatement loadTsFileStatement = + new LoadTsFileStatement(parseStringLiteral(ctx.fileName.getText())); if (ctx.loadFilesClause() != null) { - parseLoadFiles(loadFileStatement, ctx.loadFilesClause()); + parseLoadFiles(loadTsFileStatement, ctx.loadFilesClause()); } - return loadFileStatement; + return loadTsFileStatement; } /** * used for parsing load tsfile, context will be one of "SCHEMA, LEVEL, METADATA", and maybe * followed by a recursion property statement * - * @param loadFileStatement the result statement, setting by clause context + * @param loadTsFileStatement the result statement, setting by clause context * @param ctx context of property statement */ private void parseLoadFiles( - LoadFileStatement loadFileStatement, IoTDBSqlParser.LoadFilesClauseContext ctx) { + LoadTsFileStatement loadTsFileStatement, IoTDBSqlParser.LoadFilesClauseContext ctx) { if (ctx.AUTOREGISTER() != null) { - loadFileStatement.setAutoCreateSchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); + loadTsFileStatement.setAutoCreateSchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); } else if (ctx.SGLEVEL() != null) { - loadFileStatement.setSgLevel(Integer.parseInt(ctx.INTEGER_LITERAL().getText())); + loadTsFileStatement.setSgLevel(Integer.parseInt(ctx.INTEGER_LITERAL().getText())); } else if (ctx.VERIFY() != null) { - loadFileStatement.setVerifySchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); + loadTsFileStatement.setVerifySchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); } else { throw new SQLParserException( String.format( @@ -1378,7 +1378,7 @@ private void parseLoadFiles( ctx.getText())); } if (ctx.loadFilesClause() != null) { - parseLoadFiles(loadFileStatement, ctx.loadFilesClause()); + parseLoadFiles(loadTsFileStatement, ctx.loadFilesClause()); } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java index ad5cbbe1be2ec..bdae4cadbef92 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.OrderByParameter; import org.apache.iotdb.db.mpp.plan.statement.StatementNode; @@ -46,6 +47,7 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.SchemaFetchStatement; @@ -417,6 +419,11 @@ public PlanNode visitInsertRow(InsertRowStatement insertRowStatement, MPPQueryCo insertRowStatement.isNeedInferType()); } + @Override + public PlanNode visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { + return new LoadTsFileNode(context.getQueryId().genPlanNodeId(), loadTsFileStatement.getTsFiles()); + } + @Override public PlanNode visitShowTimeSeries( ShowTimeSeriesStatement showTimeSeriesStatement, MPPQueryContext context) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java new file mode 100644 index 0000000000000..40691b98dcebf --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.db.mpp.plan.analyze.Analysis; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode.AlignedChunkData; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode.ChunkData; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.exception.NotImplementedException; +import org.apache.iotdb.tsfile.exception.TsFileRuntimeException; +import org.apache.iotdb.tsfile.file.MetaMarker; +import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.reader.page.PageReader; +import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class LoadSingleTsFileNode extends WritePlanNode { + private static final Logger logger = LoggerFactory.getLogger(LoadSingleTsFileNode.class); + + private File tsFile; + private Map> replicaSet2Pieces; + + public LoadSingleTsFileNode(PlanNodeId id) { + super(id); + } + + public LoadSingleTsFileNode(PlanNodeId id, File tsFile) { + super(id); + this.tsFile = tsFile; + } + + public Map> getReplicaSet2Pieces() { + return replicaSet2Pieces; + } + + private LoadTsFilePieceNode getPieceNode( + String device, TTimePartitionSlot timePartitionSlot, DataPartition dataPartition) { + TRegionReplicaSet replicaSet = + dataPartition.getDataRegionReplicaSetForWriting(device, timePartitionSlot); + List pieceNodes = + replicaSet2Pieces.computeIfAbsent(replicaSet, o -> new ArrayList<>()); + if (pieceNodes.isEmpty() || pieceNodes.get(pieceNodes.size() - 1).exceedSize()) { + pieceNodes.add(new LoadTsFilePieceNode(getPlanNodeId(), tsFile)); + } + return pieceNodes.get(pieceNodes.size() - 1); + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return null; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + throw new NotImplementedException("clone of load single TsFile is not implemented"); + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException {} + + @Override + public List splitByPartition(Analysis analysis) { + throw new NotImplementedException("split load single TsFile is not implemented"); + } + + public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOException { + replicaSet2Pieces = new HashMap<>(); + List chunkDataList = new ArrayList<>(); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + if (checkMagic(reader)) { + throw new TsFileRuntimeException( + String.format("Magic String check error when parsing TsFile %s.", tsFile.getPath())); + } + + reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); + String curDevice = null; + Map> pageIndex2ChunkData = null; + byte marker; + while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { + switch (marker) { + case MetaMarker.CHUNK_HEADER: + case MetaMarker.TIME_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + ChunkHeader header = reader.readChunkHeader(marker); + if (header.getDataSize() == 0) { + break; + } + + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(header.getEncodingType(), header.getDataType()); + int dataSize = header.getDataSize(); + int pageIndex = 0; + + boolean isAligned = + ((header.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) + == TsFileConstant.TIME_COLUMN_MASK); + TTimePartitionSlot timePartitionSlot = null; + ChunkData chunkData = + ChunkData.createChunkData(isAligned, reader.position(), curDevice, header); + if (isAligned) { + pageIndex2ChunkData = new HashMap<>(); + } + + while (dataSize > 0) { + long pageOffset = reader.position(); + PageHeader pageHeader = + reader.readPageHeader( + header.getDataType(), + (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + long pageDataSize = pageHeader.getSerializedPageSize(); + if (timePartitionSlot == null) { // init time slot + timePartitionSlot = StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()); + chunkData.setTimePartitionSlot(timePartitionSlot); + } + if (!needDecodePage(pageHeader)) { // an entire page + TTimePartitionSlot pageTimePartitionSlot = + StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()); + if (!timePartitionSlot.equals(pageTimePartitionSlot)) { + chunkDataList.add(chunkData); + timePartitionSlot = pageTimePartitionSlot; + chunkData = ChunkData.createChunkData(isAligned, pageOffset, curDevice, header); + chunkData.setTimePartitionSlot(timePartitionSlot); + } + if (isAligned) { + pageIndex2ChunkData + .computeIfAbsent(pageIndex, o -> new ArrayList<>()) + .add((AlignedChunkData) chunkData); + } + chunkData.addDataSize(pageDataSize); + reader.position(reader.position() + pageDataSize); + } else { // split page + ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); + long[] timeBatch = + decodePage( + isAligned, pageData, pageHeader, defaultTimeDecoder, valueDecoder, header); + boolean isFirstData = true; + for (long currentTime : timeBatch) { + TTimePartitionSlot currentTimePartitionSlot = + StorageEngineV2.getTimePartitionSlot(currentTime); + if (!timePartitionSlot.equals(currentTimePartitionSlot)) { + if (!isFirstData) { + chunkData.setTailPageNeedDecode(true); // close last chunk data + chunkData.addDataSize(pageDataSize); + if (isAligned) { + pageIndex2ChunkData + .computeIfAbsent(pageIndex, o -> new ArrayList<>()) + .add((AlignedChunkData) chunkData); + } + } + chunkDataList.add(chunkData); + + chunkData = + ChunkData.createChunkData( + isAligned, pageOffset, curDevice, header); // open a new chunk data + chunkData.setTimePartitionSlot(currentTimePartitionSlot); + chunkData.setHeadPageNeedDecode(true); + timePartitionSlot = currentTimePartitionSlot; + } + isFirstData = false; + } + chunkData.addDataSize(pageDataSize); + if (isAligned) { + pageIndex2ChunkData + .computeIfAbsent(pageIndex, o -> new ArrayList<>()) + .add((AlignedChunkData) chunkData); + } + } + + pageIndex += 1; + dataSize -= pageDataSize; + } + + chunkDataList.add(chunkData); + break; + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + header = reader.readChunkHeader(marker); + if (header.getDataSize() == 0) { + break; + } + + Set allChunkData = new HashSet<>(); + dataSize = header.getDataSize(); + pageIndex = 0; + + while (dataSize > 0) { + long pageOffset = reader.position(); + PageHeader pageHeader = + reader.readPageHeader( + header.getDataType(), + (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + long pageDataSize = pageHeader.getSerializedPageSize(); + for (AlignedChunkData alignedChunkData : pageIndex2ChunkData.get(pageIndex)) { + if (!allChunkData.contains(alignedChunkData)) { + alignedChunkData.addValueChunk(pageOffset, header); + allChunkData.add(alignedChunkData); + } + alignedChunkData.addValueChunkDataSize(pageDataSize); + } + reader.position(reader.position() + pageDataSize); + + pageIndex += 1; + dataSize -= pageDataSize; + } + break; + case MetaMarker.CHUNK_GROUP_HEADER: + ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(); + curDevice = chunkGroupHeader.getDeviceID(); + break; + case MetaMarker.OPERATION_INDEX_RANGE: + reader.readPlanIndex(); + break; + default: + MetaMarker.handleUnexpectedMarker(marker); + } + } + } + + for (ChunkData chunkData : chunkDataList) { + getPieceNode(chunkData.getDevice(), chunkData.getTimePartitionSlot(), dataPartition) + .addChunkData(chunkData); + } + } + + private boolean checkMagic(TsFileSequenceReader reader) throws IOException { + String magic = reader.readHeadMagic(); + if (!magic.equals(TSFileConfig.MAGIC_STRING)) { + logger.error("the file's MAGIC STRING is incorrect, file path: {}", reader.getFileName()); + return false; + } + + byte versionNumber = reader.readVersionNumber(); + if (versionNumber != TSFileConfig.VERSION_NUMBER) { + logger.error("the file's Version Number is incorrect, file path: {}", reader.getFileName()); + return false; + } + + if (!reader.readTailMagic().equals(TSFileConfig.MAGIC_STRING)) { + logger.error("the file is not closed correctly, file path: {}", reader.getFileName()); + return false; + } + return true; + } + + private boolean needDecodePage(PageHeader pageHeader) { + if (pageHeader.getStatistics() == null) { + return true; + } + return !StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()) + .equals(StorageEngineV2.getTimePartitionSlot(pageHeader.getEndTime())); + } + + private long[] decodePage( + boolean isAligned, + ByteBuffer pageData, + PageHeader pageHeader, + Decoder timeDecoder, + Decoder valueDecoder, + ChunkHeader chunkHeader) + throws IOException { + if (isAligned) { + TimePageReader timePageReader = new TimePageReader(pageHeader, pageData, timeDecoder); + return timePageReader.getNextTimeBatch(); + } + + valueDecoder.reset(); + PageReader pageReader = + new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); + BatchData batchData = pageReader.getAllSatisfiedPageData(); + long[] timeBatch = new long[batchData.length()]; + int index = 0; + while (batchData.hasCurrent()) { + timeBatch[index++] = batchData.currentTime(); + batchData.next(); + } + return timeBatch; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java similarity index 62% rename from server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java rename to server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index 09ea74e9fc274..9361ef53cc4a5 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/LoadFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -17,23 +17,36 @@ * under the License. */ -package org.apache.iotdb.db.mpp.plan.planner.plan.node.write; +package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.tsfile.exception.NotImplementedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; -public class LoadFileNode extends WritePlanNode { +public class LoadTsFileNode extends WritePlanNode { + private static final Logger logger = LoggerFactory.getLogger(LoadTsFileNode.class); - protected LoadFileNode(PlanNodeId id) { + private final List tsFiles; + + public LoadTsFileNode(PlanNodeId id) { + this(id, new ArrayList<>()); + } + + public LoadTsFileNode(PlanNodeId id, List tsFiles) { super(id); + this.tsFiles = tsFiles; } @Override @@ -51,12 +64,12 @@ public void addChild(PlanNode child) {} @Override public PlanNode clone() { - return null; + throw new NotImplementedException("clone of load TsFile is not implemented"); } @Override public int allowedChildCount() { - return 0; + return NO_CHILD_ALLOWED; } @Override @@ -72,6 +85,16 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { @Override public List splitByPartition(Analysis analysis) { - return null; + List res = new ArrayList<>(); + for (File file : tsFiles) { + try { + LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), file); + singleTsFileNode.splitTsFileByDataPartition(analysis.getDataPartitionInfo()); + res.add(singleTsFileNode); + } catch (Exception e) { + logger.error(String.format("Parse TsFile %s error", file.getPath()), e); + } + } + return res; } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java new file mode 100644 index 0000000000000..cd0e1254490b2 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -0,0 +1,429 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.mpp.plan.analyze.Analysis; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.exception.NotImplementedException; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.MetaMarker; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.common.Field; +import org.apache.iotdb.tsfile.read.reader.page.PageReader; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class LoadTsFilePieceNode extends WritePlanNode { + private static final Logger logger = LoggerFactory.getLogger(LoadTsFilePieceNode.class); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private File tsFile; + + private long dataSize; + private List chunkDataList; + + public LoadTsFilePieceNode(PlanNodeId id) { + super(id); + } + + public LoadTsFilePieceNode(PlanNodeId id, File tsFile) { + super(id); + this.tsFile = tsFile; + this.dataSize = 0; + this.chunkDataList = new ArrayList<>(); + } + + public boolean exceedSize() { + return dataSize >= config.getMaxPlanNodeSize(); + } + + public void addChunkData(ChunkData chunkData) { + chunkDataList.add(chunkData); + dataSize += chunkData.getDataSize(); + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return null; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + throw new NotImplementedException("clone of load piece TsFile is not implemented"); + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + try { + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); + DataOutputStream stream = new DataOutputStream(byteOutputStream); + serializeAttributes(stream); + byteBuffer.put(byteOutputStream.toByteArray()); + } catch (IOException e) { + logger.error("Serialize to ByteBuffer error.", e); + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + for (ChunkData chunkData : chunkDataList) { + try { + chunkData.getChunkWriter(tsFile).serializeToDataOutputStream(stream); + } catch (PageException e) { + logger.error( + String.format( + "Parse page of TsFile %s error, skip chunk %s", tsFile.getPath(), chunkData)); + } + } + } + + @Override + public List splitByPartition(Analysis analysis) { + throw new NotImplementedException("split load piece TsFile is not implemented"); + } + + public interface ChunkData { + String getDevice(); + + TTimePartitionSlot getTimePartitionSlot(); + + long getDataSize(); + + void addDataSize(long pageSize); + + void setHeadPageNeedDecode(boolean headPageNeedDecode); + + void setTailPageNeedDecode(boolean tailPageNeedDecode); + + void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot); + + IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException; + + static ChunkData createChunkData( + boolean isAligned, long offset, String device, ChunkHeader chunkHeader) { + return isAligned + ? new AlignedChunkData(offset, device, chunkHeader) + : new NonAlignedChunkData(offset, device, chunkHeader); + } + } + + public static class NonAlignedChunkData implements ChunkData { + private long offset; + private long dataSize; + private boolean isHeadPageNeedDecode; + private boolean isTailPageNeedDecode; + + private TTimePartitionSlot timePartitionSlot; + private String device; + private ChunkHeader chunkHeader; + + public NonAlignedChunkData(long offset, String device, ChunkHeader chunkHeader) { + this.offset = offset; + this.dataSize = 0; + this.isHeadPageNeedDecode = false; + this.isTailPageNeedDecode = false; + this.device = device; + this.chunkHeader = chunkHeader; + } + + @Override + public String getDevice() { + return device; + } + + @Override + public TTimePartitionSlot getTimePartitionSlot() { + return timePartitionSlot; + } + + @Override + public long getDataSize() { + return dataSize; + } + + @Override + public void addDataSize(long pageSize) { + dataSize += pageSize; + } + + @Override + public void setHeadPageNeedDecode(boolean headPageNeedDecode) { + isHeadPageNeedDecode = headPageNeedDecode; + } + + @Override + public void setTailPageNeedDecode(boolean tailPageNeedDecode) { + isTailPageNeedDecode = tailPageNeedDecode; + } + + @Override + public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { + this.timePartitionSlot = timePartitionSlot; + } + + @Override + public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema( + chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), + chunkHeader.getEncodingType(), + chunkHeader.getCompressionType())); + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + reader.position(offset); + long dataSize = this.dataSize; + while (dataSize > 0) { + PageHeader pageHeader = + reader.readPageHeader( + chunkHeader.getDataType(), + (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + long pageDataSize = pageHeader.getSerializedPageSize(); + if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page + || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page + decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); + } else { // entire page + ByteBuffer pageData = reader.readCompressedPage(pageHeader); + chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); + } + dataSize -= pageDataSize; + } + return chunkWriter; + } + } + + private void decodePage( + TsFileSequenceReader reader, + PageHeader pageHeader, + Decoder timeDecoder, + Decoder valueDecoder, + ChunkWriterImpl chunkWriter) + throws IOException { + valueDecoder.reset(); + ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); + PageReader pageReader = + new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + long time = batchData.currentTime(); + if (time < timePartitionSlot.getStartTime()) { + continue; + } else if (time > timePartitionSlot.getStartTime()) { + break; + } + + Object value = batchData.currentValue(); + switch (chunkHeader.getDataType()) { + case INT32: + chunkWriter.write(time, (int) value); + break; + case INT64: + chunkWriter.write(time, (long) value); + break; + case FLOAT: + chunkWriter.write(time, (float) value); + break; + case DOUBLE: + chunkWriter.write(time, (double) value); + break; + case BOOLEAN: + chunkWriter.write(time, (boolean) value); + break; + case TEXT: + chunkWriter.write(time, (Binary) value); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", chunkHeader.getDataType())); + } + batchData.next(); + } + chunkWriter.sealCurrentPage(); + } + + @Override + public String toString() { + return "NonAlignedChunkData{" + + "offset=" + + offset + + ", dataSize=" + + dataSize + + ", isHeadPageNeedDecode=" + + isHeadPageNeedDecode + + ", isTailPageNeedDecode=" + + isTailPageNeedDecode + + ", timePartitionSlot=" + + timePartitionSlot + + ", device='" + + device + + '\'' + + ", chunkHeader=" + + chunkHeader + + '}'; + } + } + + public static class AlignedChunkData implements ChunkData { + private List offset; + private List dataSize; + private boolean isHeadPageNeedDecode; + private boolean isTailPageNeedDecode; + + private TTimePartitionSlot timePartitionSlot; + private String device; + private List chunkHeaderList; + + public AlignedChunkData(long timeOffset, String device, ChunkHeader chunkHeader) { + this.offset = new ArrayList<>(); + this.dataSize = new ArrayList<>(); + this.isHeadPageNeedDecode = false; + this.isTailPageNeedDecode = false; + this.device = device; + this.chunkHeaderList = new ArrayList<>(); + + offset.add(timeOffset); + dataSize.add(0L); + chunkHeaderList.add(chunkHeader); + } + + @Override + public String getDevice() { + return device; + } + + @Override + public TTimePartitionSlot getTimePartitionSlot() { + return timePartitionSlot; + } + + @Override + public long getDataSize() { + return dataSize.stream().mapToLong(o -> o).sum(); + } + + @Override + public void addDataSize(long pageSize) { + dataSize.set(0, dataSize.get(0) + pageSize); + } + + @Override + public void setHeadPageNeedDecode(boolean headPageNeedDecode) { + isHeadPageNeedDecode = headPageNeedDecode; + } + + @Override + public void setTailPageNeedDecode(boolean tailPageNeedDecode) { + isTailPageNeedDecode = tailPageNeedDecode; + } + + @Override + public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { + this.timePartitionSlot = timePartitionSlot; + } + + public void addValueChunk(long offset, ChunkHeader chunkHeader) { + this.offset.add(offset); + this.dataSize.add(0L); + this.chunkHeaderList.add(chunkHeader); + } + + public void addValueChunkDataSize(long dataSize) { + int lastIndex = this.dataSize.size() - 1; + this.dataSize.set(lastIndex, this.dataSize.get(lastIndex) + dataSize); + } + + @Override + public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { + return null; + } + + @Override + public String toString() { + return "AlignedChunkData{" + + "offset=" + + offset + + ", dataSize=" + + dataSize + + ", isHeadPageNeedDecode=" + + isHeadPageNeedDecode + + ", isTailPageNeedDecode=" + + isTailPageNeedDecode + + ", timePartitionSlot=" + + timePartitionSlot + + ", device='" + + device + + '\'' + + ", chunkHeaderList=" + + chunkHeaderList + + '}'; + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java new file mode 100644 index 0000000000000..a77aff0c668c3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.scheduler.load; + +import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; +import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; + +import java.util.List; +import java.util.concurrent.Future; + +public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher { + @Override + public Future dispatch(List instances) { + return null; + } + + @Override + public void abort() {} +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java new file mode 100644 index 0000000000000..9fa3847c60ae3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.scheduler.load; + +import io.airlift.units.Duration; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.mpp.common.FragmentInstanceId; +import org.apache.iotdb.db.mpp.common.MPPQueryContext; +import org.apache.iotdb.db.mpp.common.PlanFragmentId; +import org.apache.iotdb.db.mpp.execution.fragment.FragmentInfo; +import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceState; +import org.apache.iotdb.db.mpp.plan.analyze.QueryType; +import org.apache.iotdb.db.mpp.plan.planner.plan.DistributedQueryPlan; +import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.mpp.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadSingleTsFileNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; +import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class LoadTsFileScheduler implements IScheduler { + private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); + + private String uuid; + private LoadTsFileDispatcherImpl dispatcher; + + private List tsFileNodeList; + private MPPQueryContext queryContext; + + public LoadTsFileScheduler( + DistributedQueryPlan distributedQueryPlan, MPPQueryContext queryContext) { + this.uuid = UUID.randomUUID().toString(); + this.tsFileNodeList = new ArrayList<>(); + this.queryContext = queryContext; + + for (FragmentInstance fragmentInstance : distributedQueryPlan.getInstances()) { + tsFileNodeList.add((LoadSingleTsFileNode) fragmentInstance.getFragment().getRoot()); + } + } + + @Override + public void start() {} + + private boolean firstPhase() { + for (LoadSingleTsFileNode node : tsFileNodeList) {} + return true; + } + + private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { + for (Map.Entry> entry : + node.getReplicaSet2Pieces().entrySet()) { + for (LoadTsFilePieceNode pieceNode : entry.getValue()) { + FragmentInstance instance = + new FragmentInstance( + new PlanFragment(null, pieceNode), + null, + null, + queryContext.getQueryType(), + queryContext.getTimeOut()); + instance.setDataRegionAndHost(entry.getKey()); + Future dispatchResultFuture = dispatcher.dispatch(Collections.singletonList(instance)); + + try { + FragInstanceDispatchResult result = dispatchResultFuture.get(); + if (!result.isSuccessful()) { + logger.error("Dispatch pieceNode error"); + return false; + } + } catch (InterruptedException | ExecutionException e) { + // If the dispatch request cannot be sent or TException is caught, we will retry this query. + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } +// stateMachine.transitionToFailed(e); + return false; + } + } + } + return true; + } + + private boolean secondPhase() { + return true; + } + + @Override + public void stop() {} + + @Override + public Duration getTotalCpuTime() { + return null; + } + + @Override + public FragmentInfo getFragmentInfo() { + return null; + } + + @Override + public void abortFragmentInstance(FragmentInstanceId instanceId, Throwable failureCause) {} + + @Override + public void cancelFragment(PlanFragmentId planFragmentId) {} +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java index 0a7276dfec446..55877d8d4b001 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java @@ -26,7 +26,7 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement; -import org.apache.iotdb.db.mpp.plan.statement.crud.LoadFileStatement; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.internal.SchemaFetchStatement; @@ -180,8 +180,8 @@ public R visitInsertTablet(InsertTabletStatement insertTabletStatement, C contex return visitStatement(insertTabletStatement, context); } - public R visitLoadFile(LoadFileStatement loadFileStatement, C context) { - return visitStatement(loadFileStatement, context); + public R visitLoadFile(LoadTsFileStatement loadTsFileStatement, C context) { + return visitStatement(loadTsFileStatement, context); } /** Data Control Language (DCL) */ diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java similarity index 96% rename from server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java rename to server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java index 289f0d555022b..4fb39cad1a71b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadFileStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; -public class LoadFileStatement extends Statement { +public class LoadTsFileStatement extends Statement { private File file; private boolean autoCreateSchema; private int sgLevel; @@ -41,7 +41,7 @@ public class LoadFileStatement extends Statement { private List tsFiles; - public LoadFileStatement(String filePath) { + public LoadTsFileStatement(String filePath) { this.file = new File(filePath); this.autoCreateSchema = true; this.sgLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel(); @@ -114,7 +114,7 @@ public R accept(StatementVisitor visitor, C context) { @Override public String toString() { - return "LoadFileStatement{" + return "LoadTsFileStatement{" + "file=" + file + ", autoCreateSchema=" diff --git a/server/src/main/java/org/apache/iotdb/db/sync/datasource/TsFileOpBlock.java b/server/src/main/java/org/apache/iotdb/db/sync/datasource/TsFileOpBlock.java index 1c7983e12854c..a224c8131bccb 100644 --- a/server/src/main/java/org/apache/iotdb/db/sync/datasource/TsFileOpBlock.java +++ b/server/src/main/java/org/apache/iotdb/db/sync/datasource/TsFileOpBlock.java @@ -73,7 +73,7 @@ public class TsFileOpBlock extends AbstractOpBlock { // full Timeseries Metadata TreeMap : FileOffset => Pair(DeviceId, TimeseriesMetadata) private Map> fullTsMetadataMap; - // TreeMap: LocalIndex => Pair(SensorFullPath, ChunkOffset, PointCount) + // TreeMap: LocalIndex => Pair(SensorFullPath, NonAlignedChunkData, PointCount) private TreeMap> indexToChunkInfoMap; Decoder timeDecoder = diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index 46d82d3a9f498..9a4fb0d2ed52a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -31,6 +31,7 @@ import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -299,4 +300,9 @@ public boolean checkIsChunkSizeOverThreshold(long threshold) { public TSDataType getCurrentValueChunkType() { return valueChunkWriterList.get(valueIndex).getDataType(); } + + @Override + public void serializeToDataOutputStream(DataOutputStream stream) throws IOException { + + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index d603c58efb446..35416b98ff45a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataOutputStream; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; @@ -457,4 +458,9 @@ public boolean isMerging() { public void setLastPoint(boolean isLastPoint) { this.isLastPoint = isLastPoint; } + + @Override + public void serializeToDataOutputStream(DataOutputStream stream) throws IOException { + + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java index 992862ed830ba..1564211cc3e9a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java @@ -20,6 +20,7 @@ import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +import java.io.DataOutputStream; import java.io.IOException; /** IChunkWriter provides a list of writing methods for different value types. */ @@ -28,6 +29,8 @@ public interface IChunkWriter { /** flush data to TsFileIOWriter. */ void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException; + void serializeToDataOutputStream(DataOutputStream stream) throws IOException; + /** estimate memory usage of this series. */ long estimateMaxSeriesMemSize(); From e07f821d3d2970434259af016223af228cf3f1a1 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 15 Aug 2022 15:05:33 +0800 Subject: [PATCH 03/37] working on dispatcher --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 8 ++--- .../db/mpp/plan/execution/QueryExecution.java | 4 +-- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 5 +-- .../mpp/plan/planner/LogicalPlanVisitor.java | 5 +-- .../plan/node/load/LoadSingleTsFileNode.java | 2 +- .../plan/node/load/LoadTsFileNode.java | 1 + .../plan/node/load/LoadTsFilePieceNode.java | 2 +- .../load/LoadTsFileDispatcherImpl.java | 18 +++++++++++ .../scheduler/load/LoadTsFileScheduler.java | 32 ++++++++++++------- .../write/chunk/AlignedChunkWriterImpl.java | 4 +-- .../tsfile/write/chunk/ChunkWriterImpl.java | 4 +-- 11 files changed, 54 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index da26cf4e04013..e9ca6570b5341 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -32,6 +32,8 @@ import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.apache.iotdb.db.service.thrift.impl.InfluxDBServiceImpl; import org.apache.iotdb.db.service.thrift.impl.TSServiceImpl; import org.apache.iotdb.db.wal.utils.WALMode; @@ -42,8 +44,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.fileSystem.FSType; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -728,9 +728,7 @@ public class IoTDBConfig { */ private long partitionInterval = 604800; - /** - * Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. - */ + /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */ private long maxPlanNodeSize = 500 * 1048576L; /** diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index d3eba9a3cab6d..e58d5a86738fe 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -51,8 +51,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeUtil; import org.apache.iotdb.db.mpp.plan.scheduler.ClusterScheduler; import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; -import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.mpp.plan.scheduler.StandaloneScheduler; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement; @@ -238,7 +238,7 @@ private Analysis analyze( private void schedule() { if (rawStatement instanceof LoadTsFileStatement) { - this.scheduler = new LoadTsFileScheduler(distributedPlan, context); + this.scheduler = new LoadTsFileScheduler(distributedPlan, context, internalServiceClientManager); this.scheduler.start(); return; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index e781de13e4a19..436d7021164c4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -1364,9 +1364,10 @@ public Statement visitLoadFile(IoTDBSqlParser.LoadFileContext ctx) { * @param ctx context of property statement */ private void parseLoadFiles( - LoadTsFileStatement loadTsFileStatement, IoTDBSqlParser.LoadFilesClauseContext ctx) { + LoadTsFileStatement loadTsFileStatement, IoTDBSqlParser.LoadFilesClauseContext ctx) { if (ctx.AUTOREGISTER() != null) { - loadTsFileStatement.setAutoCreateSchema(Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); + loadTsFileStatement.setAutoCreateSchema( + Boolean.parseBoolean(ctx.BOOLEAN_LITERAL().getText())); } else if (ctx.SGLEVEL() != null) { loadTsFileStatement.setSgLevel(Integer.parseInt(ctx.INTEGER_LITERAL().getText())); } else if (ctx.VERIFY() != null) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java index bdae4cadbef92..c7957f5552c5a 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.mpp.plan.analyze.ExpressionAnalyzer; import org.apache.iotdb.db.mpp.plan.expression.Expression; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.ActivateTemplateNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.CreateAlignedTimeSeriesNode; @@ -36,7 +37,6 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFileNode; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.OrderByParameter; import org.apache.iotdb.db.mpp.plan.statement.StatementNode; @@ -421,7 +421,8 @@ public PlanNode visitInsertRow(InsertRowStatement insertRowStatement, MPPQueryCo @Override public PlanNode visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { - return new LoadTsFileNode(context.getQueryId().genPlanNodeId(), loadTsFileStatement.getTsFiles()); + return new LoadTsFileNode( + context.getQueryId().genPlanNodeId(), loadTsFileStatement.getTsFiles()); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 40691b98dcebf..0a672048b523e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -45,7 +45,7 @@ import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index 9361ef53cc4a5..a6878ba921a3b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.tsfile.exception.NotImplementedException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index cd0e1254490b2..6821d21e8d17d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -39,12 +39,12 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index a77aff0c668c3..5a9f8a40948e1 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -19,6 +19,10 @@ package org.apache.iotdb.db.mpp.plan.scheduler.load; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; @@ -27,8 +31,22 @@ import java.util.concurrent.Future; public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher { + + private final String localhostIpAddr; + private final int localhostInternalPort; + private final IClientManager + internalServiceClientManager; + + public LoadTsFileDispatcherImpl( + IClientManager internalServiceClientManager) { + this.internalServiceClientManager = internalServiceClientManager; + this.localhostIpAddr = IoTDBDescriptor.getInstance().getConfig().getInternalAddress(); + this.localhostInternalPort = IoTDBDescriptor.getInstance().getConfig().getInternalPort(); + } + @Override public Future dispatch(List instances) { + return null; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 9fa3847c60ae3..7499e67044e54 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -19,14 +19,14 @@ package org.apache.iotdb.db.mpp.plan.scheduler.load; -import io.airlift.units.Duration; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.db.mpp.common.FragmentInstanceId; import org.apache.iotdb.db.mpp.common.MPPQueryContext; import org.apache.iotdb.db.mpp.common.PlanFragmentId; import org.apache.iotdb.db.mpp.execution.fragment.FragmentInfo; -import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceState; -import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.planner.plan.DistributedQueryPlan; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.mpp.plan.planner.plan.PlanFragment; @@ -34,6 +34,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; + +import io.airlift.units.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,16 +51,18 @@ public class LoadTsFileScheduler implements IScheduler { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); private String uuid; + private MPPQueryContext queryContext; private LoadTsFileDispatcherImpl dispatcher; - private List tsFileNodeList; - private MPPQueryContext queryContext; public LoadTsFileScheduler( - DistributedQueryPlan distributedQueryPlan, MPPQueryContext queryContext) { + DistributedQueryPlan distributedQueryPlan, + MPPQueryContext queryContext, + IClientManager internalServiceClientManager) { this.uuid = UUID.randomUUID().toString(); - this.tsFileNodeList = new ArrayList<>(); this.queryContext = queryContext; + this.tsFileNodeList = new ArrayList<>(); + this.dispatcher = new LoadTsFileDispatcherImpl(internalServiceClientManager); for (FragmentInstance fragmentInstance : distributedQueryPlan.getInstances()) { tsFileNodeList.add((LoadSingleTsFileNode) fragmentInstance.getFragment().getRoot()); @@ -69,7 +73,12 @@ public LoadTsFileScheduler( public void start() {} private boolean firstPhase() { - for (LoadSingleTsFileNode node : tsFileNodeList) {} + for (LoadSingleTsFileNode node : tsFileNodeList) { + if (!dispatchOneTsFile(node)) { + // TODO: record it + return false; + } + } return true; } @@ -85,20 +94,19 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { queryContext.getQueryType(), queryContext.getTimeOut()); instance.setDataRegionAndHost(entry.getKey()); - Future dispatchResultFuture = dispatcher.dispatch(Collections.singletonList(instance)); + Future dispatchResultFuture = + dispatcher.dispatch(Collections.singletonList(instance)); try { FragInstanceDispatchResult result = dispatchResultFuture.get(); if (!result.isSuccessful()) { - logger.error("Dispatch pieceNode error"); + // TODO: log error msg, and retry. return false; } } catch (InterruptedException | ExecutionException e) { - // If the dispatch request cannot be sent or TException is caught, we will retry this query. if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } -// stateMachine.transitionToFailed(e); return false; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index 9a4fb0d2ed52a..32c137ee1a168 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -302,7 +302,5 @@ public TSDataType getCurrentValueChunkType() { } @Override - public void serializeToDataOutputStream(DataOutputStream stream) throws IOException { - - } + public void serializeToDataOutputStream(DataOutputStream stream) throws IOException {} } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 35416b98ff45a..13e69800e6848 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -460,7 +460,5 @@ public void setLastPoint(boolean isLastPoint) { } @Override - public void serializeToDataOutputStream(DataOutputStream stream) throws IOException { - - } + public void serializeToDataOutputStream(DataOutputStream stream) throws IOException {} } From be7bd568dd7cf89b75003407ad1ebbf57607a868 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 15 Aug 2022 15:21:10 +0800 Subject: [PATCH 04/37] working on dispatcher --- .../db/mpp/plan/execution/QueryExecution.java | 3 ++- .../load/LoadTsFileDispatcherImpl.java | 25 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index e58d5a86738fe..f4c22d0041689 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -238,7 +238,8 @@ private Analysis analyze( private void schedule() { if (rawStatement instanceof LoadTsFileStatement) { - this.scheduler = new LoadTsFileScheduler(distributedPlan, context, internalServiceClientManager); + this.scheduler = + new LoadTsFileScheduler(distributedPlan, context, internalServiceClientManager); this.scheduler.start(); return; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index 5a9f8a40948e1..14130eb2b3ebc 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -19,18 +19,27 @@ package org.apache.iotdb.db.mpp.plan.scheduler.load; +import io.airlift.concurrent.SetThreadName; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; import java.util.concurrent.Future; +import static com.google.common.util.concurrent.Futures.immediateFuture; + public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher { + private static final Logger logger = LoggerFactory.getLogger(LoadTsFileDispatcherImpl.class); private final String localhostIpAddr; private final int localhostInternalPort; @@ -46,8 +55,20 @@ public LoadTsFileDispatcherImpl( @Override public Future dispatch(List instances) { - - return null; + for (FragmentInstance instance : instances) { + try (SetThreadName threadName = new SetThreadName(instance.getId().getFullId())) { + dispatchOneInstance(instance); + } catch (FragmentInstanceDispatchException e) { + return immediateFuture(new FragInstanceDispatchResult(e.getFailureStatus())); + } catch (Throwable t) { + logger.error("cannot dispatch FI for write operation", t); + return immediateFuture( + new FragInstanceDispatchResult( + RpcUtils.getStatus( + TSStatusCode.INTERNAL_SERVER_ERROR, "Unexpected errors: " + t.getMessage()))); + } + } + return immediateFuture(new FragInstanceDispatchResult(true)); } @Override From 1e35b5c4c504525e31df4aa174fbf2e34a71b6a9 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 15 Aug 2022 22:44:01 +0800 Subject: [PATCH 05/37] finish two phase --- .../plan/node/load/LoadSingleTsFileNode.java | 8 + .../plan/node/load/LoadTsFilePieceNode.java | 9 + .../load/LoadTsFileDispatcherImpl.java | 158 +++++++++++++++++- .../scheduler/load/LoadTsFileScheduler.java | 73 +++++++- .../impl/DataNodeInternalRPCServiceImpl.java | 13 ++ .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + thrift/src/main/thrift/datanode.thrift | 19 +++ 7 files changed, 270 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 0a672048b523e..141705c6eea8b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -130,6 +130,14 @@ public List splitByPartition(Analysis analysis) { throw new NotImplementedException("split load single TsFile is not implemented"); } + @Override + public String toString() { + return "LoadSingleTsFileNode{" + + "tsFile=" + tsFile + + ", replicaSets=" + replicaSet2Pieces.keySet() + + '}'; + } + public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOException { replicaSet2Pieces = new HashMap<>(); List chunkDataList = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 6821d21e8d17d..a2dd21c718f90 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -168,6 +168,15 @@ static ChunkData createChunkData( } } + @Override + public String toString() { + return "LoadTsFilePieceNode{" + + "tsFile=" + tsFile + + ", dataSize=" + dataSize + + ", chunkDataList=" + chunkDataList + + '}'; + } + public static class NonAlignedChunkData implements ChunkData { private long offset; private long dataSize; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index 14130eb2b3ebc..d35fc0936016d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -20,20 +20,34 @@ package org.apache.iotdb.db.mpp.plan.scheduler.load; import io.airlift.concurrent.SetThreadName; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; +import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; +import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; +import org.apache.iotdb.mpp.rpc.thrift.TLoadTsFileReq; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.Future; import static com.google.common.util.concurrent.Futures.immediateFuture; @@ -41,13 +55,16 @@ public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileDispatcherImpl.class); + private final String uuid; private final String localhostIpAddr; private final int localhostInternalPort; private final IClientManager internalServiceClientManager; public LoadTsFileDispatcherImpl( + String uuid, IClientManager internalServiceClientManager) { + this.uuid = uuid; this.internalServiceClientManager = internalServiceClientManager; this.localhostIpAddr = IoTDBDescriptor.getInstance().getConfig().getInternalAddress(); this.localhostInternalPort = IoTDBDescriptor.getInstance().getConfig().getInternalPort(); @@ -56,12 +73,123 @@ public LoadTsFileDispatcherImpl( @Override public Future dispatch(List instances) { for (FragmentInstance instance : instances) { - try (SetThreadName threadName = new SetThreadName(instance.getId().getFullId())) { + try (SetThreadName threadName = + new SetThreadName(LoadTsFileScheduler.class.getName() + instance.getId().getFullId())) { dispatchOneInstance(instance); } catch (FragmentInstanceDispatchException e) { return immediateFuture(new FragInstanceDispatchResult(e.getFailureStatus())); } catch (Throwable t) { - logger.error("cannot dispatch FI for write operation", t); + logger.error("cannot dispatch FI for load operation", t); + return immediateFuture( + new FragInstanceDispatchResult( + RpcUtils.getStatus( + TSStatusCode.INTERNAL_SERVER_ERROR, "Unexpected errors: " + t.getMessage()))); + } + } + return immediateFuture(new FragInstanceDispatchResult(true)); + } + + private void dispatchOneInstance(FragmentInstance instance) + throws FragmentInstanceDispatchException { + for (TDataNodeLocation dataNodeLocation : instance.getDataRegionId().getDataNodeLocations()) { + TEndPoint endPoint = dataNodeLocation.getInternalEndPoint(); + if (isDispatchedToLocal(endPoint)) { + dispatchLocally(instance); + } else { + dispatchRemote(instance, endPoint); + } + } + } + + private boolean isDispatchedToLocal(TEndPoint endPoint) { + return this.localhostIpAddr.equals(endPoint.getIp()) && localhostInternalPort == endPoint.port; + } + + private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) + throws FragmentInstanceDispatchException { + try (SyncDataNodeInternalServiceClient client = + internalServiceClientManager.borrowClient(endPoint)) { + TLoadTsFileReq loadTsFileReq = + new TLoadTsFileReq(instance.getFragment().getRoot().serializeToByteBuffer(), uuid); + TLoadResp loadResp = client.sendLoadNode(loadTsFileReq); + if (!LoadTsFileScheduler.LoadResult.SUCCESS.equals( + LoadTsFileScheduler.LoadResult.values()[loadResp.loadRespStatus])) { + logger.error(loadResp.info); + TSStatus status = new TSStatus(); + status.setCode(TSStatusCode.LOAD_TSFILE_ERROR.getStatusCode()); + status.setMessage(loadResp.info); + throw new FragmentInstanceDispatchException(status); + } + } catch (IOException | TException e) { + logger.error("can't connect to node {}", endPoint, e); + TSStatus status = new TSStatus(); + status.setCode(TSStatusCode.SYNC_CONNECTION_EXCEPTION.getStatusCode()); + status.setMessage("can't connect to node {}" + endPoint); + throw new FragmentInstanceDispatchException(status); + } + } + + private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDispatchException { + PlanNode planNode = instance.getFragment().getRoot(); + boolean hasFailedMeasurement = false; + String partialInsertMessage = null; + if (planNode instanceof InsertNode) { + InsertNode insertNode = (InsertNode) planNode; + try { + SchemaValidator.validate(insertNode); + } catch (SemanticException e) { + throw new FragmentInstanceDispatchException(e); + } + hasFailedMeasurement = insertNode.hasFailedMeasurements(); + if (hasFailedMeasurement) { + partialInsertMessage = + String.format( + "Fail to insert measurements %s caused by %s", + insertNode.getFailedMeasurements(), insertNode.getFailedMessages()); + logger.warn(partialInsertMessage); + } + } + // TODO: write to local interface + + // ConsensusWriteResponse writeResponse; + // if (groupId instanceof DataRegionId) { + // writeResponse = DataRegionConsensusImpl.getInstance().write(groupId, planNode); + // } else { + // writeResponse = SchemaRegionConsensusImpl.getInstance().write(groupId, planNode); + // } + // + // if (writeResponse.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + // logger.error(writeResponse.getStatus().message); + // throw new FragmentInstanceDispatchException(writeResponse.getStatus()); + // } else if (hasFailedMeasurement) { + // throw new FragmentInstanceDispatchException( + // RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(), + // partialInsertMessage)); + // } + } + + public Future dispatchCommand( + TLoadCommandReq loadCommandReq, Set replicaSets) { + Set allEndPoint = new HashSet<>(); + for (TRegionReplicaSet replicaSet : replicaSets) { + for (TDataNodeLocation dataNodeLocation : replicaSet.getDataNodeLocations()) { + allEndPoint.add(dataNodeLocation.getInternalEndPoint()); + } + } + + for (TEndPoint endPoint : allEndPoint) { + try (SetThreadName threadName = + new SetThreadName( + LoadTsFileScheduler.class.getName() + "-" + loadCommandReq.commandType)) { + if (isDispatchedToLocal(endPoint)) { + dispatchLocally(loadCommandReq); + } else { + dispatchRemote(loadCommandReq, endPoint); + } + } catch (FragmentInstanceDispatchException e) { + return immediateFuture(new FragInstanceDispatchResult(e.getFailureStatus())); + } catch (Throwable t) { + logger.error("cannot dispatch LoadCommand for load operation", t); return immediateFuture( new FragInstanceDispatchResult( RpcUtils.getStatus( @@ -71,6 +199,32 @@ public Future dispatch(List instan return immediateFuture(new FragInstanceDispatchResult(true)); } + private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint) + throws FragmentInstanceDispatchException { + try (SyncDataNodeInternalServiceClient client = + internalServiceClientManager.borrowClient(endPoint)) { + TLoadResp loadResp = client.sendLoadCommand(loadCommandReq); + if (!LoadTsFileScheduler.LoadResult.SUCCESS.equals( + LoadTsFileScheduler.LoadResult.values()[loadResp.loadRespStatus])) { + logger.error(loadResp.info); + TSStatus status = new TSStatus(); + status.setCode(TSStatusCode.LOAD_TSFILE_ERROR.getStatusCode()); + status.setMessage(loadResp.info); + throw new FragmentInstanceDispatchException(status); + } + } catch (IOException | TException e) { + logger.error("can't connect to node {}", endPoint, e); + TSStatus status = new TSStatus(); + status.setCode(TSStatusCode.SYNC_CONNECTION_EXCEPTION.getStatusCode()); + status.setMessage("can't connect to node {}" + endPoint); + throw new FragmentInstanceDispatchException(status); + } + } + + private void dispatchLocally(TLoadCommandReq loadCommandReq) throws FragmentInstanceDispatchException { + // TODO: use local interface + } + @Override public void abort() {} } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 7499e67044e54..aca6a2433488e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -36,13 +36,16 @@ import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; import io.airlift.units.Duration; +import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -50,10 +53,13 @@ public class LoadTsFileScheduler implements IScheduler { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); - private String uuid; - private MPPQueryContext queryContext; + private final String uuid; + private final MPPQueryContext queryContext; private LoadTsFileDispatcherImpl dispatcher; private List tsFileNodeList; + private PlanFragmentId fragmentId; + + private Set allReplicaSets; public LoadTsFileScheduler( DistributedQueryPlan distributedQueryPlan, @@ -62,7 +68,9 @@ public LoadTsFileScheduler( this.uuid = UUID.randomUUID().toString(); this.queryContext = queryContext; this.tsFileNodeList = new ArrayList<>(); - this.dispatcher = new LoadTsFileDispatcherImpl(internalServiceClientManager); + this.fragmentId = distributedQueryPlan.getRootSubPlan().getPlanFragment().getId(); + this.dispatcher = new LoadTsFileDispatcherImpl(uuid, internalServiceClientManager); + this.allReplicaSets = new HashSet<>(); for (FragmentInstance fragmentInstance : distributedQueryPlan.getInstances()) { tsFileNodeList.add((LoadSingleTsFileNode) fragmentInstance.getFragment().getRoot()); @@ -70,12 +78,16 @@ public LoadTsFileScheduler( } @Override - public void start() {} + public void start() { + boolean isFirstPhaseSuccess = firstPhase(); + boolean isSuccess = secondPhase(isFirstPhaseSuccess); + } private boolean firstPhase() { for (LoadSingleTsFileNode node : tsFileNodeList) { if (!dispatchOneTsFile(node)) { - // TODO: record it + logger.error( + String.format("Dispatch Single TsFile Node error, LoadSingleTsFileNode %s.", node)); return false; } } @@ -85,11 +97,12 @@ private boolean firstPhase() { private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { for (Map.Entry> entry : node.getReplicaSet2Pieces().entrySet()) { + allReplicaSets.add(entry.getKey()); for (LoadTsFilePieceNode pieceNode : entry.getValue()) { FragmentInstance instance = new FragmentInstance( - new PlanFragment(null, pieceNode), - null, + new PlanFragment(fragmentId, pieceNode), + fragmentId.genFragmentInstanceId(), null, queryContext.getQueryType(), queryContext.getTimeOut()); @@ -100,13 +113,21 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { try { FragInstanceDispatchResult result = dispatchResultFuture.get(); if (!result.isSuccessful()) { - // TODO: log error msg, and retry. + // TODO: retry. + logger.error( + String.format( + "Dispatch one piece to ReplicaSet %s error, result status code %s.", + entry.getKey(), result.getFailureStatus().getCode())); + logger.error( + String.format("Result message %s.", result.getFailureStatus().getMessage())); + logger.error(String.format("Dispatch piece node:%n%s", pieceNode)); return false; } } catch (InterruptedException | ExecutionException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } + logger.warn("Interrupt or Execution error.", e); return false; } } @@ -114,7 +135,31 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { return true; } - private boolean secondPhase() { + private boolean secondPhase(boolean isFirstPhaseSuccess) { + TLoadCommandReq loadCommandReq = + new TLoadCommandReq( + (isFirstPhaseSuccess ? LoadCommand.EXECUTE : LoadCommand.ROLLBACK).ordinal(), uuid); + Future dispatchResultFuture = + dispatcher.dispatchCommand(loadCommandReq, allReplicaSets); + + try { + FragInstanceDispatchResult result = dispatchResultFuture.get(); + if (!result.isSuccessful()) { + // TODO: retry. + logger.error( + String.format("Dispatch LoadCommand error to replicaSets %s error.", allReplicaSets)); + logger.error(String.format("Result status code %s.", result.getFailureStatus().getCode())); + logger.error( + String.format("Result status message %s.", result.getFailureStatus().getMessage())); + return false; + } + } catch (InterruptedException | ExecutionException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.warn("Interrupt or Execution error.", e); + return false; + } return true; } @@ -136,4 +181,14 @@ public void abortFragmentInstance(FragmentInstanceId instanceId, Throwable failu @Override public void cancelFragment(PlanFragmentId planFragmentId) {} + + public enum LoadCommand { + EXECUTE, + ROLLBACK + } + + public enum LoadResult { + SUCCESS, + FAILED + } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index a7c9a06d23802..a339f8bec5ce8 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -91,6 +91,9 @@ import org.apache.iotdb.mpp.rpc.thrift.THeartbeatResp; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; +import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; +import org.apache.iotdb.mpp.rpc.thrift.TLoadTsFileReq; import org.apache.iotdb.mpp.rpc.thrift.TMigrateRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -297,6 +300,16 @@ public TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) { throw new UnsupportedOperationException(); } + @Override + public TLoadResp sendLoadNode(TLoadTsFileReq req) throws TException { + return null; + } + + @Override + public TLoadResp sendLoadCommand(TLoadCommandReq req) throws TException { + return null; + } + @Override public TSStatus createSchemaRegion(TCreateSchemaRegionReq req) { TSStatus tsStatus; diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index b455746b0961c..2d71d7d32a567 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -125,6 +125,7 @@ public enum TSStatusCode { CREATE_REGION_ERROR(711), DELETE_REGION_ERROR(712), CACHE_UPDATE_FAIL(713), + LOAD_TSFILE_ERROR(714), // configuration CONFIG_ERROR(800), diff --git a/thrift/src/main/thrift/datanode.thrift b/thrift/src/main/thrift/datanode.thrift index 5f436c12e3422..cca64744c6d2b 100644 --- a/thrift/src/main/thrift/datanode.thrift +++ b/thrift/src/main/thrift/datanode.thrift @@ -203,6 +203,21 @@ struct TUpdateTemplateReq{ 2: required binary templateInfo } +struct TLoadTsFileReq{ + 1: required binary body + 2: required string uuid +} + +struct TLoadCommandReq{ + 1: required i32 commandType + 2: required string uuid +} + +struct TLoadResp{ + 1: required i32 loadRespStatus + 2: required string info +} + service IDataNodeRPCService { // -----------------------------------For Data Node----------------------------------------------- @@ -227,6 +242,10 @@ service IDataNodeRPCService { TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) + TLoadResp sendLoadNode(TLoadTsFileReq req); + + TLoadResp sendLoadCommand(TLoadCommandReq req); + // -----------------------------------For Config Node----------------------------------------------- From 4a062b04a5602d3570255d3bad1474ad3146495d Mon Sep 17 00:00:00 2001 From: yschengzi Date: Tue, 16 Aug 2022 21:04:22 +0800 Subject: [PATCH 06/37] finish framework --- .../iotdb/commons/conf/IoTDBConstant.java | 1 + .../org/apache/iotdb/db/conf/IoTDBConfig.java | 11 + .../iotdb/db/engine/StorageEngineV2.java | 68 ++++ .../db/engine/load/AlignedChunkData.java | 126 ++++++++ .../iotdb/db/engine/load/ChunkData.java | 53 ++++ .../db/engine/load/LoadTsFileManager.java | 202 ++++++++++++ .../db/engine/load/NonAlignedChunkData.java | 205 ++++++++++++ .../plan/node/load/LoadSingleTsFileNode.java | 5 +- .../plan/node/load/LoadTsFilePieceNode.java | 293 +----------------- .../load/LoadTsFileDispatcherImpl.java | 40 ++- .../scheduler/load/LoadTsFileScheduler.java | 20 +- .../impl/DataNodeInternalRPCServiceImpl.java | 29 +- .../org/apache/iotdb/rpc/TSStatusCode.java | 2 +- thrift/src/main/thrift/datanode.thrift | 7 +- 14 files changed, 747 insertions(+), 315 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 41013228ac078..b975c4a0072c7 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -189,6 +189,7 @@ private IoTDBConstant() {} // system folder name public static final String SYSTEM_FOLDER_NAME = "system"; public static final String SCHEMA_FOLDER_NAME = "schema"; + public static final String LOAD_TSFILE_FOLDER_NAME = "load"; public static final String SYNC_FOLDER_NAME = "sync"; public static final String QUERY_FOLDER_NAME = "query"; public static final String TRACING_FOLDER_NAME = "tracing"; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index e9ca6570b5341..ea0ac0982500e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -242,6 +242,8 @@ public class IoTDBConfig { + File.separator + IoTDBConstant.SCHEMA_FOLDER_NAME; + private String loadTsFileDir = systemDir + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME; + /** Sync directory, including the log and hardlink tsfiles */ private String syncDir = IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.SYNC_FOLDER_NAME; @@ -1061,6 +1063,7 @@ void updatePath() { private void formulateFolders() { systemDir = addHomeDir(systemDir); schemaDir = addHomeDir(schemaDir); + loadTsFileDir = addHomeDir(loadTsFileDir); syncDir = addHomeDir(syncDir); tracingDir = addHomeDir(tracingDir); consensusDir = addHomeDir(consensusDir); @@ -1212,6 +1215,14 @@ void setSystemDir(String systemDir) { this.systemDir = systemDir; } + public String getLoadTsFileDir() { + return loadTsFileDir; + } + + public void setLoadTsFileDir(String loadTsFileDir) { + this.loadTsFileDir = loadTsFileDir; + } + public String getSchemaDir() { return schemaDir; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index 2c51263a1e958..289daec006ff6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.engine.flush.FlushListener; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy; +import org.apache.iotdb.db.engine.load.LoadTsFileManager; import org.apache.iotdb.db.engine.storagegroup.DataRegion; import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; import org.apache.iotdb.db.exception.DataRegionException; @@ -48,6 +49,8 @@ import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.db.utils.ThreadUtils; import org.apache.iotdb.db.utils.UpgradeUtils; @@ -56,6 +59,7 @@ import org.apache.iotdb.db.wal.recover.WALRecoverManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.utils.FilePathUtils; import org.apache.commons.io.FileUtils; @@ -131,6 +135,8 @@ public class StorageEngineV2 implements IService { private List customFlushListeners = new ArrayList<>(); private int recoverDataRegionNum = 0; + private LoadTsFileManager loadTsFileManager = new LoadTsFileManager(); + private StorageEngineV2() {} public static StorageEngineV2 getInstance() { @@ -723,6 +729,68 @@ public TsFileFlushPolicy getFileFlushPolicy() { return fileFlushPolicy; } + public TSStatus writeLoadTsFileNode( + DataRegionId dataRegionId, LoadTsFilePieceNode pieceNode, String uuid) { + TSStatus status = new TSStatus(); + + try { + loadTsFileManager.writeToDataRegion(getDataRegion(dataRegionId), pieceNode, uuid); + } catch (PageException e) { + logger.error( + String.format( + "Parse Page error when writing piece node of TsFile %s to DataRegion %s.", + pieceNode.getTsFile(), dataRegionId), + e); + status.setCode(TSStatusCode.TSFILE_RUNTIME_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return status; + } catch (IOException e) { + logger.error( + String.format( + "IO error when writing piece node of TsFile %s to DataRegion %s.", + pieceNode.getTsFile(), dataRegionId), + e); + status.setCode(TSStatusCode.DATA_REGION_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return status; + } + + return RpcUtils.SUCCESS_STATUS; + } + + public TSStatus executeLoadCommand(LoadTsFileScheduler.LoadCommand loadCommand, String uuid) { + TSStatus status = new TSStatus(); + + try { + switch (loadCommand) { + case EXECUTE: + if (loadTsFileManager.loadAll(uuid)) { + status.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(String.format("No uuid %s recorded.", uuid)); + } + break; + case ROLLBACK: + if (loadTsFileManager.deleteAll(uuid)) { + status.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(String.format("No uuid %s recorded.", uuid)); + } + break; + default: + status.setCode(TSStatusCode.ILLEGAL_PARAMETER.getStatusCode()); + status.setMessage(String.format("Wrong load command %s.", loadCommand)); + } + } catch (IOException e) { + status.setCode(TSStatusCode.DATA_REGION_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + + return RpcUtils.SUCCESS_STATUS; + } + static class InstanceHolder { private static final StorageEngineV2 INSTANCE = new StorageEngineV2(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java new file mode 100644 index 0000000000000..c9e5345e0adf0 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.load; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class AlignedChunkData implements ChunkData { + private List offset; + private List dataSize; + private boolean isHeadPageNeedDecode; + private boolean isTailPageNeedDecode; + + private TTimePartitionSlot timePartitionSlot; + private String device; + private List chunkHeaderList; + + public AlignedChunkData(long timeOffset, String device, ChunkHeader chunkHeader) { + this.offset = new ArrayList<>(); + this.dataSize = new ArrayList<>(); + this.isHeadPageNeedDecode = false; + this.isTailPageNeedDecode = false; + this.device = device; + this.chunkHeaderList = new ArrayList<>(); + + offset.add(timeOffset); + dataSize.add(0L); + chunkHeaderList.add(chunkHeader); + } + + @Override + public String getDevice() { + return device; + } + + @Override + public TTimePartitionSlot getTimePartitionSlot() { + return timePartitionSlot; + } + + @Override + public long getDataSize() { + return dataSize.stream().mapToLong(o -> o).sum(); + } + + @Override + public void addDataSize(long pageSize) { + dataSize.set(0, dataSize.get(0) + pageSize); + } + + @Override + public void setHeadPageNeedDecode(boolean headPageNeedDecode) { + isHeadPageNeedDecode = headPageNeedDecode; + } + + @Override + public void setTailPageNeedDecode(boolean tailPageNeedDecode) { + isTailPageNeedDecode = tailPageNeedDecode; + } + + @Override + public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { + this.timePartitionSlot = timePartitionSlot; + } + + public void addValueChunk(long offset, ChunkHeader chunkHeader) { + this.offset.add(offset); + this.dataSize.add(0L); + this.chunkHeaderList.add(chunkHeader); + } + + public void addValueChunkDataSize(long dataSize) { + int lastIndex = this.dataSize.size() - 1; + this.dataSize.set(lastIndex, this.dataSize.get(lastIndex) + dataSize); + } + + @Override + public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { + return null; + } + + @Override + public String toString() { + return "AlignedChunkData{" + + "offset=" + + offset + + ", dataSize=" + + dataSize + + ", isHeadPageNeedDecode=" + + isHeadPageNeedDecode + + ", isTailPageNeedDecode=" + + isTailPageNeedDecode + + ", timePartitionSlot=" + + timePartitionSlot + + ", device='" + + device + + '\'' + + ", chunkHeaderList=" + + chunkHeaderList + + '}'; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java new file mode 100644 index 0000000000000..8011528fff1d6 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.load; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; + +import java.io.File; +import java.io.IOException; + +public interface ChunkData { + String getDevice(); + + TTimePartitionSlot getTimePartitionSlot(); + + long getDataSize(); + + void addDataSize(long pageSize); + + void setHeadPageNeedDecode(boolean headPageNeedDecode); + + void setTailPageNeedDecode(boolean tailPageNeedDecode); + + void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot); + + IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException; + + static ChunkData createChunkData( + boolean isAligned, long offset, String device, ChunkHeader chunkHeader) { + return isAligned + ? new AlignedChunkData(offset, device, chunkHeader) + : new NonAlignedChunkData(offset, device, chunkHeader); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java new file mode 100644 index 0000000000000..1c422e2cffc6f --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.load; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.storagegroup.DataRegion; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +public class LoadTsFileManager { + private static final Logger logger = LoggerFactory.getLogger(LoadTsFileManager.class); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final File loadDir; + + private Map uuid2WriterManager; + private Map dataPartition2NextTsFileIndex; + + private final ReentrantLock lock; + + public LoadTsFileManager() { + this.loadDir = SystemFileFactory.INSTANCE.getFile(config.getLoadTsFileDir()); + this.uuid2WriterManager = new HashMap<>(); + this.dataPartition2NextTsFileIndex = new HashMap<>(); + this.lock = new ReentrantLock(); + + clearDir(loadDir); + } + + private void clearDir(File dir) { + if (dir.delete()) { + logger.info(String.format("Delete origin load TsFile dir %s.", dir.getPath())); + } + if (dir.mkdirs()) { + logger.warn(String.format("load TsFile dir %s can not be created.", dir.getPath())); + } + } + + public void writeToDataRegion(DataRegion dataRegion, LoadTsFilePieceNode pieceNode, String uuid) + throws PageException, IOException { + dataRegion.getLogicalStorageGroupName(); + dataRegion.getDataRegionId(); + TsFileWriterManager writerManager = + uuid2WriterManager.computeIfAbsent( + uuid, o -> new TsFileWriterManager(SystemFileFactory.INSTANCE.getFile(loadDir, uuid))); + for (ChunkData chunkData : pieceNode.getAllChunkData()) { + writerManager.write( + getDataPartition( + dataRegion.getLogicalStorageGroupName(), + dataRegion.getDataRegionId(), + chunkData.getTimePartitionSlot()), + chunkData); + } + } + + public boolean loadAll(String uuid) throws IOException { + if (!uuid2WriterManager.containsKey(uuid)) { + return false; + } + uuid2WriterManager.get(uuid).loadAll(); + return true; + } + + public boolean deleteAll(String uuid) throws IOException { + if (!uuid2WriterManager.containsKey(uuid)) { + return false; + } + uuid2WriterManager.get(uuid).close(); + return true; + } + + private String getDataPartition( + String logicalName, String dataRegionId, TTimePartitionSlot timePartitionSlot) { + return String.join( + IoTDBConstant.FILE_NAME_SEPARATOR, + logicalName, + dataRegionId, + Long.toString(timePartitionSlot.getStartTime())); + } + + private String getNewTsFileName(String dataPartition) { + lock.lock(); + try { + int nextIndex = dataPartition2NextTsFileIndex.getOrDefault(dataPartition, 0) + 1; + dataPartition2NextTsFileIndex.put(dataPartition, nextIndex); + return dataPartition + + IoTDBConstant.FILE_NAME_SEPARATOR + + nextIndex + + TsFileConstant.TSFILE_SUFFIX; + } finally { + lock.unlock(); + } + } + + private class TsFileWriterManager { + private final File taskDir; + private Map dataPartition2Writer; + private Map dataPartition2LastDevice; + + private TsFileWriterManager(File taskDir) { + this.taskDir = taskDir; + this.dataPartition2Writer = new HashMap<>(); + this.dataPartition2LastDevice = new HashMap<>(); + + clearDir(taskDir); + } + + private void write(String dataPartition, ChunkData chunkData) + throws IOException, PageException { + if (!dataPartition2Writer.containsKey(dataPartition)) { + File newTsFile = + SystemFileFactory.INSTANCE.getFile(taskDir, getNewTsFileName(dataPartition)); + if (!newTsFile.createNewFile()) { + logger.error(String.format("Can not create TsFile %s for writing.", newTsFile.getPath())); + return; + } + + dataPartition2Writer.put(dataPartition, new TsFileIOWriter(newTsFile)); + } + TsFileIOWriter writer = dataPartition2Writer.get(dataPartition); + if (!chunkData.getDevice().equals(dataPartition2LastDevice.getOrDefault(dataPartition, ""))) { + if (dataPartition2LastDevice.containsKey(dataPartition)) { + writer.endChunkGroup(); + } + writer.startChunkGroup(chunkData.getDevice()); + } + chunkData.getChunkWriter(null).writeToFileWriter(writer); // TODO: get writer + } + + private void loadAll() throws IOException { + for (Map.Entry entry : dataPartition2Writer.entrySet()) { + TsFileIOWriter writer = entry.getValue(); + if (writer.isWritingChunkGroup()) { + writer.endChunkGroup(); + } + writer.endFile(); + generateResource(writer); // TODO: load TsFileResource + } + } + + private TsFileResource generateResource(TsFileIOWriter writer) throws IOException { + TsFileResource tsFileResource = new TsFileResource(writer.getFile()); + Map> deviceTimeseriesMetadataMap = + writer.getDeviceTimeseriesMetadataMap(); + for (Map.Entry> entry : + deviceTimeseriesMetadataMap.entrySet()) { + String device = entry.getKey(); + for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { + tsFileResource.updateStartTime(device, timeseriesMetaData.getStatistics().getStartTime()); + tsFileResource.updateEndTime(device, timeseriesMetaData.getStatistics().getEndTime()); + } + } + tsFileResource.setStatus(TsFileResourceStatus.CLOSED); + tsFileResource.serialize(); + return tsFileResource; + } + + private void close() throws IOException { + for (Map.Entry entry : dataPartition2Writer.entrySet()) { + entry.getValue().close(); + } + if (taskDir.delete()) { + logger.warn(String.format("Can not delete load uuid dir %s.", taskDir.getPath())); + } + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java new file mode 100644 index 0000000000000..234282d9b65cd --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.load; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.MetaMarker; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.reader.page.PageReader; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class NonAlignedChunkData implements ChunkData { + private long offset; + private long dataSize; + private boolean isHeadPageNeedDecode; + private boolean isTailPageNeedDecode; + + private TTimePartitionSlot timePartitionSlot; + private String device; + private ChunkHeader chunkHeader; + + public NonAlignedChunkData(long offset, String device, ChunkHeader chunkHeader) { + this.offset = offset; + this.dataSize = 0; + this.isHeadPageNeedDecode = false; + this.isTailPageNeedDecode = false; + this.device = device; + this.chunkHeader = chunkHeader; + } + + @Override + public String getDevice() { + return device; + } + + @Override + public TTimePartitionSlot getTimePartitionSlot() { + return timePartitionSlot; + } + + @Override + public long getDataSize() { + return dataSize; + } + + @Override + public void addDataSize(long pageSize) { + dataSize += pageSize; + } + + @Override + public void setHeadPageNeedDecode(boolean headPageNeedDecode) { + isHeadPageNeedDecode = headPageNeedDecode; + } + + @Override + public void setTailPageNeedDecode(boolean tailPageNeedDecode) { + isTailPageNeedDecode = tailPageNeedDecode; + } + + @Override + public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { + this.timePartitionSlot = timePartitionSlot; + } + + @Override + public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema( + chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), + chunkHeader.getEncodingType(), + chunkHeader.getCompressionType())); + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + reader.position(offset); + long dataSize = this.dataSize; + while (dataSize > 0) { + PageHeader pageHeader = + reader.readPageHeader( + chunkHeader.getDataType(), + (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + long pageDataSize = pageHeader.getSerializedPageSize(); + if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page + || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page + decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); + } else { // entire page + ByteBuffer pageData = reader.readCompressedPage(pageHeader); + chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); + } + dataSize -= pageDataSize; + } + return chunkWriter; + } + } + + private void decodePage( + TsFileSequenceReader reader, + PageHeader pageHeader, + Decoder timeDecoder, + Decoder valueDecoder, + ChunkWriterImpl chunkWriter) + throws IOException { + valueDecoder.reset(); + ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); + PageReader pageReader = + new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + long time = batchData.currentTime(); + if (time < timePartitionSlot.getStartTime()) { + continue; + } else if (time > timePartitionSlot.getStartTime()) { + break; + } + + Object value = batchData.currentValue(); + switch (chunkHeader.getDataType()) { + case INT32: + chunkWriter.write(time, (int) value); + break; + case INT64: + chunkWriter.write(time, (long) value); + break; + case FLOAT: + chunkWriter.write(time, (float) value); + break; + case DOUBLE: + chunkWriter.write(time, (double) value); + break; + case BOOLEAN: + chunkWriter.write(time, (boolean) value); + break; + case TEXT: + chunkWriter.write(time, (Binary) value); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", chunkHeader.getDataType())); + } + batchData.next(); + } + chunkWriter.sealCurrentPage(); + } + + @Override + public String toString() { + return "NonAlignedChunkData{" + + "offset=" + + offset + + ", dataSize=" + + dataSize + + ", isHeadPageNeedDecode=" + + isHeadPageNeedDecode + + ", isTailPageNeedDecode=" + + isTailPageNeedDecode + + ", timePartitionSlot=" + + timePartitionSlot + + ", device='" + + device + + '\'' + + ", chunkHeader=" + + chunkHeader + + '}'; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 141705c6eea8b..3d111505e48fd 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -23,12 +23,12 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.db.engine.load.AlignedChunkData; +import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode.AlignedChunkData; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode.ChunkData; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -45,7 +45,6 @@ import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index a2dd21c718f90..3b3da953a6883 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; @@ -85,6 +86,14 @@ public void addChunkData(ChunkData chunkData) { dataSize += chunkData.getDataSize(); } + public List getAllChunkData() { + return chunkDataList; + } + + public File getTsFile() { + return tsFile; + } + @Override public TRegionReplicaSet getRegionReplicaSet() { return null; @@ -143,31 +152,6 @@ public List splitByPartition(Analysis analysis) { throw new NotImplementedException("split load piece TsFile is not implemented"); } - public interface ChunkData { - String getDevice(); - - TTimePartitionSlot getTimePartitionSlot(); - - long getDataSize(); - - void addDataSize(long pageSize); - - void setHeadPageNeedDecode(boolean headPageNeedDecode); - - void setTailPageNeedDecode(boolean tailPageNeedDecode); - - void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot); - - IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException; - - static ChunkData createChunkData( - boolean isAligned, long offset, String device, ChunkHeader chunkHeader) { - return isAligned - ? new AlignedChunkData(offset, device, chunkHeader) - : new NonAlignedChunkData(offset, device, chunkHeader); - } - } - @Override public String toString() { return "LoadTsFilePieceNode{" + @@ -176,263 +160,4 @@ public String toString() { ", chunkDataList=" + chunkDataList + '}'; } - - public static class NonAlignedChunkData implements ChunkData { - private long offset; - private long dataSize; - private boolean isHeadPageNeedDecode; - private boolean isTailPageNeedDecode; - - private TTimePartitionSlot timePartitionSlot; - private String device; - private ChunkHeader chunkHeader; - - public NonAlignedChunkData(long offset, String device, ChunkHeader chunkHeader) { - this.offset = offset; - this.dataSize = 0; - this.isHeadPageNeedDecode = false; - this.isTailPageNeedDecode = false; - this.device = device; - this.chunkHeader = chunkHeader; - } - - @Override - public String getDevice() { - return device; - } - - @Override - public TTimePartitionSlot getTimePartitionSlot() { - return timePartitionSlot; - } - - @Override - public long getDataSize() { - return dataSize; - } - - @Override - public void addDataSize(long pageSize) { - dataSize += pageSize; - } - - @Override - public void setHeadPageNeedDecode(boolean headPageNeedDecode) { - isHeadPageNeedDecode = headPageNeedDecode; - } - - @Override - public void setTailPageNeedDecode(boolean tailPageNeedDecode) { - isTailPageNeedDecode = tailPageNeedDecode; - } - - @Override - public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { - this.timePartitionSlot = timePartitionSlot; - } - - @Override - public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { - ChunkWriterImpl chunkWriter = - new ChunkWriterImpl( - new MeasurementSchema( - chunkHeader.getMeasurementID(), - chunkHeader.getDataType(), - chunkHeader.getEncodingType(), - chunkHeader.getCompressionType())); - try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { - Decoder defaultTimeDecoder = - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64); - Decoder valueDecoder = - Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); - - reader.position(offset); - long dataSize = this.dataSize; - while (dataSize > 0) { - PageHeader pageHeader = - reader.readPageHeader( - chunkHeader.getDataType(), - (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); - long pageDataSize = pageHeader.getSerializedPageSize(); - if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page - || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page - decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); - } else { // entire page - ByteBuffer pageData = reader.readCompressedPage(pageHeader); - chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); - } - dataSize -= pageDataSize; - } - return chunkWriter; - } - } - - private void decodePage( - TsFileSequenceReader reader, - PageHeader pageHeader, - Decoder timeDecoder, - Decoder valueDecoder, - ChunkWriterImpl chunkWriter) - throws IOException { - valueDecoder.reset(); - ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); - PageReader pageReader = - new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - long time = batchData.currentTime(); - if (time < timePartitionSlot.getStartTime()) { - continue; - } else if (time > timePartitionSlot.getStartTime()) { - break; - } - - Object value = batchData.currentValue(); - switch (chunkHeader.getDataType()) { - case INT32: - chunkWriter.write(time, (int) value); - break; - case INT64: - chunkWriter.write(time, (long) value); - break; - case FLOAT: - chunkWriter.write(time, (float) value); - break; - case DOUBLE: - chunkWriter.write(time, (double) value); - break; - case BOOLEAN: - chunkWriter.write(time, (boolean) value); - break; - case TEXT: - chunkWriter.write(time, (Binary) value); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Data type %s is not supported.", chunkHeader.getDataType())); - } - batchData.next(); - } - chunkWriter.sealCurrentPage(); - } - - @Override - public String toString() { - return "NonAlignedChunkData{" - + "offset=" - + offset - + ", dataSize=" - + dataSize - + ", isHeadPageNeedDecode=" - + isHeadPageNeedDecode - + ", isTailPageNeedDecode=" - + isTailPageNeedDecode - + ", timePartitionSlot=" - + timePartitionSlot - + ", device='" - + device - + '\'' - + ", chunkHeader=" - + chunkHeader - + '}'; - } - } - - public static class AlignedChunkData implements ChunkData { - private List offset; - private List dataSize; - private boolean isHeadPageNeedDecode; - private boolean isTailPageNeedDecode; - - private TTimePartitionSlot timePartitionSlot; - private String device; - private List chunkHeaderList; - - public AlignedChunkData(long timeOffset, String device, ChunkHeader chunkHeader) { - this.offset = new ArrayList<>(); - this.dataSize = new ArrayList<>(); - this.isHeadPageNeedDecode = false; - this.isTailPageNeedDecode = false; - this.device = device; - this.chunkHeaderList = new ArrayList<>(); - - offset.add(timeOffset); - dataSize.add(0L); - chunkHeaderList.add(chunkHeader); - } - - @Override - public String getDevice() { - return device; - } - - @Override - public TTimePartitionSlot getTimePartitionSlot() { - return timePartitionSlot; - } - - @Override - public long getDataSize() { - return dataSize.stream().mapToLong(o -> o).sum(); - } - - @Override - public void addDataSize(long pageSize) { - dataSize.set(0, dataSize.get(0) + pageSize); - } - - @Override - public void setHeadPageNeedDecode(boolean headPageNeedDecode) { - isHeadPageNeedDecode = headPageNeedDecode; - } - - @Override - public void setTailPageNeedDecode(boolean tailPageNeedDecode) { - isTailPageNeedDecode = tailPageNeedDecode; - } - - @Override - public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { - this.timePartitionSlot = timePartitionSlot; - } - - public void addValueChunk(long offset, ChunkHeader chunkHeader) { - this.offset.add(offset); - this.dataSize.add(0L); - this.chunkHeaderList.add(chunkHeader); - } - - public void addValueChunkDataSize(long dataSize) { - int lastIndex = this.dataSize.size() - 1; - this.dataSize.set(lastIndex, this.dataSize.get(lastIndex) + dataSize); - } - - @Override - public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { - return null; - } - - @Override - public String toString() { - return "AlignedChunkData{" - + "offset=" - + offset - + ", dataSize=" - + dataSize - + ", isHeadPageNeedDecode=" - + isHeadPageNeedDecode - + ", isTailPageNeedDecode=" - + isTailPageNeedDecode - + ", timePartitionSlot=" - + timePartitionSlot - + ", device='" - + device - + '\'' - + ", chunkHeaderList=" - + chunkHeaderList - + '}'; - } - } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index d35fc0936016d..abda6e4c8fb04 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -91,7 +91,7 @@ public Future dispatch(List instan private void dispatchOneInstance(FragmentInstance instance) throws FragmentInstanceDispatchException { - for (TDataNodeLocation dataNodeLocation : instance.getDataRegionId().getDataNodeLocations()) { + for (TDataNodeLocation dataNodeLocation : instance.getRegionReplicaSet().getDataNodeLocations()) { TEndPoint endPoint = dataNodeLocation.getInternalEndPoint(); if (isDispatchedToLocal(endPoint)) { dispatchLocally(instance); @@ -110,14 +110,19 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) try (SyncDataNodeInternalServiceClient client = internalServiceClientManager.borrowClient(endPoint)) { TLoadTsFileReq loadTsFileReq = - new TLoadTsFileReq(instance.getFragment().getRoot().serializeToByteBuffer(), uuid); + new TLoadTsFileReq( + instance.getFragment().getRoot().serializeToByteBuffer(), + uuid, + instance.getRegionReplicaSet().getRegionId()); TLoadResp loadResp = client.sendLoadNode(loadTsFileReq); - if (!LoadTsFileScheduler.LoadResult.SUCCESS.equals( - LoadTsFileScheduler.LoadResult.values()[loadResp.loadRespStatus])) { - logger.error(loadResp.info); + if (!loadResp.isAccepted()) { + logger.error(loadResp.message); TSStatus status = new TSStatus(); - status.setCode(TSStatusCode.LOAD_TSFILE_ERROR.getStatusCode()); - status.setMessage(loadResp.info); + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(loadResp.message); + if (loadResp.status != null) { + status.addToSubStatus(loadResp.status); + } throw new FragmentInstanceDispatchException(status); } } catch (IOException | TException e) { @@ -191,9 +196,9 @@ public Future dispatchCommand( } catch (Throwable t) { logger.error("cannot dispatch LoadCommand for load operation", t); return immediateFuture( - new FragInstanceDispatchResult( - RpcUtils.getStatus( - TSStatusCode.INTERNAL_SERVER_ERROR, "Unexpected errors: " + t.getMessage()))); + new FragInstanceDispatchResult( + RpcUtils.getStatus( + TSStatusCode.INTERNAL_SERVER_ERROR, "Unexpected errors: " + t.getMessage()))); } } return immediateFuture(new FragInstanceDispatchResult(true)); @@ -204,12 +209,14 @@ private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint) try (SyncDataNodeInternalServiceClient client = internalServiceClientManager.borrowClient(endPoint)) { TLoadResp loadResp = client.sendLoadCommand(loadCommandReq); - if (!LoadTsFileScheduler.LoadResult.SUCCESS.equals( - LoadTsFileScheduler.LoadResult.values()[loadResp.loadRespStatus])) { - logger.error(loadResp.info); + if (!loadResp.isAccepted()) { + logger.error(loadResp.message); TSStatus status = new TSStatus(); - status.setCode(TSStatusCode.LOAD_TSFILE_ERROR.getStatusCode()); - status.setMessage(loadResp.info); + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(loadResp.message); + if (loadResp.status != null) { + status.addToSubStatus(loadResp.status); + } throw new FragmentInstanceDispatchException(status); } } catch (IOException | TException e) { @@ -221,7 +228,8 @@ private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint) } } - private void dispatchLocally(TLoadCommandReq loadCommandReq) throws FragmentInstanceDispatchException { + private void dispatchLocally(TLoadCommandReq loadCommandReq) + throws FragmentInstanceDispatchException { // TODO: use local interface } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index aca6a2433488e..f00e4cf53a13d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.db.mpp.common.FragmentInstanceId; @@ -37,6 +38,7 @@ import io.airlift.units.Duration; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; +import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,9 +119,18 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { logger.error( String.format( "Dispatch one piece to ReplicaSet %s error, result status code %s.", - entry.getKey(), result.getFailureStatus().getCode())); + entry.getKey(), + TSStatusCode.representOf(result.getFailureStatus().getCode()).name())); logger.error( - String.format("Result message %s.", result.getFailureStatus().getMessage())); + String.format("Result status message %s.", result.getFailureStatus().getMessage())); + if (result.getFailureStatus().getSubStatus().size() > 0) { + for (TSStatus status : result.getFailureStatus().getSubStatus()) { + logger.error( + String.format( + "Sub status code %s.", TSStatusCode.representOf(status.getCode()).name())); + logger.error(String.format("Sub status message %s.", status.getMessage())); + } + } logger.error(String.format("Dispatch piece node:%n%s", pieceNode)); return false; } @@ -186,9 +197,4 @@ public enum LoadCommand { EXECUTE, ROLLBACK } - - public enum LoadResult { - SUCCESS, - FAILED - } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index a339f8bec5ce8..ff08658943470 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -65,7 +65,9 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.service.DataNode; import org.apache.iotdb.db.service.RegionMigrateService; import org.apache.iotdb.db.service.metrics.MetricsService; @@ -302,12 +304,35 @@ public TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) { @Override public TLoadResp sendLoadNode(TLoadTsFileReq req) throws TException { - return null; + LOGGER.info(String.format("Receive load node from uuid %s.", req.uuid)); + + ConsensusGroupId groupId = + ConsensusGroupId.Factory.createFromTConsensusGroupId(req.consensusGroupId); + LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) PlanNodeType.deserialize(req.body); + TSStatus resultStatus = + StorageEngineV2.getInstance() + .writeLoadTsFileNode((DataRegionId) groupId, pieceNode, req.uuid); + + return createTLoadResp(resultStatus); } @Override public TLoadResp sendLoadCommand(TLoadCommandReq req) throws TException { - return null; + + TSStatus resultStatus = StorageEngineV2.getInstance() + .executeLoadCommand( + LoadTsFileScheduler.LoadCommand.values()[req.commandType], req.uuid); + return createTLoadResp(resultStatus); + } + + private TLoadResp createTLoadResp(TSStatus resultStatus) { + boolean isAccepted = RpcUtils.SUCCESS_STATUS.equals(resultStatus); + TLoadResp loadResp = new TLoadResp(isAccepted); + if (!isAccepted) { + loadResp.setMessage(resultStatus.getMessage()); + loadResp.setStatus(resultStatus); + } + return loadResp; } @Override diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 2d71d7d32a567..eec22f46bb919 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -70,6 +70,7 @@ public enum TSStatusCode { MEASUREMENT_ALREADY_EXIST(338), TEMPLATE_NOT_EXIST(339), CREATE_TEMPLATE_ERROR(340), + TSFILE_RUNTIME_ERROR(341), EXECUTE_STATEMENT_ERROR(400), SQL_PARSE_ERROR(401), @@ -125,7 +126,6 @@ public enum TSStatusCode { CREATE_REGION_ERROR(711), DELETE_REGION_ERROR(712), CACHE_UPDATE_FAIL(713), - LOAD_TSFILE_ERROR(714), // configuration CONFIG_ERROR(800), diff --git a/thrift/src/main/thrift/datanode.thrift b/thrift/src/main/thrift/datanode.thrift index cca64744c6d2b..7b225da14187e 100644 --- a/thrift/src/main/thrift/datanode.thrift +++ b/thrift/src/main/thrift/datanode.thrift @@ -118,6 +118,7 @@ struct TSendPlanNodeResp { 1: required bool accepted 2: optional string message 3: optional common.TSStatus status + } struct TFetchFragmentInstanceStateReq { @@ -206,6 +207,7 @@ struct TUpdateTemplateReq{ struct TLoadTsFileReq{ 1: required binary body 2: required string uuid + 3: required common.TConsensusGroupId consensusGroupId } struct TLoadCommandReq{ @@ -214,8 +216,9 @@ struct TLoadCommandReq{ } struct TLoadResp{ - 1: required i32 loadRespStatus - 2: required string info + 1: required bool accepted + 2: optional string message + 3: optional common.TSStatus status } service IDataNodeRPCService { From 47a0de0857f11cc20208ab6bcb47a8dd328a04ac Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 17 Aug 2022 15:01:31 +0800 Subject: [PATCH 07/37] working on serialize --- .../db/engine/load/AlignedChunkData.java | 6 +++ .../iotdb/db/engine/load/ChunkData.java | 3 ++ .../db/engine/load/LoadTsFileManager.java | 2 - .../db/engine/load/NonAlignedChunkData.java | 43 +++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index c9e5345e0adf0..0fda2e4b379c9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -103,6 +104,11 @@ public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageExceptio return null; } + @Override + public void serialize(DataOutputStream stream, File tsFile) throws IOException { + + } + @Override public String toString() { return "AlignedChunkData{" diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java index 8011528fff1d6..d370855f28fa8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java @@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; @@ -44,6 +45,8 @@ public interface ChunkData { IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException; + void serialize(DataOutputStream stream, File tsFile) throws IOException; + static ChunkData createChunkData( boolean isAligned, long offset, String device, ChunkHeader chunkHeader) { return isAligned diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 1c422e2cffc6f..52b9eacfe7f1e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -73,8 +73,6 @@ private void clearDir(File dir) { public void writeToDataRegion(DataRegion dataRegion, LoadTsFilePieceNode pieceNode, String uuid) throws PageException, IOException { - dataRegion.getLogicalStorageGroupName(); - dataRegion.getDataRegionId(); TsFileWriterManager writerManager = uuid2WriterManager.computeIfAbsent( uuid, o -> new TsFileWriterManager(SystemFileFactory.INSTANCE.getFile(loadDir, uuid))); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java index 234282d9b65cd..becaaf0e16c4f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -33,10 +33,12 @@ import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -133,6 +135,47 @@ public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageExceptio } } + @Override + public void serialize(DataOutputStream stream, File tsFile) throws IOException { + serializeAttr(stream); + } + + private void serializeAttr(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); + ReadWriteIOUtils.write(device, stream); + chunkHeader.serializeTo(stream); + } + + private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + reader.position(offset); + long dataSize = this.dataSize; + while (dataSize > 0) { + PageHeader pageHeader = + reader.readPageHeader( + chunkHeader.getDataType(), + (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + long pageDataSize = pageHeader.getSerializedPageSize(); + if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page + || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page + decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); + } else { // entire page + ByteBuffer pageData = reader.readCompressedPage(pageHeader); + chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); + } + dataSize -= pageDataSize; + } + return chunkWriter; + } + } + private void decodePage( TsFileSequenceReader reader, PageHeader pageHeader, From c3b4b3ed1f5d5276bbacbb6e3b7c874ac9dc7648 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 17 Aug 2022 22:56:27 +0800 Subject: [PATCH 08/37] finish nonaligned --- .../db/engine/load/AlignedChunkData.java | 13 +- .../iotdb/db/engine/load/ChunkData.java | 13 +- .../db/engine/load/LoadTsFileManager.java | 2 +- .../db/engine/load/NonAlignedChunkData.java | 162 ++++++++++++------ .../plan/planner/plan/node/PlanNodeId.java | 5 + .../plan/planner/plan/node/PlanNodeType.java | 6 +- .../plan/node/load/LoadTsFilePieceNode.java | 57 +++--- .../impl/DataNodeInternalRPCServiceImpl.java | 11 +- .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../write/chunk/AlignedChunkWriterImpl.java | 3 - .../tsfile/write/chunk/ChunkWriterImpl.java | 3 - .../tsfile/write/chunk/IChunkWriter.java | 2 - 12 files changed, 183 insertions(+), 95 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index 0fda2e4b379c9..1447ca87a0269 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -20,13 +20,13 @@ package org.apache.iotdb.db.engine.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -88,6 +88,11 @@ public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { this.timePartitionSlot = timePartitionSlot; } + @Override + public boolean isAligned() { + return true; + } + public void addValueChunk(long offset, ChunkHeader chunkHeader) { this.offset.add(offset); this.dataSize.add(0L); @@ -100,7 +105,7 @@ public void addValueChunkDataSize(long dataSize) { } @Override - public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { + public IChunkWriter getChunkWriter() { return null; } @@ -109,6 +114,10 @@ public void serialize(DataOutputStream stream, File tsFile) throws IOException { } + public static AlignedChunkData deserialize(InputStream stream) { + return null; + } + @Override public String toString() { return "AlignedChunkData{" diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java index d370855f28fa8..a3c004673b91c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java @@ -22,11 +22,13 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; public interface ChunkData { String getDevice(); @@ -43,10 +45,19 @@ public interface ChunkData { void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot); - IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException; + boolean isAligned(); + + IChunkWriter getChunkWriter(); void serialize(DataOutputStream stream, File tsFile) throws IOException; + static ChunkData deserialize(InputStream stream) throws PageException, IOException { + boolean isAligned = ReadWriteIOUtils.readBool(stream); + return isAligned + ? AlignedChunkData.deserialize(stream) + : NonAlignedChunkData.deserialize(stream); + } + static ChunkData createChunkData( boolean isAligned, long offset, String device, ChunkHeader chunkHeader) { return isAligned diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 52b9eacfe7f1e..165911a53d55a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -157,7 +157,7 @@ private void write(String dataPartition, ChunkData chunkData) } writer.startChunkGroup(chunkData.getDevice()); } - chunkData.getChunkWriter(null).writeToFileWriter(writer); // TODO: get writer + chunkData.getChunkWriter().writeToFileWriter(writer); // TODO: get writer } private void loadAll() throws IOException { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java index becaaf0e16c4f..c755515fb9cfe 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.exception.write.PageException; @@ -41,6 +42,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; public class NonAlignedChunkData implements ChunkData { @@ -53,6 +55,8 @@ public class NonAlignedChunkData implements ChunkData { private String device; private ChunkHeader chunkHeader; + private ChunkWriterImpl chunkWriter; + public NonAlignedChunkData(long offset, String device, ChunkHeader chunkHeader) { this.offset = offset; this.dataSize = 0; @@ -98,82 +102,61 @@ public void setTimePartitionSlot(TTimePartitionSlot timePartitionSlot) { } @Override - public IChunkWriter getChunkWriter(File tsFile) throws IOException, PageException { - ChunkWriterImpl chunkWriter = - new ChunkWriterImpl( - new MeasurementSchema( - chunkHeader.getMeasurementID(), - chunkHeader.getDataType(), - chunkHeader.getEncodingType(), - chunkHeader.getCompressionType())); - try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { - Decoder defaultTimeDecoder = - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64); - Decoder valueDecoder = - Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + public boolean isAligned() { + return false; + } - reader.position(offset); - long dataSize = this.dataSize; - while (dataSize > 0) { - PageHeader pageHeader = - reader.readPageHeader( - chunkHeader.getDataType(), - (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); - long pageDataSize = pageHeader.getSerializedPageSize(); - if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page - || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page - decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); - } else { // entire page - ByteBuffer pageData = reader.readCompressedPage(pageHeader); - chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); - } - dataSize -= pageDataSize; - } - return chunkWriter; - } + @Override + public IChunkWriter getChunkWriter() { + return chunkWriter; } @Override public void serialize(DataOutputStream stream, File tsFile) throws IOException { + ReadWriteIOUtils.write(isAligned(), stream); serializeAttr(stream); + serializeTsFileData(stream, tsFile); } private void serializeAttr(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); ReadWriteIOUtils.write(device, stream); + ReadWriteIOUtils.write(chunkHeader.getChunkType(), stream); chunkHeader.serializeTo(stream); } private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { Decoder defaultTimeDecoder = - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64); + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); Decoder valueDecoder = - Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); reader.position(offset); long dataSize = this.dataSize; while (dataSize > 0) { - PageHeader pageHeader = - reader.readPageHeader( - chunkHeader.getDataType(), - (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + boolean hasStatistic = (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER; + PageHeader pageHeader = reader.readPageHeader(chunkHeader.getDataType(), hasStatistic); long pageDataSize = pageHeader.getSerializedPageSize(); if ((dataSize == this.dataSize && isHeadPageNeedDecode) // decode head page - || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page - decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, chunkWriter); + || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page + ReadWriteIOUtils.write(true, stream); // decode + decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, stream); } else { // entire page + ReadWriteIOUtils.write(false, stream); // don't decode + ReadWriteIOUtils.write(hasStatistic, stream); + pageHeader.serializeTo(stream); // TODO: can save this time by recording ByteBuffer ByteBuffer pageData = reader.readCompressedPage(pageHeader); - chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); + ReadWriteIOUtils.write(pageData, stream); } dataSize -= pageDataSize; } - return chunkWriter; } + + ReadWriteIOUtils.write(true, stream); // means ending + ReadWriteIOUtils.write(-1, stream); } private void decodePage( @@ -181,40 +164,43 @@ private void decodePage( PageHeader pageHeader, Decoder timeDecoder, Decoder valueDecoder, - ChunkWriterImpl chunkWriter) + DataOutputStream stream) throws IOException { valueDecoder.reset(); ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); PageReader pageReader = new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); BatchData batchData = pageReader.getAllSatisfiedPageData(); + + ReadWriteIOUtils.write(batchData.length(), stream); // TODO: check if correct while (batchData.hasCurrent()) { long time = batchData.currentTime(); if (time < timePartitionSlot.getStartTime()) { continue; - } else if (time > timePartitionSlot.getStartTime()) { + } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time))) { break; } + ReadWriteIOUtils.write(time, stream); Object value = batchData.currentValue(); switch (chunkHeader.getDataType()) { case INT32: - chunkWriter.write(time, (int) value); + ReadWriteIOUtils.write((int) value, stream); break; case INT64: - chunkWriter.write(time, (long) value); + ReadWriteIOUtils.write((long) value, stream); break; case FLOAT: - chunkWriter.write(time, (float) value); + ReadWriteIOUtils.write((float) value, stream); break; case DOUBLE: - chunkWriter.write(time, (double) value); + ReadWriteIOUtils.write((double) value, stream); break; case BOOLEAN: - chunkWriter.write(time, (boolean) value); + ReadWriteIOUtils.write((boolean) value, stream); break; case TEXT: - chunkWriter.write(time, (Binary) value); + ReadWriteIOUtils.write((Binary) value, stream); break; default: throw new UnSupportedDataTypeException( @@ -222,7 +208,73 @@ private void decodePage( } batchData.next(); } - chunkWriter.sealCurrentPage(); + } + + private void buildChunkWriter(InputStream stream) throws IOException, PageException { + chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema( + chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), + chunkHeader.getEncodingType(), + chunkHeader.getCompressionType())); + boolean needDecode; + while (true) { + needDecode = ReadWriteIOUtils.readBool(stream); + if (needDecode) { + int length = ReadWriteIOUtils.readInt(stream); + if (length == -1) { + break; + } + + for (int i = 0; i < length; i++) { + long time = ReadWriteIOUtils.readLong(stream); + switch (chunkHeader.getDataType()) { + case INT32: + chunkWriter.write(time, ReadWriteIOUtils.readInt(stream)); + break; + case INT64: + chunkWriter.write(time, ReadWriteIOUtils.readLong(stream)); + break; + case FLOAT: + chunkWriter.write(time, ReadWriteIOUtils.readFloat(stream)); + break; + case DOUBLE: + chunkWriter.write(time, ReadWriteIOUtils.readDouble(stream)); + break; + case BOOLEAN: + chunkWriter.write(time, ReadWriteIOUtils.readBool(stream)); + break; + case TEXT: + chunkWriter.write(time, ReadWriteIOUtils.readBinary(stream)); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", chunkHeader.getDataType())); + } + } + + chunkWriter.sealCurrentPage(); + } else { + boolean hasStatistic = ReadWriteIOUtils.readBool(stream); + PageHeader pageHeader = + PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), hasStatistic); + chunkWriter.writePageHeaderAndDataIntoBuff( + ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), + pageHeader); + } + } + } + + public static NonAlignedChunkData deserialize(InputStream stream) throws IOException, PageException { + long timePartition = ReadWriteIOUtils.readLong(stream); + String device = ReadWriteIOUtils.readString(stream); + byte chunkType = ReadWriteIOUtils.readByte(stream); + ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); + NonAlignedChunkData chunkData = new NonAlignedChunkData(-1, device, chunkHeader); + chunkData.setTimePartitionSlot(StorageEngineV2.getTimePartitionSlot(timePartition)); + chunkData.buildChunkWriter(stream); + return chunkData; } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeId.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeId.java index 554a9159455c8..fd511b2ba1700 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeId.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeId.java @@ -22,6 +22,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; public class PlanNodeId { @@ -57,6 +58,10 @@ public static PlanNodeId deserialize(ByteBuffer byteBuffer) { return new PlanNodeId(ReadWriteIOUtils.readString(byteBuffer)); } + public static PlanNodeId deserialize(InputStream stream) throws IOException { + return new PlanNodeId(ReadWriteIOUtils.readString(stream)); + } + public void serialize(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(id, byteBuffer); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java index 349060a1bba58..875b0bb65259a 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.mpp.plan.planner.plan.node; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; @@ -132,7 +133,8 @@ public enum PlanNodeType { NODE_PATHS_COUNT((short) 51), INTERNAL_CREATE_TIMESERIES((short) 52), ACTIVATE_TEMPLATE((short) 53), - PATHS_USING_TEMPLATE_SCAN((short) 54); + PATHS_USING_TEMPLATE_SCAN((short) 54), + LOAD_TSFILE((short) 55); public static final int BYTES = Short.BYTES; @@ -287,6 +289,8 @@ public static PlanNode deserialize(ByteBuffer buffer) { return ActivateTemplateNode.deserialize(buffer); case 54: return PathsUsingTemplateScanNode.deserialize(buffer); + case 55: + return LoadTsFilePieceNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 3b3da953a6883..c5ff0d501d54f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -20,39 +20,26 @@ package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; -import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.exception.write.PageException; -import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; -import org.apache.iotdb.tsfile.file.MetaMarker; -import org.apache.iotdb.tsfile.file.header.ChunkHeader; -import org.apache.iotdb.tsfile.file.header.PageHeader; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.reader.page.PageReader; -import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; -import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; - +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -136,10 +123,12 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.LOAD_TSFILE.serialize(stream); + ReadWriteIOUtils.write(tsFile.getPath(), stream); // TODO: can save this space for (ChunkData chunkData : chunkDataList) { try { - chunkData.getChunkWriter(tsFile).serializeToDataOutputStream(stream); - } catch (PageException e) { + chunkData.serialize(stream, tsFile); + } catch (IOException e) { logger.error( String.format( "Parse page of TsFile %s error, skip chunk %s", tsFile.getPath(), chunkData)); @@ -152,12 +141,32 @@ public List splitByPartition(Analysis analysis) { throw new NotImplementedException("split load piece TsFile is not implemented"); } + public static PlanNode deserialize(ByteBuffer buffer) { + InputStream stream = new ByteArrayInputStream(buffer.array()); + try { + File tsFile = new File(ReadWriteIOUtils.readString(stream)); + LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); + while (stream.available() > 0) { + ChunkData chunkData = ChunkData.deserialize(stream); + pieceNode.addChunkData(chunkData); + } + pieceNode.setPlanNodeId(PlanNodeId.deserialize(stream)); + return pieceNode; + } catch (IOException | PageException e) { + logger.error(String.format("Deserialize %s error.", LoadTsFilePieceNode.class.getName()), e); + return null; + } + } + @Override public String toString() { - return "LoadTsFilePieceNode{" + - "tsFile=" + tsFile + - ", dataSize=" + dataSize + - ", chunkDataList=" + chunkDataList + - '}'; + return "LoadTsFilePieceNode{" + + "tsFile=" + + tsFile + + ", dataSize=" + + dataSize + + ", chunkDataList=" + + chunkDataList + + '}'; } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index ff08658943470..208d01bf73afd 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -309,6 +309,10 @@ public TLoadResp sendLoadNode(TLoadTsFileReq req) throws TException { ConsensusGroupId groupId = ConsensusGroupId.Factory.createFromTConsensusGroupId(req.consensusGroupId); LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) PlanNodeType.deserialize(req.body); + if (pieceNode == null) { + return createTLoadResp(new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); + } + TSStatus resultStatus = StorageEngineV2.getInstance() .writeLoadTsFileNode((DataRegionId) groupId, pieceNode, req.uuid); @@ -319,9 +323,10 @@ public TLoadResp sendLoadNode(TLoadTsFileReq req) throws TException { @Override public TLoadResp sendLoadCommand(TLoadCommandReq req) throws TException { - TSStatus resultStatus = StorageEngineV2.getInstance() - .executeLoadCommand( - LoadTsFileScheduler.LoadCommand.values()[req.commandType], req.uuid); + TSStatus resultStatus = + StorageEngineV2.getInstance() + .executeLoadCommand( + LoadTsFileScheduler.LoadCommand.values()[req.commandType], req.uuid); return createTLoadResp(resultStatus); } diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index eec22f46bb919..15d4f0bbae4f6 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -126,6 +126,7 @@ public enum TSStatusCode { CREATE_REGION_ERROR(711), DELETE_REGION_ERROR(712), CACHE_UPDATE_FAIL(713), + NODE_DESERIALIZE_ERROR(714), // configuration CONFIG_ERROR(800), diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index 32c137ee1a168..871e181c38ce8 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -300,7 +300,4 @@ public boolean checkIsChunkSizeOverThreshold(long threshold) { public TSDataType getCurrentValueChunkType() { return valueChunkWriterList.get(valueIndex).getDataType(); } - - @Override - public void serializeToDataOutputStream(DataOutputStream stream) throws IOException {} } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 13e69800e6848..25efb86f25058 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -458,7 +458,4 @@ public boolean isMerging() { public void setLastPoint(boolean isLastPoint) { this.isLastPoint = isLastPoint; } - - @Override - public void serializeToDataOutputStream(DataOutputStream stream) throws IOException {} } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java index 1564211cc3e9a..bc0acdca7ea9e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java @@ -29,8 +29,6 @@ public interface IChunkWriter { /** flush data to TsFileIOWriter. */ void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException; - void serializeToDataOutputStream(DataOutputStream stream) throws IOException; - /** estimate memory usage of this series. */ long estimateMaxSeriesMemSize(); From 4380fa827dbfd9bd094c74f01616170940c4933e Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 22 Aug 2022 11:20:19 +0800 Subject: [PATCH 09/37] finish serialize --- .../db/engine/load/AlignedChunkData.java | 285 +++++++++++++++++- .../plan/node/load/LoadSingleTsFileNode.java | 9 + .../load/LoadTsFileDispatcherImpl.java | 11 +- .../scheduler/load/LoadTsFileScheduler.java | 28 +- .../write/chunk/AlignedChunkWriterImpl.java | 37 +++ .../tsfile/write/chunk/TimeChunkWriter.java | 52 ++++ .../tsfile/write/chunk/ValueChunkWriter.java | 56 ++++ 7 files changed, 458 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index 1447ca87a0269..d0b5460b11c76 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -20,17 +20,45 @@ package org.apache.iotdb.db.engine.load; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; +import org.apache.iotdb.tsfile.read.reader.page.ValuePageReader; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.checkerframework.checker.units.qual.A; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; public class AlignedChunkData implements ChunkData { + private static final int DEFAULT_INT32 = 0; + private static final long DEFAULT_INT64 = 0L; + private static final float DEFAULT_FLOAT = 0; + private static final double DEFAULT_DOUBLE = 0.0; + private static final boolean DEFAULT_BOOLEAN = false; + private static final Binary DEFAULT_BINARY = null; + private List offset; private List dataSize; private boolean isHeadPageNeedDecode; @@ -40,6 +68,10 @@ public class AlignedChunkData implements ChunkData { private String device; private List chunkHeaderList; + private List timeBatch; + + private AlignedChunkWriterImpl chunkWriter; + public AlignedChunkData(long timeOffset, String device, ChunkHeader chunkHeader) { this.offset = new ArrayList<>(); this.dataSize = new ArrayList<>(); @@ -106,16 +138,263 @@ public void addValueChunkDataSize(long dataSize) { @Override public IChunkWriter getChunkWriter() { - return null; + return chunkWriter; } @Override public void serialize(DataOutputStream stream, File tsFile) throws IOException { + ReadWriteIOUtils.write(isAligned(), stream); + serializeAttr(stream); + serializeTsFileData(stream, tsFile); + } + + private void serializeAttr(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); + ReadWriteIOUtils.write(device, stream); + ReadWriteIOUtils.write(chunkHeaderList.size(), stream); + for (ChunkHeader chunkHeader : chunkHeaderList) { + ReadWriteIOUtils.write(chunkHeader.getChunkType(), stream); + chunkHeader.serializeTo(stream); + } + } + + private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { + timeBatch = new ArrayList<>(); + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + int chunkSize = offset.size(); + for (int i = 0; i < chunkSize; i++) { + serializeChunk(stream, reader, chunkHeaderList.get(i), i); + } + } + timeBatch = null; + } + + private void serializeChunk( + DataOutputStream stream, TsFileSequenceReader reader, ChunkHeader chunkHeader, int chunkIndex) + throws IOException { + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + reader.position(offset.get(chunkIndex)); + int pageIndex = 0; + long dataSize = this.dataSize.get(chunkIndex); + while (dataSize > 0) { + boolean hasStatistic = (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER; + PageHeader pageHeader = reader.readPageHeader(chunkHeader.getDataType(), hasStatistic); + long pageDataSize = pageHeader.getSerializedPageSize(); + if ((dataSize == this.dataSize.get(chunkIndex) && isHeadPageNeedDecode) // decode head page + || (dataSize == pageDataSize && isTailPageNeedDecode)) { // decode tail page + ReadWriteIOUtils.write(true, stream); // decode + if (chunkIndex == 0) { + decodeTimePage(reader, chunkHeader, pageHeader, defaultTimeDecoder, valueDecoder, stream); + } else { + decodeValuePage(reader, chunkHeader, pageHeader, pageIndex, valueDecoder, stream); + } + } else { // entire page + ReadWriteIOUtils.write(false, stream); // don't decode + ReadWriteIOUtils.write(hasStatistic, stream); + pageHeader.serializeTo(stream); // TODO: can save this time by recording ByteBuffer + ByteBuffer pageData = reader.readCompressedPage(pageHeader); + ReadWriteIOUtils.write(pageData, stream); + } + pageIndex += 1; + dataSize -= pageDataSize; + } + ReadWriteIOUtils.write(true, stream); // means ending + ReadWriteIOUtils.write(-1, stream); } - public static AlignedChunkData deserialize(InputStream stream) { - return null; + private void decodeTimePage( + TsFileSequenceReader reader, + ChunkHeader chunkHeader, + PageHeader pageHeader, + Decoder timeDecoder, + Decoder valueDecoder, + DataOutputStream stream) + throws IOException { + valueDecoder.reset(); + ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); + TimePageReader timePageReader = new TimePageReader(pageHeader, pageData, timeDecoder); + long[] time = timePageReader.getNextTimeBatch(); + timeBatch.add(time); + ReadWriteIOUtils.write(time.length, stream); + for (int i = 0; i < time.length; i++) { + if (time[i] < timePartitionSlot.getStartTime()) { + continue; + } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time[i]))) { + break; + } + ReadWriteIOUtils.write(time[i], stream); + } + } + + private void decodeValuePage( + TsFileSequenceReader reader, + ChunkHeader chunkHeader, + PageHeader pageHeader, + int pageIndex, + Decoder valueDecoder, + DataOutputStream stream) + throws IOException { + valueDecoder.reset(); + ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); + ValuePageReader valuePageReader = + new ValuePageReader(pageHeader, pageData, chunkHeader.getDataType(), valueDecoder); + TsPrimitiveType[] valueBatch = valuePageReader.nextValueBatch(timeBatch.get(pageIndex)); + ReadWriteIOUtils.write(valueBatch.length, stream); + for (int i = 0; i < valueBatch.length; i++) { + if (valueBatch[i] == null) { + ReadWriteIOUtils.write(true, stream); + continue; + } + ReadWriteIOUtils.write(false, stream); + switch (chunkHeader.getDataType()) { + case INT32: + ReadWriteIOUtils.write(valueBatch[i].getInt(), stream); + break; + case INT64: + ReadWriteIOUtils.write(valueBatch[i].getLong(), stream); + break; + case FLOAT: + ReadWriteIOUtils.write(valueBatch[i].getFloat(), stream); + break; + case DOUBLE: + ReadWriteIOUtils.write(valueBatch[i].getDouble(), stream); + break; + case BOOLEAN: + ReadWriteIOUtils.write(valueBatch[i].getBoolean(), stream); + break; + case TEXT: + ReadWriteIOUtils.write(valueBatch[i].getBinary(), stream); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", chunkHeader.getDataType())); + } + } + } + + private void buildChunkWriter(InputStream stream) throws IOException, PageException { + List measurementSchemaList = new ArrayList<>(); + for (ChunkHeader chunkHeader : chunkHeaderList) { + if (TSDataType.VECTOR.equals(chunkHeader.getDataType())) { + continue; + } + measurementSchemaList.add( + new MeasurementSchema( + chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), + chunkHeader.getEncodingType(), + chunkHeader.getCompressionType())); + } + chunkWriter = new AlignedChunkWriterImpl(measurementSchemaList); + timeBatch = new ArrayList<>(); + int chunkHeaderSize = chunkHeaderList.size(); + for (int i = 0; i < chunkHeaderSize; i++) { + buildChunk(stream, chunkHeaderList.get(i), i - 1, i == 0); + } + timeBatch = null; + } + + private void buildChunk( + InputStream stream, ChunkHeader chunkHeader, int valueChunkIndex, boolean isTimeChunk) + throws IOException, PageException { + boolean needDecode; + int decodePageIndex = 0; + while (true) { + needDecode = ReadWriteIOUtils.readBool(stream); + if (needDecode) { + int length = ReadWriteIOUtils.readInt(stream); + if (length == -1) { + break; + } + + long[] timePageBatch = new long[length]; + if (!isTimeChunk) { + timePageBatch = timeBatch.get(decodePageIndex); + } + for (int i = 0; i < length; i++) { + if (isTimeChunk) { + long time = ReadWriteIOUtils.readLong(stream); + timePageBatch[i] = time; + chunkWriter.write(time); + } else { + boolean isNull = ReadWriteIOUtils.readBool(stream); + switch (chunkHeader.getDataType()) { + case INT32: + int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream); + chunkWriter.write(timePageBatch[i], int32Value, isNull, valueChunkIndex); + break; + case INT64: + long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream); + chunkWriter.write(timePageBatch[i], int64Value, isNull, valueChunkIndex); + break; + case FLOAT: + float floatValue = isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream); + chunkWriter.write(timePageBatch[i], floatValue, isNull, valueChunkIndex); + break; + case DOUBLE: + double doubleValue = isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream); + chunkWriter.write(timePageBatch[i], doubleValue, isNull, valueChunkIndex); + break; + case BOOLEAN: + boolean boolValue = isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream); + chunkWriter.write(timePageBatch[i], boolValue, isNull, valueChunkIndex); + break; + case TEXT: + Binary binaryValue = isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); + chunkWriter.write(timePageBatch[i], binaryValue, isNull, valueChunkIndex); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", chunkHeader.getDataType())); + } + } + } + if (isTimeChunk) { + timeBatch.add(timePageBatch); + } + decodePageIndex += 1; + } else { + boolean hasStatistic = ReadWriteIOUtils.readBool(stream); + PageHeader pageHeader = + PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), hasStatistic); + if (isTimeChunk) { + chunkWriter.writePageHeaderAndDataIntoTimeBuff( + ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), + pageHeader); + } else { + chunkWriter.writePageHeaderAndDataIntoValueBuff( + ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), + pageHeader, + valueChunkIndex); + } + } + } + } + + public static AlignedChunkData deserialize(InputStream stream) throws IOException, PageException { + long timePartition = ReadWriteIOUtils.readLong(stream); + String device = ReadWriteIOUtils.readString(stream); + int chunkHeaderListSize = ReadWriteIOUtils.readInt(stream); + ChunkHeader[] chunkHeaderList = new ChunkHeader[chunkHeaderListSize]; + for (int i = 0; i < chunkHeaderListSize; i++) { + byte chunkType = ReadWriteIOUtils.readByte(stream); + ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); + } + + AlignedChunkData chunkData = new AlignedChunkData(-1, device, chunkHeaderList[0]); + for (int i = 1; i < chunkHeaderListSize; i++) { + chunkData.addValueChunk(-1, chunkHeaderList[i]); + } + chunkData.setTimePartitionSlot(StorageEngineV2.getTimePartitionSlot(timePartition)); + chunkData.buildChunkWriter(stream); + return chunkData; } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 3d111505e48fd..84b9aad0f9e1a 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -256,6 +256,15 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: header = reader.readChunkHeader(marker); if (header.getDataSize() == 0) { + Set allChunkData = new HashSet<>(); + for (Map.Entry> entry : pageIndex2ChunkData.entrySet()) { + for (AlignedChunkData alignedChunkData : entry.getValue()) { + if (!allChunkData.contains(alignedChunkData)) { + alignedChunkData.addValueChunk(-2, header); + allChunkData.add(alignedChunkData); + } + } + } break; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index abda6e4c8fb04..c6459142bd5cb 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -55,21 +55,23 @@ public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileDispatcherImpl.class); - private final String uuid; + private String uuid; private final String localhostIpAddr; private final int localhostInternalPort; private final IClientManager internalServiceClientManager; public LoadTsFileDispatcherImpl( - String uuid, IClientManager internalServiceClientManager) { - this.uuid = uuid; this.internalServiceClientManager = internalServiceClientManager; this.localhostIpAddr = IoTDBDescriptor.getInstance().getConfig().getInternalAddress(); this.localhostInternalPort = IoTDBDescriptor.getInstance().getConfig().getInternalPort(); } + public void setUuid(String uuid) { + this.uuid = uuid; + } + @Override public Future dispatch(List instances) { for (FragmentInstance instance : instances) { @@ -91,7 +93,8 @@ public Future dispatch(List instan private void dispatchOneInstance(FragmentInstance instance) throws FragmentInstanceDispatchException { - for (TDataNodeLocation dataNodeLocation : instance.getRegionReplicaSet().getDataNodeLocations()) { + for (TDataNodeLocation dataNodeLocation : + instance.getRegionReplicaSet().getDataNodeLocations()) { TEndPoint endPoint = dataNodeLocation.getInternalEndPoint(); if (isDispatchedToLocal(endPoint)) { dispatchLocally(instance); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index f00e4cf53a13d..fe642fc9416af 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -55,7 +55,6 @@ public class LoadTsFileScheduler implements IScheduler { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); - private final String uuid; private final MPPQueryContext queryContext; private LoadTsFileDispatcherImpl dispatcher; private List tsFileNodeList; @@ -67,11 +66,10 @@ public LoadTsFileScheduler( DistributedQueryPlan distributedQueryPlan, MPPQueryContext queryContext, IClientManager internalServiceClientManager) { - this.uuid = UUID.randomUUID().toString(); this.queryContext = queryContext; this.tsFileNodeList = new ArrayList<>(); this.fragmentId = distributedQueryPlan.getRootSubPlan().getPlanFragment().getId(); - this.dispatcher = new LoadTsFileDispatcherImpl(uuid, internalServiceClientManager); + this.dispatcher = new LoadTsFileDispatcherImpl(internalServiceClientManager); this.allReplicaSets = new HashSet<>(); for (FragmentInstance fragmentInstance : distributedQueryPlan.getInstances()) { @@ -81,17 +79,21 @@ public LoadTsFileScheduler( @Override public void start() { - boolean isFirstPhaseSuccess = firstPhase(); - boolean isSuccess = secondPhase(isFirstPhaseSuccess); + for (LoadSingleTsFileNode node : tsFileNodeList) { + String uuid = UUID.randomUUID().toString(); + dispatcher.setUuid(uuid); + allReplicaSets.clear(); + + boolean isFirstPhaseSuccess = firstPhase(node); + boolean isSuccess = secondPhase(isFirstPhaseSuccess, uuid); + } } - private boolean firstPhase() { - for (LoadSingleTsFileNode node : tsFileNodeList) { - if (!dispatchOneTsFile(node)) { - logger.error( - String.format("Dispatch Single TsFile Node error, LoadSingleTsFileNode %s.", node)); - return false; - } + private boolean firstPhase(LoadSingleTsFileNode node) { + if (!dispatchOneTsFile(node)) { + logger.error( + String.format("Dispatch Single TsFile Node error, LoadSingleTsFileNode %s.", node)); + return false; } return true; } @@ -146,7 +148,7 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { return true; } - private boolean secondPhase(boolean isFirstPhaseSuccess) { + private boolean secondPhase(boolean isFirstPhaseSuccess, String uuid) { TLoadCommandReq loadCommandReq = new TLoadCommandReq( (isFirstPhaseSuccess ? LoadCommand.EXECUTE : LoadCommand.ROLLBACK).ordinal(), uuid); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index 871e181c38ce8..d382d046b0b1f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -21,6 +21,8 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.iotdb.tsfile.exception.write.PageException; +import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.block.column.Column; @@ -33,6 +35,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -126,6 +129,30 @@ public void write(long time, Binary value, boolean isNull) { valueChunkWriterList.get(valueIndex++).write(time, value, isNull); } + public void write(long time, int value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + + public void write(long time, long value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + + public void write(long time, boolean value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + + public void write(long time, float value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + + public void write(long time, double value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + + public void write(long time, Binary value, boolean isNull, int valueIndex) { + valueChunkWriterList.get(valueIndex).write(time, value, isNull); + } + public void write(long time, TsPrimitiveType[] points) { valueIndex = 0; for (TsPrimitiveType point : points) { @@ -243,6 +270,16 @@ private void writePageToPageBuffer() { } } + public void writePageHeaderAndDataIntoTimeBuff(ByteBuffer data, PageHeader header) + throws PageException { + timeChunkWriter.writePageHeaderAndDataIntoBuff(data, header); + } + + public void writePageHeaderAndDataIntoValueBuff( + ByteBuffer data, PageHeader header, int valueIndex) throws PageException { + valueChunkWriterList.get(valueIndex).writePageHeaderAndDataIntoBuff(data, header); + } + @Override public void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException { timeChunkWriter.writeToFileWriter(tsfileWriter); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/TimeChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/TimeChunkWriter.java index 59a1fd465b6aa..dc2f2d15f8890 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/TimeChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/TimeChunkWriter.java @@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.compress.ICompressor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; +import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -30,6 +31,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.file.metadata.statistics.TimeStatistics; import org.apache.iotdb.tsfile.utils.PublicBAOS; +import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.write.page.TimePageWriter; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; @@ -37,6 +39,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; public class TimeChunkWriter { @@ -168,6 +173,53 @@ public void writePageToPageBuffer() { } } + public void writePageHeaderAndDataIntoBuff(ByteBuffer data, PageHeader header) + throws PageException { + // write the page header to pageBuffer + try { + logger.debug( + "start to flush a page header into buffer, buffer position {} ", pageBuffer.size()); + // serialize pageHeader see writePageToPageBuffer method + if (numOfPages == 0) { // record the firstPageStatistics + this.firstPageStatistics = header.getStatistics(); + this.sizeWithoutStatistic += + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + this.sizeWithoutStatistic += + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer + byte[] b = pageBuffer.toByteArray(); + pageBuffer.reset(); + pageBuffer.write(b, 0, this.sizeWithoutStatistic); + firstPageStatistics.serialize(pageBuffer); + pageBuffer.write(b, this.sizeWithoutStatistic, b.length - this.sizeWithoutStatistic); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + header.getStatistics().serialize(pageBuffer); + firstPageStatistics = null; + } else { + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + header.getStatistics().serialize(pageBuffer); + } + logger.debug( + "finish to flush a page header {} of time page into buffer, buffer position {} ", + header, + pageBuffer.size()); + + statistics.mergeStatistics(header.getStatistics()); + + } catch (IOException e) { + throw new PageException("IO Exception in writeDataPageHeader,ignore this page", e); + } + numOfPages++; + // write page content to temp PBAOS + try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) { + channel.write(data); + } catch (IOException e) { + throw new PageException(e); + } + } + public void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException { sealCurrentPage(); writeAllPagesOfChunkToTsFile(tsfileWriter); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java index 8b75269388f6d..7ad496988ce1a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java @@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.compress.ICompressor; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; +import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -30,6 +31,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.PublicBAOS; +import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.write.page.ValuePageWriter; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; @@ -38,6 +40,9 @@ import java.io.IOException; import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; public class ValueChunkWriter { @@ -187,6 +192,57 @@ public void writePageToPageBuffer() { } } + public void writePageHeaderAndDataIntoBuff(ByteBuffer data, PageHeader header) + throws PageException { + // write the page header to pageBuffer + try { + logger.debug( + "start to flush a page header into buffer, buffer position {} ", pageBuffer.size()); + // serialize pageHeader see writePageToPageBuffer method + if (numOfPages == 0) { // record the firstPageStatistics + if (header.getStatistics() != null) { + this.firstPageStatistics = header.getStatistics(); + } + this.sizeWithoutStatistic += + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + this.sizeWithoutStatistic += + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer + if (firstPageStatistics != null) { + byte[] b = pageBuffer.toByteArray(); + pageBuffer.reset(); + pageBuffer.write(b, 0, this.sizeWithoutStatistic); + firstPageStatistics.serialize(pageBuffer); + pageBuffer.write(b, this.sizeWithoutStatistic, b.length - this.sizeWithoutStatistic); + } + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + header.getStatistics().serialize(pageBuffer); + firstPageStatistics = null; + } else { + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + header.getStatistics().serialize(pageBuffer); + } + logger.debug( + "finish to flush a page header {} of time page into buffer, buffer position {} ", + header, + pageBuffer.size()); + + statistics.mergeStatistics(header.getStatistics()); + + } catch (IOException e) { + throw new PageException("IO Exception in writeDataPageHeader,ignore this page", e); + } + numOfPages++; + // write page content to temp PBAOS + try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) { + channel.write(data); + } catch (IOException e) { + throw new PageException(e); + } + } + public void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException { sealCurrentPage(); writeAllPagesOfChunkToTsFile(tsfileWriter); From 50b63be505ba05113eb5e0666d66aaaa52a034b0 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 22 Aug 2022 16:28:46 +0800 Subject: [PATCH 10/37] spotless --- .../iotdb/db/engine/load/AlignedChunkData.java | 2 -- .../iotdb/db/engine/load/LoadTsFileManager.java | 1 + .../iotdb/db/engine/load/NonAlignedChunkData.java | 3 ++- .../plan/node/load/LoadSingleTsFileNode.java | 14 +++++++++----- .../plan/node/load/LoadTsFilePieceNode.java | 1 + .../scheduler/load/LoadTsFileDispatcherImpl.java | 3 ++- .../plan/scheduler/load/LoadTsFileScheduler.java | 4 ++-- .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 1 - .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 1 - .../iotdb/tsfile/write/chunk/IChunkWriter.java | 1 - .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 14 +++++++------- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index d0b5460b11c76..ae922aa9e05df 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -40,7 +40,6 @@ import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -import org.checkerframework.checker.units.qual.A; import java.io.DataOutputStream; import java.io.File; @@ -49,7 +48,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import java.util.function.Function; public class AlignedChunkData implements ChunkData { private static final int DEFAULT_INT32 = 0; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 165911a53d55a..10e179002651b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -32,6 +32,7 @@ import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java index c755515fb9cfe..eabc21ca4dd4d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -266,7 +266,8 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept } } - public static NonAlignedChunkData deserialize(InputStream stream) throws IOException, PageException { + public static NonAlignedChunkData deserialize(InputStream stream) + throws IOException, PageException { long timePartition = ReadWriteIOUtils.readLong(stream); String device = ReadWriteIOUtils.readString(stream); byte chunkType = ReadWriteIOUtils.readByte(stream); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 84b9aad0f9e1a..fbbd25f5f6a23 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -45,6 +45,7 @@ import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,10 +132,12 @@ public List splitByPartition(Analysis analysis) { @Override public String toString() { - return "LoadSingleTsFileNode{" + - "tsFile=" + tsFile + - ", replicaSets=" + replicaSet2Pieces.keySet() + - '}'; + return "LoadSingleTsFileNode{" + + "tsFile=" + + tsFile + + ", replicaSets=" + + replicaSet2Pieces.keySet() + + '}'; } public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOException { @@ -257,7 +260,8 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc header = reader.readChunkHeader(marker); if (header.getDataSize() == 0) { Set allChunkData = new HashSet<>(); - for (Map.Entry> entry : pageIndex2ChunkData.entrySet()) { + for (Map.Entry> entry : + pageIndex2ChunkData.entrySet()) { for (AlignedChunkData alignedChunkData : entry.getValue()) { if (!allChunkData.contains(alignedChunkData)) { alignedChunkData.addValueChunk(-2, header); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index c5ff0d501d54f..57c6c5afaff92 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index c6459142bd5cb..ad9d31dce043b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.mpp.plan.scheduler.load; -import io.airlift.concurrent.SetThreadName; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; @@ -40,6 +39,8 @@ import org.apache.iotdb.mpp.rpc.thrift.TLoadTsFileReq; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; + +import io.airlift.concurrent.SetThreadName; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index fe642fc9416af..b5822da667a65 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -35,10 +35,10 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IScheduler; - -import io.airlift.units.Duration; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.apache.iotdb.rpc.TSStatusCode; + +import io.airlift.units.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index d382d046b0b1f..810715b5ca2c0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -33,7 +33,6 @@ import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; -import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 25efb86f25058..d603c58efb446 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -36,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataOutputStream; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java index bc0acdca7ea9e..992862ed830ba 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java @@ -20,7 +20,6 @@ import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; -import java.io.DataOutputStream; import java.io.IOException; /** IChunkWriter provides a list of writing methods for different value types. */ diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java index 7ad496988ce1a..98d67a6954447 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ValueChunkWriter.java @@ -193,20 +193,20 @@ public void writePageToPageBuffer() { } public void writePageHeaderAndDataIntoBuff(ByteBuffer data, PageHeader header) - throws PageException { + throws PageException { // write the page header to pageBuffer try { logger.debug( - "start to flush a page header into buffer, buffer position {} ", pageBuffer.size()); + "start to flush a page header into buffer, buffer position {} ", pageBuffer.size()); // serialize pageHeader see writePageToPageBuffer method if (numOfPages == 0) { // record the firstPageStatistics if (header.getStatistics() != null) { this.firstPageStatistics = header.getStatistics(); } this.sizeWithoutStatistic += - ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getUncompressedSize(), pageBuffer); this.sizeWithoutStatistic += - ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(header.getCompressedSize(), pageBuffer); } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer if (firstPageStatistics != null) { byte[] b = pageBuffer.toByteArray(); @@ -225,9 +225,9 @@ public void writePageHeaderAndDataIntoBuff(ByteBuffer data, PageHeader header) header.getStatistics().serialize(pageBuffer); } logger.debug( - "finish to flush a page header {} of time page into buffer, buffer position {} ", - header, - pageBuffer.size()); + "finish to flush a page header {} of time page into buffer, buffer position {} ", + header, + pageBuffer.size()); statistics.mergeStatistics(header.getStatistics()); From 2fa85e117fe7d5665634679d02d1050c9ec3898b Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 22 Aug 2022 19:02:38 +0800 Subject: [PATCH 11/37] finish first phase --- .../iotdb/db/engine/load/LoadTsFileManager.java | 2 +- .../db/mpp/plan/execution/QueryExecution.java | 3 ++- .../scheduler/load/LoadTsFileDispatcherImpl.java | 16 ++-------------- .../plan/scheduler/load/LoadTsFileScheduler.java | 15 ++++++++++++++- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 10e179002651b..bcd43cfbe3a2b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -80,7 +80,7 @@ public void writeToDataRegion(DataRegion dataRegion, LoadTsFilePieceNode pieceNo for (ChunkData chunkData : pieceNode.getAllChunkData()) { writerManager.write( getDataPartition( - dataRegion.getLogicalStorageGroupName(), + dataRegion.getStorageGroupName(), dataRegion.getDataRegionId(), chunkData.getTimePartitionSlot()), chunkData); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index 51866bd023b5f..8d6b3ffcea2ee 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -237,7 +237,8 @@ private Analysis analyze( private void schedule() { if (rawStatement instanceof LoadTsFileStatement) { this.scheduler = - new LoadTsFileScheduler(distributedPlan, context, internalServiceClientManager); + new LoadTsFileScheduler( + distributedPlan, context, stateMachine, internalServiceClientManager); this.scheduler.start(); return; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index ad9d31dce043b..b944800ea02ef 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -121,13 +121,7 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) TLoadResp loadResp = client.sendLoadNode(loadTsFileReq); if (!loadResp.isAccepted()) { logger.error(loadResp.message); - TSStatus status = new TSStatus(); - status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); - status.setMessage(loadResp.message); - if (loadResp.status != null) { - status.addToSubStatus(loadResp.status); - } - throw new FragmentInstanceDispatchException(status); + throw new FragmentInstanceDispatchException(loadResp.status); } } catch (IOException | TException e) { logger.error("can't connect to node {}", endPoint, e); @@ -215,13 +209,7 @@ private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint) TLoadResp loadResp = client.sendLoadCommand(loadCommandReq); if (!loadResp.isAccepted()) { logger.error(loadResp.message); - TSStatus status = new TSStatus(); - status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); - status.setMessage(loadResp.message); - if (loadResp.status != null) { - status.addToSubStatus(loadResp.status); - } - throw new FragmentInstanceDispatchException(status); + throw new FragmentInstanceDispatchException(loadResp.status); } } catch (IOException | TException e) { logger.error("can't connect to node {}", endPoint, e); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index b5822da667a65..1ff2a6c50f0f6 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.mpp.common.FragmentInstanceId; import org.apache.iotdb.db.mpp.common.MPPQueryContext; import org.apache.iotdb.db.mpp.common.PlanFragmentId; +import org.apache.iotdb.db.mpp.execution.QueryStateMachine; import org.apache.iotdb.db.mpp.execution.fragment.FragmentInfo; import org.apache.iotdb.db.mpp.plan.planner.plan.DistributedQueryPlan; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; @@ -56,6 +57,7 @@ public class LoadTsFileScheduler implements IScheduler { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); private final MPPQueryContext queryContext; + private QueryStateMachine stateMachine; private LoadTsFileDispatcherImpl dispatcher; private List tsFileNodeList; private PlanFragmentId fragmentId; @@ -65,8 +67,10 @@ public class LoadTsFileScheduler implements IScheduler { public LoadTsFileScheduler( DistributedQueryPlan distributedQueryPlan, MPPQueryContext queryContext, + QueryStateMachine stateMachine, IClientManager internalServiceClientManager) { this.queryContext = queryContext; + this.stateMachine = stateMachine; this.tsFileNodeList = new ArrayList<>(); this.fragmentId = distributedQueryPlan.getRootSubPlan().getPlanFragment().getId(); this.dispatcher = new LoadTsFileDispatcherImpl(internalServiceClientManager); @@ -79,14 +83,19 @@ public LoadTsFileScheduler( @Override public void start() { + stateMachine.transitionToRunning(); for (LoadSingleTsFileNode node : tsFileNodeList) { String uuid = UUID.randomUUID().toString(); dispatcher.setUuid(uuid); allReplicaSets.clear(); boolean isFirstPhaseSuccess = firstPhase(node); - boolean isSuccess = secondPhase(isFirstPhaseSuccess, uuid); + boolean isSecondPhaseSuccess = secondPhase(isFirstPhaseSuccess, uuid); + if (!isFirstPhaseSuccess || !isSecondPhaseSuccess) { + return; + } } + stateMachine.transitionToFinished(); } private boolean firstPhase(LoadSingleTsFileNode node) { @@ -134,6 +143,7 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { } } logger.error(String.format("Dispatch piece node:%n%s", pieceNode)); + stateMachine.transitionToFailed(result.getFailureStatus()); // TODO: record more status return false; } } catch (InterruptedException | ExecutionException e) { @@ -141,6 +151,7 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { Thread.currentThread().interrupt(); } logger.warn("Interrupt or Execution error.", e); + stateMachine.transitionToFailed(e); return false; } } @@ -164,6 +175,7 @@ private boolean secondPhase(boolean isFirstPhaseSuccess, String uuid) { logger.error(String.format("Result status code %s.", result.getFailureStatus().getCode())); logger.error( String.format("Result status message %s.", result.getFailureStatus().getMessage())); + stateMachine.transitionToFailed(result.getFailureStatus()); return false; } } catch (InterruptedException | ExecutionException e) { @@ -171,6 +183,7 @@ private boolean secondPhase(boolean isFirstPhaseSuccess, String uuid) { Thread.currentThread().interrupt(); } logger.warn("Interrupt or Execution error.", e); + stateMachine.transitionToFailed(e); return false; } return true; From 519b5ec78dfb7f66b32721a6f5617fdfb47208ca Mon Sep 17 00:00:00 2001 From: yschengzi Date: Tue, 23 Aug 2022 01:02:42 +0800 Subject: [PATCH 12/37] finish local dispatcher --- .../load/LoadTsFileDispatcherImpl.java | 68 ++++++++----------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index b944800ea02ef..aa5333d94176c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -25,13 +25,13 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; -import org.apache.iotdb.db.exception.sql.SemanticException; -import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; @@ -133,42 +133,23 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) } private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDispatchException { - PlanNode planNode = instance.getFragment().getRoot(); - boolean hasFailedMeasurement = false; - String partialInsertMessage = null; - if (planNode instanceof InsertNode) { - InsertNode insertNode = (InsertNode) planNode; - try { - SchemaValidator.validate(insertNode); - } catch (SemanticException e) { - throw new FragmentInstanceDispatchException(e); - } - hasFailedMeasurement = insertNode.hasFailedMeasurements(); - if (hasFailedMeasurement) { - partialInsertMessage = - String.format( - "Fail to insert measurements %s caused by %s", - insertNode.getFailedMeasurements(), insertNode.getFailedMessages()); - logger.warn(partialInsertMessage); - } + logger.info(String.format("Receive load node from uuid %s.", uuid)); + + ConsensusGroupId groupId = + ConsensusGroupId.Factory.createFromTConsensusGroupId( + instance.getRegionReplicaSet().getRegionId()); + LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) instance.getFragment().getRoot(); + if (pieceNode == null) { + throw new FragmentInstanceDispatchException( + new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); + } + + TSStatus resultStatus = + StorageEngineV2.getInstance().writeLoadTsFileNode((DataRegionId) groupId, pieceNode, uuid); + + if (!RpcUtils.SUCCESS_STATUS.equals(resultStatus)) { + throw new FragmentInstanceDispatchException(resultStatus); } - // TODO: write to local interface - - // ConsensusWriteResponse writeResponse; - // if (groupId instanceof DataRegionId) { - // writeResponse = DataRegionConsensusImpl.getInstance().write(groupId, planNode); - // } else { - // writeResponse = SchemaRegionConsensusImpl.getInstance().write(groupId, planNode); - // } - // - // if (writeResponse.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // logger.error(writeResponse.getStatus().message); - // throw new FragmentInstanceDispatchException(writeResponse.getStatus()); - // } else if (hasFailedMeasurement) { - // throw new FragmentInstanceDispatchException( - // RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(), - // partialInsertMessage)); - // } } public Future dispatchCommand( @@ -222,7 +203,14 @@ private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint) private void dispatchLocally(TLoadCommandReq loadCommandReq) throws FragmentInstanceDispatchException { - // TODO: use local interface + TSStatus resultStatus = + StorageEngineV2.getInstance() + .executeLoadCommand( + LoadTsFileScheduler.LoadCommand.values()[loadCommandReq.commandType], + loadCommandReq.uuid); + if (!RpcUtils.SUCCESS_STATUS.equals(resultStatus)) { + throw new FragmentInstanceDispatchException(resultStatus); + } } @Override From 7251f204c0ed693022237290e1697d52cb282a3b Mon Sep 17 00:00:00 2001 From: yschengzi Date: Tue, 23 Aug 2022 15:08:33 +0800 Subject: [PATCH 13/37] finish data region interface --- .../iotdb/db/engine/StorageEngineV2.java | 4 + .../db/engine/load/LoadTsFileManager.java | 91 +++++++++++++------ 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index a4aa15d9cf526..9358c688c71b1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.engine.storagegroup.DataRegion; import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.exception.WriteProcessRejectException; @@ -790,6 +791,9 @@ public TSStatus executeLoadCommand(LoadTsFileScheduler.LoadCommand loadCommand, } catch (IOException e) { status.setCode(TSStatusCode.DATA_REGION_ERROR.getStatusCode()); status.setMessage(e.getMessage()); + } catch (LoadFileException e) { + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); } return RpcUtils.SUCCESS_STATUS; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index bcd43cfbe3a2b..2298f6d181362 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.engine.storagegroup.DataRegion; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.exception.write.PageException; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; public class LoadTsFileManager { @@ -79,15 +81,11 @@ public void writeToDataRegion(DataRegion dataRegion, LoadTsFilePieceNode pieceNo uuid, o -> new TsFileWriterManager(SystemFileFactory.INSTANCE.getFile(loadDir, uuid))); for (ChunkData chunkData : pieceNode.getAllChunkData()) { writerManager.write( - getDataPartition( - dataRegion.getStorageGroupName(), - dataRegion.getDataRegionId(), - chunkData.getTimePartitionSlot()), - chunkData); + new DataPartitionInfo(dataRegion, chunkData.getTimePartitionSlot()), chunkData); } } - public boolean loadAll(String uuid) throws IOException { + public boolean loadAll(String uuid) throws IOException, LoadFileException { if (!uuid2WriterManager.containsKey(uuid)) { return false; } @@ -103,15 +101,6 @@ public boolean deleteAll(String uuid) throws IOException { return true; } - private String getDataPartition( - String logicalName, String dataRegionId, TTimePartitionSlot timePartitionSlot) { - return String.join( - IoTDBConstant.FILE_NAME_SEPARATOR, - logicalName, - dataRegionId, - Long.toString(timePartitionSlot.getStartTime())); - } - private String getNewTsFileName(String dataPartition) { lock.lock(); try { @@ -128,8 +117,8 @@ private String getNewTsFileName(String dataPartition) { private class TsFileWriterManager { private final File taskDir; - private Map dataPartition2Writer; - private Map dataPartition2LastDevice; + private Map dataPartition2Writer; + private Map dataPartition2LastDevice; private TsFileWriterManager(File taskDir) { this.taskDir = taskDir; @@ -139,36 +128,35 @@ private TsFileWriterManager(File taskDir) { clearDir(taskDir); } - private void write(String dataPartition, ChunkData chunkData) - throws IOException, PageException { - if (!dataPartition2Writer.containsKey(dataPartition)) { + private void write(DataPartitionInfo partitionInfo, ChunkData chunkData) throws IOException { + if (!dataPartition2Writer.containsKey(partitionInfo)) { File newTsFile = - SystemFileFactory.INSTANCE.getFile(taskDir, getNewTsFileName(dataPartition)); + SystemFileFactory.INSTANCE.getFile(taskDir, getNewTsFileName(partitionInfo.toString())); if (!newTsFile.createNewFile()) { logger.error(String.format("Can not create TsFile %s for writing.", newTsFile.getPath())); return; } - dataPartition2Writer.put(dataPartition, new TsFileIOWriter(newTsFile)); + dataPartition2Writer.put(partitionInfo, new TsFileIOWriter(newTsFile)); } - TsFileIOWriter writer = dataPartition2Writer.get(dataPartition); - if (!chunkData.getDevice().equals(dataPartition2LastDevice.getOrDefault(dataPartition, ""))) { - if (dataPartition2LastDevice.containsKey(dataPartition)) { + TsFileIOWriter writer = dataPartition2Writer.get(partitionInfo); + if (!chunkData.getDevice().equals(dataPartition2LastDevice.getOrDefault(partitionInfo, ""))) { + if (dataPartition2LastDevice.containsKey(partitionInfo)) { writer.endChunkGroup(); } writer.startChunkGroup(chunkData.getDevice()); } - chunkData.getChunkWriter().writeToFileWriter(writer); // TODO: get writer + chunkData.getChunkWriter().writeToFileWriter(writer); } - private void loadAll() throws IOException { - for (Map.Entry entry : dataPartition2Writer.entrySet()) { + private void loadAll() throws IOException, LoadFileException { + for (Map.Entry entry : dataPartition2Writer.entrySet()) { TsFileIOWriter writer = entry.getValue(); if (writer.isWritingChunkGroup()) { writer.endChunkGroup(); } writer.endFile(); - generateResource(writer); // TODO: load TsFileResource + entry.getKey().getDataRegion().loadNewTsFile(generateResource(writer), true); } } @@ -190,12 +178,55 @@ private TsFileResource generateResource(TsFileIOWriter writer) throws IOExceptio } private void close() throws IOException { - for (Map.Entry entry : dataPartition2Writer.entrySet()) { + for (Map.Entry entry : dataPartition2Writer.entrySet()) { entry.getValue().close(); } if (taskDir.delete()) { logger.warn(String.format("Can not delete load uuid dir %s.", taskDir.getPath())); } + dataPartition2Writer = null; + dataPartition2LastDevice = null; + } + } + + private class DataPartitionInfo { + private final DataRegion dataRegion; + private final TTimePartitionSlot timePartitionSlot; + + private DataPartitionInfo(DataRegion dataRegion, TTimePartitionSlot timePartitionSlot) { + this.dataRegion = dataRegion; + this.timePartitionSlot = timePartitionSlot; + } + + public DataRegion getDataRegion() { + return dataRegion; + } + + public TTimePartitionSlot getTimePartitionSlot() { + return timePartitionSlot; + } + + @Override + public String toString() { + return String.join( + IoTDBConstant.FILE_NAME_SEPARATOR, + dataRegion.getStorageGroupName(), + dataRegion.getDataRegionId(), + Long.toString(timePartitionSlot.getStartTime())); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataPartitionInfo that = (DataPartitionInfo) o; + return Objects.equals(dataRegion, that.dataRegion) + && timePartitionSlot.getStartTime() == that.timePartitionSlot.getStartTime(); + } + + @Override + public int hashCode() { + return Objects.hash(dataRegion, timePartitionSlot.getStartTime()); } } } From 42dbf5145db73efca8c3dfa1c9be79c6a9a07ef8 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 25 Aug 2022 14:12:07 +0800 Subject: [PATCH 14/37] finish entire chunk --- .../iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../db/engine/load/AlignedChunkData.java | 95 +++++++++++--- .../iotdb/db/engine/load/ChunkData.java | 9 +- .../db/engine/load/LoadTsFileManager.java | 2 +- .../db/engine/load/NonAlignedChunkData.java | 76 +++++++++-- .../plan/planner/plan/FragmentInstance.java | 3 + .../plan/node/load/LoadSingleTsFileNode.java | 121 +++++++++++++----- .../plan/node/load/LoadTsFilePieceNode.java | 1 + .../load/LoadTsFileDispatcherImpl.java | 5 +- .../tsfile/read/TsFileSequenceReader.java | 2 +- .../tsfile/write/writer/TsFileIOWriter.java | 13 ++ 11 files changed, 261 insertions(+), 68 deletions(-) diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index 5aacec0152e5c..2eee1264ff37e 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "alignedTablet.tsfile"; + String filename = "D:\\Pro\\IoTDB\\data\\load\\1655902209941-1-0-0.tsfile"; if (args.length >= 1) { filename = args[0]; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index ae922aa9e05df..e423ca5881aac 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -28,23 +28,27 @@ import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; import org.apache.iotdb.tsfile.read.reader.page.ValuePageReader; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl; -import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -65,10 +69,12 @@ public class AlignedChunkData implements ChunkData { private TTimePartitionSlot timePartitionSlot; private String device; private List chunkHeaderList; + private List chunkMetadataList; private List timeBatch; private AlignedChunkWriterImpl chunkWriter; + private List chunkList; public AlignedChunkData(long timeOffset, String device, ChunkHeader chunkHeader) { this.offset = new ArrayList<>(); @@ -103,6 +109,17 @@ public void addDataSize(long pageSize) { dataSize.set(0, dataSize.get(0) + pageSize); } + @Override + public void setNotDecode(IChunkMetadata chunkMetadata) { + chunkMetadataList = new ArrayList<>(); + chunkMetadataList.add(chunkMetadata); + } + + @Override + public boolean needDecodeChunk() { + return chunkMetadataList == null; + } + @Override public void setHeadPageNeedDecode(boolean headPageNeedDecode) { isHeadPageNeedDecode = headPageNeedDecode; @@ -123,10 +140,24 @@ public boolean isAligned() { return true; } - public void addValueChunk(long offset, ChunkHeader chunkHeader) { + @Override + public void writeToFileWriter(TsFileIOWriter writer) throws IOException { + if (chunkList != null) { + for (Chunk chunk : chunkList) { + writer.writeChunk(chunk); + } + } else { + chunkWriter.writeToFileWriter(writer); + } + } + + public void addValueChunk(long offset, ChunkHeader chunkHeader, IChunkMetadata chunkMetadata) { this.offset.add(offset); this.dataSize.add(0L); this.chunkHeaderList.add(chunkHeader); + if (chunkMetadataList != null) { + chunkMetadataList.add(chunkMetadata); + } } public void addValueChunkDataSize(long dataSize) { @@ -134,11 +165,6 @@ public void addValueChunkDataSize(long dataSize) { this.dataSize.set(lastIndex, this.dataSize.get(lastIndex) + dataSize); } - @Override - public IChunkWriter getChunkWriter() { - return chunkWriter; - } - @Override public void serialize(DataOutputStream stream, File tsFile) throws IOException { ReadWriteIOUtils.write(isAligned(), stream); @@ -158,16 +184,35 @@ private void serializeAttr(DataOutputStream stream) throws IOException { private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { timeBatch = new ArrayList<>(); + ReadWriteIOUtils.write(needDecodeChunk(), stream); try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { int chunkSize = offset.size(); for (int i = 0; i < chunkSize; i++) { - serializeChunk(stream, reader, chunkHeaderList.get(i), i); + if (needDecodeChunk()) { + serializeDecodeChunk(stream, reader, chunkHeaderList.get(i), i); + } else { + serializeEntireChunk(stream, reader, chunkHeaderList.get(i), chunkMetadataList.get(i)); + } } } timeBatch = null; } - private void serializeChunk( + private void serializeEntireChunk( + DataOutputStream stream, + TsFileSequenceReader reader, + ChunkHeader chunkHeader, + IChunkMetadata chunkMetadata) + throws IOException { + ByteBuffer chunkData = + reader.readChunk( + chunkMetadata.getOffsetOfChunkHeader() + chunkHeader.getSerializedSize(), + chunkHeader.getDataSize()); + ReadWriteIOUtils.write(chunkData, stream); + chunkMetadata.getStatistics().serialize(stream); + } + + private void serializeDecodeChunk( DataOutputStream stream, TsFileSequenceReader reader, ChunkHeader chunkHeader, int chunkIndex) throws IOException { Decoder defaultTimeDecoder = @@ -194,8 +239,7 @@ private void serializeChunk( } } else { // entire page ReadWriteIOUtils.write(false, stream); // don't decode - ReadWriteIOUtils.write(hasStatistic, stream); - pageHeader.serializeTo(stream); // TODO: can save this time by recording ByteBuffer + pageHeader.serializeTo(stream); ByteBuffer pageData = reader.readCompressedPage(pageHeader); ReadWriteIOUtils.write(pageData, stream); } @@ -277,6 +321,27 @@ private void decodeValuePage( } } + private void deserializeTsFileData(InputStream stream) throws IOException, PageException { + boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); + if (needDecodeChunk) { + buildChunkWriter(stream); + } else { + deserializeEntireChunk(stream); + } + } + + private void deserializeEntireChunk(InputStream stream) throws IOException { + chunkList = new ArrayList<>(); + int chunkSize = chunkHeaderList.size(); + for (int i = 0; i < chunkSize; i++) { + ByteBuffer chunkData = + ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)); + Statistics statistics = + Statistics.deserialize(stream, chunkHeaderList.get(i).getDataType()); + chunkList.add(new Chunk(chunkHeaderList.get(i), chunkData, null, statistics)); + } + } + private void buildChunkWriter(InputStream stream) throws IOException, PageException { List measurementSchemaList = new ArrayList<>(); for (ChunkHeader chunkHeader : chunkHeaderList) { @@ -359,9 +424,7 @@ private void buildChunk( } decodePageIndex += 1; } else { - boolean hasStatistic = ReadWriteIOUtils.readBool(stream); - PageHeader pageHeader = - PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), hasStatistic); + PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); if (isTimeChunk) { chunkWriter.writePageHeaderAndDataIntoTimeBuff( ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), @@ -388,10 +451,10 @@ public static AlignedChunkData deserialize(InputStream stream) throws IOExceptio AlignedChunkData chunkData = new AlignedChunkData(-1, device, chunkHeaderList[0]); for (int i = 1; i < chunkHeaderListSize; i++) { - chunkData.addValueChunk(-1, chunkHeaderList[i]); + chunkData.addValueChunk(-1, chunkHeaderList[i], null); } chunkData.setTimePartitionSlot(StorageEngineV2.getTimePartitionSlot(timePartition)); - chunkData.buildChunkWriter(stream); + chunkData.deserializeTsFileData(stream); return chunkData; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java index a3c004673b91c..36ab1e87fa026 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/ChunkData.java @@ -22,8 +22,9 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; import java.io.DataOutputStream; import java.io.File; @@ -39,6 +40,10 @@ public interface ChunkData { void addDataSize(long pageSize); + void setNotDecode(IChunkMetadata chunkMetadata); + + boolean needDecodeChunk(); + void setHeadPageNeedDecode(boolean headPageNeedDecode); void setTailPageNeedDecode(boolean tailPageNeedDecode); @@ -47,7 +52,7 @@ public interface ChunkData { boolean isAligned(); - IChunkWriter getChunkWriter(); + void writeToFileWriter(TsFileIOWriter writer) throws IOException; void serialize(DataOutputStream stream, File tsFile) throws IOException; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 2298f6d181362..4bcc8a5137296 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -146,7 +146,7 @@ private void write(DataPartitionInfo partitionInfo, ChunkData chunkData) throws } writer.startChunkGroup(chunkData.getDevice()); } - chunkData.getChunkWriter().writeToFileWriter(writer); + chunkData.writeToFileWriter(writer); } private void loadAll() throws IOException, LoadFileException { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java index eabc21ca4dd4d..141f91f9d3c19 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -28,21 +28,25 @@ import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; -import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.ByteBuffer; public class NonAlignedChunkData implements ChunkData { @@ -54,8 +58,10 @@ public class NonAlignedChunkData implements ChunkData { private TTimePartitionSlot timePartitionSlot; private String device; private ChunkHeader chunkHeader; + private IChunkMetadata chunkMetadata; private ChunkWriterImpl chunkWriter; + private Chunk chunk; public NonAlignedChunkData(long offset, String device, ChunkHeader chunkHeader) { this.offset = offset; @@ -86,6 +92,16 @@ public void addDataSize(long pageSize) { dataSize += pageSize; } + @Override + public void setNotDecode(IChunkMetadata chunkMetadata) { + this.chunkMetadata = chunkMetadata; + } + + @Override + public boolean needDecodeChunk() { + return chunkMetadata == null; + } + @Override public void setHeadPageNeedDecode(boolean headPageNeedDecode) { isHeadPageNeedDecode = headPageNeedDecode; @@ -107,25 +123,45 @@ public boolean isAligned() { } @Override - public IChunkWriter getChunkWriter() { - return chunkWriter; + public void writeToFileWriter(TsFileIOWriter writer) throws IOException { + if (chunk != null) { + writer.writeChunk(chunk); + } else { + chunkWriter.writeToFileWriter(writer); + } } @Override public void serialize(DataOutputStream stream, File tsFile) throws IOException { ReadWriteIOUtils.write(isAligned(), stream); serializeAttr(stream); - serializeTsFileData(stream, tsFile); + if (needDecodeChunk()) { + ReadWriteIOUtils.write(true, stream); + serializeDecodeChunk(stream, tsFile); + } else { + ReadWriteIOUtils.write(false, stream); + serializeEntireChunk(stream, tsFile); + } } private void serializeAttr(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); ReadWriteIOUtils.write(device, stream); - ReadWriteIOUtils.write(chunkHeader.getChunkType(), stream); - chunkHeader.serializeTo(stream); + chunkHeader.serializeTo(stream); // chunk header already serialize chunk type } - private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { + private void serializeEntireChunk(DataOutputStream stream, File tsFile) throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + ByteBuffer chunkData = + reader.readChunk( + chunkMetadata.getOffsetOfChunkHeader() + chunkHeader.getSerializedSize(), + chunkHeader.getDataSize()); + ReadWriteIOUtils.write(chunkData, stream); + chunkMetadata.getStatistics().serialize(stream); + } + } + + private void serializeDecodeChunk(DataOutputStream stream, File tsFile) throws IOException { try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { Decoder defaultTimeDecoder = Decoder.getDecoderByType( @@ -146,8 +182,7 @@ private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IO decodePage(reader, pageHeader, defaultTimeDecoder, valueDecoder, stream); } else { // entire page ReadWriteIOUtils.write(false, stream); // don't decode - ReadWriteIOUtils.write(hasStatistic, stream); - pageHeader.serializeTo(stream); // TODO: can save this time by recording ByteBuffer + pageHeader.serializeTo(stream); ByteBuffer pageData = reader.readCompressedPage(pageHeader); ReadWriteIOUtils.write(pageData, stream); } @@ -210,6 +245,15 @@ private void decodePage( } } + private void deserializeTsFileData(InputStream stream) throws IOException, PageException { + boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); + if (needDecodeChunk) { + buildChunkWriter(stream); + } else { + deserializeEntireChunk(stream); + } + } + private void buildChunkWriter(InputStream stream) throws IOException, PageException { chunkWriter = new ChunkWriterImpl( @@ -256,9 +300,7 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept chunkWriter.sealCurrentPage(); } else { - boolean hasStatistic = ReadWriteIOUtils.readBool(stream); - PageHeader pageHeader = - PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), hasStatistic); + PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); chunkWriter.writePageHeaderAndDataIntoBuff( ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), pageHeader); @@ -266,6 +308,14 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept } } + private void deserializeEntireChunk(InputStream stream) throws IOException { + ByteBuffer chunkData = + ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)); + Statistics statistics = + Statistics.deserialize(stream, chunkHeader.getDataType()); + chunk = new Chunk(chunkHeader, chunkData, null, statistics); + } + public static NonAlignedChunkData deserialize(InputStream stream) throws IOException, PageException { long timePartition = ReadWriteIOUtils.readLong(stream); @@ -274,7 +324,7 @@ public static NonAlignedChunkData deserialize(InputStream stream) ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); NonAlignedChunkData chunkData = new NonAlignedChunkData(-1, device, chunkHeader); chunkData.setTimePartitionSlot(StorageEngineV2.getTimePartitionSlot(timePartition)); - chunkData.buildChunkWriter(stream); + chunkData.deserializeTsFileData(stream); return chunkData; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/FragmentInstance.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/FragmentInstance.java index bcc54931de362..13ae613c83192 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/FragmentInstance.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/FragmentInstance.java @@ -84,6 +84,9 @@ public TRegionReplicaSet getDataRegionId() { } public void setDataRegionAndHost(TRegionReplicaSet regionReplicaSet) { + if (regionReplicaSet == null) { + return; + } this.regionReplicaSet = regionReplicaSet; // TODO: (xingtanzjr) We select the first Endpoint as the default target host for current // instance diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index fbbd25f5f6a23..1720792d8767b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -39,6 +39,8 @@ import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -145,14 +147,17 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc List chunkDataList = new ArrayList<>(); try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { - if (checkMagic(reader)) { + if (!checkMagic(reader)) { throw new TsFileRuntimeException( String.format("Magic String check error when parsing TsFile %s.", tsFile.getPath())); } reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); String curDevice = null; + boolean isTimeChunkNeedDecode = true; Map> pageIndex2ChunkData = null; + Map offset2ChunkMetadata = new HashMap<>(); + getChunkMetadata(reader, offset2ChunkMetadata); byte marker; while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { switch (marker) { @@ -160,30 +165,46 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc case MetaMarker.TIME_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + long chunkOffset = reader.position(); ChunkHeader header = reader.readChunkHeader(marker); if (header.getDataSize() == 0) { - break; + throw new TsFileRuntimeException( + String.format("Chunk data error when parsing TsFile %s.", tsFile.getPath())); } - Decoder defaultTimeDecoder = - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64); - Decoder valueDecoder = - Decoder.getDecoderByType(header.getEncodingType(), header.getDataType()); - int dataSize = header.getDataSize(); - int pageIndex = 0; - boolean isAligned = ((header.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) == TsFileConstant.TIME_COLUMN_MASK); - TTimePartitionSlot timePartitionSlot = null; + IChunkMetadata chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES); + TTimePartitionSlot timePartitionSlot = + StorageEngineV2.getTimePartitionSlot(chunkMetadata.getStartTime()); ChunkData chunkData = ChunkData.createChunkData(isAligned, reader.position(), curDevice, header); + chunkData.setTimePartitionSlot(timePartitionSlot); + if (!needDecodeChunk(chunkMetadata)) { + if (isAligned) { + isTimeChunkNeedDecode = false; + pageIndex2ChunkData = new HashMap<>(); + pageIndex2ChunkData + .computeIfAbsent(1, o -> new ArrayList<>()) + .add((AlignedChunkData) chunkData); + } + chunkData.setNotDecode(chunkMetadata); + break; + } if (isAligned) { + isTimeChunkNeedDecode = true; pageIndex2ChunkData = new HashMap<>(); } + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(header.getEncodingType(), header.getDataType()); + int dataSize = header.getDataSize(); + int pageIndex = 0; while (dataSize > 0) { long pageOffset = reader.position(); PageHeader pageHeader = @@ -191,13 +212,13 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc header.getDataType(), (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); long pageDataSize = pageHeader.getSerializedPageSize(); - if (timePartitionSlot == null) { // init time slot - timePartitionSlot = StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()); - chunkData.setTimePartitionSlot(timePartitionSlot); - } - if (!needDecodePage(pageHeader)) { // an entire page + if (!needDecodePage(pageHeader, chunkMetadata)) { // an entire page + long startTime = + pageHeader.getStatistics() == null + ? chunkMetadata.getStartTime() + : pageHeader.getStartTime(); TTimePartitionSlot pageTimePartitionSlot = - StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()); + StorageEngineV2.getTimePartitionSlot(startTime); if (!timePartitionSlot.equals(pageTimePartitionSlot)) { chunkDataList.add(chunkData); timePartitionSlot = pageTimePartitionSlot; @@ -210,7 +231,7 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc .add((AlignedChunkData) chunkData); } chunkData.addDataSize(pageDataSize); - reader.position(reader.position() + pageDataSize); + reader.position(pageOffset + pageDataSize); } else { // split page ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); long[] timeBatch = @@ -257,22 +278,21 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc break; case MetaMarker.VALUE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + chunkOffset = reader.position(); + chunkMetadata = offset2ChunkMetadata.get(chunkOffset); header = reader.readChunkHeader(marker); if (header.getDataSize() == 0) { - Set allChunkData = new HashSet<>(); - for (Map.Entry> entry : - pageIndex2ChunkData.entrySet()) { - for (AlignedChunkData alignedChunkData : entry.getValue()) { - if (!allChunkData.contains(alignedChunkData)) { - alignedChunkData.addValueChunk(-2, header); - allChunkData.add(alignedChunkData); - } - } - } + handleEmptyValueChunk(chunkOffset, header, chunkMetadata, pageIndex2ChunkData); break; } Set allChunkData = new HashSet<>(); + chunkMetadata = offset2ChunkMetadata.get(chunkOffset); + if (!isTimeChunkNeedDecode) { + pageIndex2ChunkData.get(1).get(0).addValueChunk(chunkOffset, header, chunkMetadata); + break; + } + dataSize = header.getDataSize(); pageIndex = 0; @@ -285,12 +305,12 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc long pageDataSize = pageHeader.getSerializedPageSize(); for (AlignedChunkData alignedChunkData : pageIndex2ChunkData.get(pageIndex)) { if (!allChunkData.contains(alignedChunkData)) { - alignedChunkData.addValueChunk(pageOffset, header); + alignedChunkData.addValueChunk(pageOffset, header, chunkMetadata); allChunkData.add(alignedChunkData); } alignedChunkData.addValueChunkDataSize(pageDataSize); } - reader.position(reader.position() + pageDataSize); + reader.position(pageOffset + pageDataSize); pageIndex += 1; dataSize -= pageDataSize; @@ -335,9 +355,28 @@ private boolean checkMagic(TsFileSequenceReader reader) throws IOException { return true; } - private boolean needDecodePage(PageHeader pageHeader) { + private void getChunkMetadata( + TsFileSequenceReader reader, Map offset2ChunkMetadata) + throws IOException { + Map> device2Metadata = reader.getAllTimeseriesMetadata(true); + for (Map.Entry> entry : device2Metadata.entrySet()) { + for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) { + for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { + offset2ChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata); + } + } + } + } + + private boolean needDecodeChunk(IChunkMetadata chunkMetadata) { + return !StorageEngineV2.getTimePartitionSlot(chunkMetadata.getStartTime()) + .equals(StorageEngineV2.getTimePartitionSlot(chunkMetadata.getEndTime())); + } + + private boolean needDecodePage(PageHeader pageHeader, IChunkMetadata chunkMetadata) { if (pageHeader.getStatistics() == null) { - return true; + return !StorageEngineV2.getTimePartitionSlot(chunkMetadata.getStartTime()) + .equals(StorageEngineV2.getTimePartitionSlot(chunkMetadata.getEndTime())); } return !StorageEngineV2.getTimePartitionSlot(pageHeader.getStartTime()) .equals(StorageEngineV2.getTimePartitionSlot(pageHeader.getEndTime())); @@ -368,4 +407,20 @@ private long[] decodePage( } return timeBatch; } + + private void handleEmptyValueChunk( + long chunkOffset, + ChunkHeader header, + IChunkMetadata chunkMetadata, + Map> pageIndex2ChunkData) { + Set allChunkData = new HashSet<>(); + for (Map.Entry> entry : pageIndex2ChunkData.entrySet()) { + for (AlignedChunkData alignedChunkData : entry.getValue()) { + if (!allChunkData.contains(alignedChunkData)) { + alignedChunkData.addValueChunk(chunkOffset, header, chunkMetadata); + allChunkData.add(alignedChunkData); + } + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 57c6c5afaff92..6b0c76b5ef3c1 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -145,6 +145,7 @@ public List splitByPartition(Analysis analysis) { public static PlanNode deserialize(ByteBuffer buffer) { InputStream stream = new ByteArrayInputStream(buffer.array()); try { + ReadWriteIOUtils.readShort(stream); // read PlanNodeType File tsFile = new File(ReadWriteIOUtils.readString(stream)); LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); while (stream.available() > 0) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index aa5333d94176c..de437e48121d9 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; @@ -138,7 +139,9 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD ConsensusGroupId groupId = ConsensusGroupId.Factory.createFromTConsensusGroupId( instance.getRegionReplicaSet().getRegionId()); - LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) instance.getFragment().getRoot(); + LoadTsFilePieceNode pieceNode = + (LoadTsFilePieceNode) + PlanNodeType.deserialize(instance.getFragment().getRoot().serializeToByteBuffer()); if (pieceNode == null) { throw new FragmentInstanceDispatchException( new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 5f9aea46cc0f5..bc93d9c66788e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -1071,7 +1071,7 @@ private ChunkHeader readChunkHeader(long position, int chunkHeaderSize) throws I * @param position the offset of the chunk data * @return the pages of this chunk */ - private ByteBuffer readChunk(long position, int dataSize) throws IOException { + public ByteBuffer readChunk(long position, int dataSize) throws IOException { try { return readData(position, dataSize); } catch (Throwable t) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 2f865f297f081..22a68bdfd7612 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -234,6 +234,19 @@ public void writeChunk(Chunk chunk, ChunkMetadata chunkMetadata) throws IOExcept } } + public void writeChunk(Chunk chunk) throws IOException { + ChunkHeader chunkHeader = chunk.getHeader(); + currentChunkMetadata = + new ChunkMetadata( + chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), + out.getPosition(), + chunk.getChunkStatistic()); + chunkHeader.serializeTo(out.wrapAsStream()); + out.write(chunk.getData()); + endCurrentChunk(); + } + /** end chunk and write some log. */ public void endCurrentChunk() { chunkMetadataList.add(currentChunkMetadata); From eda0975e3001407aa5b300cb7aee5e7b47c4e122 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 26 Aug 2022 00:10:42 +0800 Subject: [PATCH 15/37] finish debug --- .../confignode/conf/ConfigNodeConfig.java | 2 +- .../iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../apache/iotdb/db/engine/StorageEngine.java | 3 +- .../iotdb/db/engine/StorageEngineV2.java | 3 +- .../db/engine/load/AlignedChunkData.java | 33 ++++++++++++------- .../db/engine/load/LoadTsFileManager.java | 9 ++++- .../db/engine/load/NonAlignedChunkData.java | 17 +++++++++- .../db/engine/storagegroup/DataRegion.java | 10 +++--- .../plan/node/load/LoadSingleTsFileNode.java | 6 ++-- .../plan/node/load/LoadTsFilePieceNode.java | 4 ++- .../iotdb/tsfile/file/header/ChunkHeader.java | 3 +- 12 files changed, 65 insertions(+), 29 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index faef338857a0a..eb300d8711135 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -105,7 +105,7 @@ public class ConfigNodeConfig { IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME; /** Time partition interval in seconds */ - private long timePartitionInterval = 86400; + private long timePartitionInterval = 10; /** Default number of SchemaRegion replicas */ private int schemaReplicationFactor = 1; diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index 2eee1264ff37e..d049bfe3d5e24 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "D:\\Pro\\IoTDB\\data\\load\\1655902209941-1-0-0.tsfile"; + String filename = "D:\\Pro\\IoTDB\\data\\load\\load\\1661433500850-1-0-0.tsfile"; if (args.length >= 1) { filename = args[0]; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 520a2aa3fb24a..105f47da692be 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -740,7 +740,7 @@ public class IoTDBConfig { * Time range for partitioning data inside each storage group, the unit is second. Default time is * a day. */ - private long partitionInterval = 86400; + private long partitionInterval = 10; /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */ private long maxPlanNodeSize = 500 * 1048576L; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index 60e0fecefa5bf..252c41c01d7f6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -146,8 +146,7 @@ public static StorageEngine getInstance() { private static void initTimePartition() { timePartitionInterval = - convertMilliWithPrecision( - IoTDBDescriptor.getInstance().getConfig().getPartitionInterval() * 1000L); + convertMilliWithPrecision(IoTDBDescriptor.getInstance().getConfig().getPartitionInterval()); } public static long convertMilliWithPrecision(long milliTime) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index 9358c688c71b1..e1b7b90c5bb9f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -146,8 +146,7 @@ public static StorageEngineV2 getInstance() { private static void initTimePartition() { timePartitionInterval = - convertMilliWithPrecision( - IoTDBDescriptor.getInstance().getConfig().getPartitionInterval() * 1000L); + convertMilliWithPrecision(IoTDBDescriptor.getInstance().getConfig().getPartitionInterval()); } public static long convertMilliWithPrecision(long milliTime) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index e423ca5881aac..fbefa3fa7bab0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -72,6 +72,7 @@ public class AlignedChunkData implements ChunkData { private List chunkMetadataList; private List timeBatch; + private int satisfiedLength; private AlignedChunkWriterImpl chunkWriter; private List chunkList; @@ -177,8 +178,7 @@ private void serializeAttr(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(device, stream); ReadWriteIOUtils.write(chunkHeaderList.size(), stream); for (ChunkHeader chunkHeader : chunkHeaderList) { - ReadWriteIOUtils.write(chunkHeader.getChunkType(), stream); - chunkHeader.serializeTo(stream); + chunkHeader.serializeTo(stream); // chunk header already serialize chunk type } } @@ -262,15 +262,20 @@ private void decodeTimePage( valueDecoder.reset(); ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); TimePageReader timePageReader = new TimePageReader(pageHeader, pageData, timeDecoder); - long[] time = timePageReader.getNextTimeBatch(); - timeBatch.add(time); - ReadWriteIOUtils.write(time.length, stream); - for (int i = 0; i < time.length; i++) { - if (time[i] < timePartitionSlot.getStartTime()) { + long[] decodeTime = timePageReader.getNextTimeBatch(); + satisfiedLength = 0; + long[] time = new long[decodeTime.length]; + for (int i = 0; i < decodeTime.length; i++) { + if (decodeTime[i] < timePartitionSlot.getStartTime()) { continue; - } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time[i]))) { + } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(decodeTime[i]))) { break; } + time[satisfiedLength++] = decodeTime[i]; + } + timeBatch.add(decodeTime); + ReadWriteIOUtils.write(satisfiedLength, stream); + for (int i = 0; i < satisfiedLength; i++) { ReadWriteIOUtils.write(time[i], stream); } } @@ -287,9 +292,15 @@ private void decodeValuePage( ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); ValuePageReader valuePageReader = new ValuePageReader(pageHeader, pageData, chunkHeader.getDataType(), valueDecoder); - TsPrimitiveType[] valueBatch = valuePageReader.nextValueBatch(timeBatch.get(pageIndex)); - ReadWriteIOUtils.write(valueBatch.length, stream); + long[] time = timeBatch.get(pageIndex); + TsPrimitiveType[] valueBatch = valuePageReader.nextValueBatch(time); + ReadWriteIOUtils.write(satisfiedLength, stream); for (int i = 0; i < valueBatch.length; i++) { + if (time[i] < timePartitionSlot.getStartTime()) { + continue; + } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time[i]))) { + break; + } if (valueBatch[i] == null) { ReadWriteIOUtils.write(true, stream); continue; @@ -446,7 +457,7 @@ public static AlignedChunkData deserialize(InputStream stream) throws IOExceptio ChunkHeader[] chunkHeaderList = new ChunkHeader[chunkHeaderListSize]; for (int i = 0; i < chunkHeaderListSize; i++) { byte chunkType = ReadWriteIOUtils.readByte(stream); - ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); + chunkHeaderList[i] = ChunkHeader.deserializeFrom(stream, chunkType); } AlignedChunkData chunkData = new AlignedChunkData(-1, device, chunkHeaderList[0]); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 4bcc8a5137296..7147f5212c8e8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -90,6 +90,8 @@ public boolean loadAll(String uuid) throws IOException, LoadFileException { return false; } uuid2WriterManager.get(uuid).loadAll(); + uuid2WriterManager.get(uuid).close(); + uuid2WriterManager.remove(uuid); return true; } @@ -98,6 +100,7 @@ public boolean deleteAll(String uuid) throws IOException { return false; } uuid2WriterManager.get(uuid).close(); + uuid2WriterManager.remove(uuid); return true; } @@ -145,6 +148,7 @@ private void write(DataPartitionInfo partitionInfo, ChunkData chunkData) throws writer.endChunkGroup(); } writer.startChunkGroup(chunkData.getDevice()); + dataPartition2LastDevice.put(partitionInfo, chunkData.getDevice()); } chunkData.writeToFileWriter(writer); } @@ -179,7 +183,10 @@ private TsFileResource generateResource(TsFileIOWriter writer) throws IOExceptio private void close() throws IOException { for (Map.Entry entry : dataPartition2Writer.entrySet()) { - entry.getValue().close(); + TsFileIOWriter writer = entry.getValue(); + if (writer.canWrite()) { + entry.getValue().close(); + } } if (taskDir.delete()) { logger.warn(String.format("Can not delete load uuid dir %s.", taskDir.getPath())); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java index 141f91f9d3c19..5f0e9f084794a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java @@ -207,10 +207,25 @@ private void decodePage( new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null); BatchData batchData = pageReader.getAllSatisfiedPageData(); - ReadWriteIOUtils.write(batchData.length(), stream); // TODO: check if correct + int length = 0; while (batchData.hasCurrent()) { long time = batchData.currentTime(); if (time < timePartitionSlot.getStartTime()) { + batchData.next(); + continue; + } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time))) { + break; + } + length += 1; + batchData.next(); + } + + ReadWriteIOUtils.write(length, stream); + batchData.resetBatchData(); + while (batchData.hasCurrent()) { + long time = batchData.currentTime(); + if (time < timePartitionSlot.getStartTime()) { + batchData.next(); continue; } else if (!timePartitionSlot.equals(StorageEngineV2.getTimePartitionSlot(time))) { break; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index 6169901c294df..58089eb1a6143 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -2626,11 +2626,11 @@ private void tryToDeleteLastCacheByDevice(PartialPath deviceId) { if (!IoTDBDescriptor.getInstance().getConfig().isLastCacheEnabled()) { return; } - try { - IoTDB.schemaProcessor.deleteLastCacheByDevice(deviceId); - } catch (MetadataException e) { - // the path doesn't cache in cluster mode now, ignore - } + // try { + // IoTDB.schemaProcessor.deleteLastCacheByDevice(deviceId); + // } catch (MetadataException e) { + // // the path doesn't cache in cluster mode now, ignore + // } } /** diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 1720792d8767b..e23f02adc51e5 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -190,6 +190,8 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc .add((AlignedChunkData) chunkData); } chunkData.setNotDecode(chunkMetadata); + chunkDataList.add(chunkData); + reader.position(reader.position() + header.getDataSize()); break; } if (isAligned) { @@ -279,7 +281,7 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc case MetaMarker.VALUE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: chunkOffset = reader.position(); - chunkMetadata = offset2ChunkMetadata.get(chunkOffset); + chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES); header = reader.readChunkHeader(marker); if (header.getDataSize() == 0) { handleEmptyValueChunk(chunkOffset, header, chunkMetadata, pageIndex2ChunkData); @@ -287,9 +289,9 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc } Set allChunkData = new HashSet<>(); - chunkMetadata = offset2ChunkMetadata.get(chunkOffset); if (!isTimeChunkNeedDecode) { pageIndex2ChunkData.get(1).get(0).addValueChunk(chunkOffset, header, chunkMetadata); + reader.position(reader.position() + header.getDataSize()); break; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 6b0c76b5ef3c1..cb3ae522889d8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -126,6 +126,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.LOAD_TSFILE.serialize(stream); ReadWriteIOUtils.write(tsFile.getPath(), stream); // TODO: can save this space + ReadWriteIOUtils.write(chunkDataList.size(), stream); for (ChunkData chunkData : chunkDataList) { try { chunkData.serialize(stream, tsFile); @@ -148,7 +149,8 @@ public static PlanNode deserialize(ByteBuffer buffer) { ReadWriteIOUtils.readShort(stream); // read PlanNodeType File tsFile = new File(ReadWriteIOUtils.readString(stream)); LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); - while (stream.available() > 0) { + int chunkDataSize = ReadWriteIOUtils.readInt(stream); + for (int i = 0; i < chunkDataSize; i++) { ChunkData chunkData = ChunkData.deserialize(stream); pieceNode.addChunkData(chunkData); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java index 05b32e81f40fc..24a441870221c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java @@ -121,7 +121,8 @@ public ChunkHeader( /** the exact serialized size of chunk header */ public static int getSerializedSize(String measurementID, int dataSize) { - int measurementIdLength = measurementID.getBytes(TSFileConfig.STRING_CHARSET).length; + int measurementIdLength = + measurementID == null ? 0 : measurementID.getBytes(TSFileConfig.STRING_CHARSET).length; return Byte.BYTES // chunkType + ReadWriteForEncodingUtils.varIntSize(measurementIdLength) // measurementID length + measurementIdLength // measurementID From c5b69f3bce2290ecf96f204e015ce0cc5cd1daa3 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 26 Aug 2022 10:57:31 +0800 Subject: [PATCH 16/37] merge master --- .../org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 2 +- .../db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java | 5 +++-- .../db/mpp/plan/scheduler/load/LoadTsFileScheduler.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 5b2c4be286b24..17ce429d9ca8c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -95,8 +95,8 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ShowSchemaTemplateStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.ShowVersionStatement; -import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.db.mpp.plan.statement.sys.sync.ShowPipeSinkTypeStatement; +import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.filter.GroupByFilter; import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index de437e48121d9..e8a49a1fccb73 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -116,7 +116,7 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) internalServiceClientManager.borrowClient(endPoint)) { TLoadTsFileReq loadTsFileReq = new TLoadTsFileReq( - instance.getFragment().getRoot().serializeToByteBuffer(), + instance.getFragment().getPlanNodeTree().serializeToByteBuffer(), uuid, instance.getRegionReplicaSet().getRegionId()); TLoadResp loadResp = client.sendLoadNode(loadTsFileReq); @@ -141,7 +141,8 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD instance.getRegionReplicaSet().getRegionId()); LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) - PlanNodeType.deserialize(instance.getFragment().getRoot().serializeToByteBuffer()); + PlanNodeType.deserialize( + instance.getFragment().getPlanNodeTree().serializeToByteBuffer()); if (pieceNode == null) { throw new FragmentInstanceDispatchException( new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 1ff2a6c50f0f6..3cb08e8d4cf60 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -77,7 +77,7 @@ public LoadTsFileScheduler( this.allReplicaSets = new HashSet<>(); for (FragmentInstance fragmentInstance : distributedQueryPlan.getInstances()) { - tsFileNodeList.add((LoadSingleTsFileNode) fragmentInstance.getFragment().getRoot()); + tsFileNodeList.add((LoadSingleTsFileNode) fragmentInstance.getFragment().getPlanNodeTree()); } } From 265f39016a6087a35647ac8752982de7254fcd71 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 29 Aug 2022 09:32:42 +0800 Subject: [PATCH 17/37] reset usage for debug --- .../org/apache/iotdb/confignode/conf/ConfigNodeConfig.java | 2 +- server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../main/java/org/apache/iotdb/db/engine/StorageEngine.java | 3 ++- .../main/java/org/apache/iotdb/db/engine/StorageEngineV2.java | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index eb300d8711135..faef338857a0a 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -105,7 +105,7 @@ public class ConfigNodeConfig { IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME; /** Time partition interval in seconds */ - private long timePartitionInterval = 10; + private long timePartitionInterval = 86400; /** Default number of SchemaRegion replicas */ private int schemaReplicationFactor = 1; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 448e208ee17b3..c8e8204e3f4a8 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -751,7 +751,7 @@ public class IoTDBConfig { * Time range for partitioning data inside each storage group, the unit is second. Default time is * a day. */ - private long partitionInterval = 10; + private long partitionInterval = 86400; /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */ private long maxPlanNodeSize = 500 * 1048576L; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index 252c41c01d7f6..60e0fecefa5bf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -146,7 +146,8 @@ public static StorageEngine getInstance() { private static void initTimePartition() { timePartitionInterval = - convertMilliWithPrecision(IoTDBDescriptor.getInstance().getConfig().getPartitionInterval()); + convertMilliWithPrecision( + IoTDBDescriptor.getInstance().getConfig().getPartitionInterval() * 1000L); } public static long convertMilliWithPrecision(long milliTime) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java index e1b7b90c5bb9f..9358c688c71b1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java @@ -146,7 +146,8 @@ public static StorageEngineV2 getInstance() { private static void initTimePartition() { timePartitionInterval = - convertMilliWithPrecision(IoTDBDescriptor.getInstance().getConfig().getPartitionInterval()); + convertMilliWithPrecision( + IoTDBDescriptor.getInstance().getConfig().getPartitionInterval() * 1000L); } public static long convertMilliWithPrecision(long milliTime) { From c9d2e958590cb65c3490c22b17d33a2d21692279 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 29 Aug 2022 18:54:59 +0800 Subject: [PATCH 18/37] finish IT --- integration-test/import-control.xml | 7 + .../apache/iotdb/itbase/env/BaseConfig.java | 4 + .../apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 412 ++++++++++++++++++ .../iotdb/db/it/env/StandaloneEnvConfig.java | 5 + 4 files changed, 428 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java diff --git a/integration-test/import-control.xml b/integration-test/import-control.xml index 4a13fc9078019..b0c7d372eecfa 100644 --- a/integration-test/import-control.xml +++ b/integration-test/import-control.xml @@ -27,6 +27,7 @@ + @@ -43,4 +44,10 @@ + + + + + + diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseConfig.java index 5cf5f09d6c472..f3c2490abddbf 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseConfig.java @@ -70,6 +70,10 @@ default BaseConfig setDataRegionNum(int dataRegionNum) { return this; } + default boolean isEnablePartition() { + return true; + } + default BaseConfig setPartitionInterval(long partitionInterval) { return this; } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java new file mode 100644 index 0000000000000..3334155273158 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.it.env.ConfigFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.write.TsFileWriter; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IOTDBLoadTsFileIT { + private static final Logger LOGGER = LoggerFactory.getLogger(IOTDBLoadTsFileIT.class); + private static final long PARTITION_INTERVAL = 10L; + + private boolean originEnablePartition; + private long originPartitionInterval; + + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load").toUri()); + originEnablePartition = ConfigFactory.getConfig().isEnablePartition(); + originPartitionInterval = ConfigFactory.getConfig().getPartitionInterval(); + ConfigFactory.getConfig().setPartitionInterval(PARTITION_INTERVAL); + EnvFactory.getEnv().initBeforeTest(); + + registerSchema(); + } + + @After + public void tearDown() throws Exception { + deleteSG(); + + EnvFactory.getEnv().cleanAfterTest(); + ConfigFactory.getConfig().setEnablePartition(originEnablePartition); + ConfigFactory.getConfig().setPartitionInterval(originPartitionInterval); + + if (!deleteDir()) { + LOGGER.error("Can not delete tmp dir for loading tsfile."); + } + } + + private void registerSchema() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute("set storage group to " + SchemaConfig.STORAGE_GROUP_0); + statement.execute("set storage group to " + SchemaConfig.STORAGE_GROUP_1); + + statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00)); + statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_01)); + statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_02)); + statement.execute(convert2SQL(SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_03)); + + statement.execute( + convert2AlignedSQL( + SchemaConfig.DEVICE_1, + Arrays.asList( + SchemaConfig.MEASUREMENT_10, + SchemaConfig.MEASUREMENT_11, + SchemaConfig.MEASUREMENT_12, + SchemaConfig.MEASUREMENT_13))); + + statement.execute(convert2SQL(SchemaConfig.DEVICE_2, SchemaConfig.MEASUREMENT_20)); + + statement.execute(convert2SQL(SchemaConfig.DEVICE_3, SchemaConfig.MEASUREMENT_30)); + + statement.execute( + convert2AlignedSQL(SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40))); + } + } + + private String convert2SQL(String device, MeasurementSchema schema) { + String sql = + String.format( + "create timeseries %s %s", + new Path(device, schema.getMeasurementId()).getFullPath(), schema.getType().name()); + LOGGER.info(String.format("schema execute: %s.", sql)); + return sql; + } + + private String convert2AlignedSQL(String device, List schemas) { + String sql = String.format("create aligned timeseries %s(", device); + for (int i = 0; i < schemas.size(); i++) { + MeasurementSchema schema = schemas.get(i); + sql += (String.format("%s %s", schema.getMeasurementId(), schema.getType().name())); + sql += (i == schemas.size() - 1 ? ")" : ","); + } + LOGGER.info(String.format("schema execute: %s.", sql)); + return sql; + } + + private void deleteSG() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute(String.format("delete storage group %s", SchemaConfig.STORAGE_GROUP_0)); + statement.execute(String.format("delete storage group %s", SchemaConfig.STORAGE_GROUP_1)); + } + } + + private boolean deleteDir() { + for (File file : tmpDir.listFiles()) { + if (!file.delete()) { + return false; + } + } + return tmpDir.delete(); + } + + @Test + public void testLoad() throws Exception { + long writtenPoint1 = 0; + // device 0, device 1, sg 0 + try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_0), + Arrays.asList( + SchemaConfig.MEASUREMENT_00, + SchemaConfig.MEASUREMENT_01, + SchemaConfig.MEASUREMENT_02, + SchemaConfig.MEASUREMENT_03)); + generator.registerAlignedTimeseries( + new Path(SchemaConfig.DEVICE_1), + Arrays.asList( + SchemaConfig.MEASUREMENT_10, + SchemaConfig.MEASUREMENT_11, + SchemaConfig.MEASUREMENT_12, + SchemaConfig.MEASUREMENT_13)); + generator.generateData(new Path(SchemaConfig.DEVICE_0), 100000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_1), 100000, true); + writtenPoint1 = generator.getTotalNumber(); + } + + long writtenPoint2 = 0; + // device 2, device 3, device4, sg 1 + try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_2), Arrays.asList(SchemaConfig.MEASUREMENT_20)); + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_3), Arrays.asList(SchemaConfig.MEASUREMENT_30)); + generator.registerAlignedTimeseries( + new Path(SchemaConfig.DEVICE_4), Arrays.asList(SchemaConfig.MEASUREMENT_40)); + generator.generateData(new Path(SchemaConfig.DEVICE_2), 10000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_3), 10000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_4), 10000, true); + writtenPoint2 = generator.getTotalNumber(); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath())); + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from root.** group by level=1,2")) { + if (resultSet.next()) { + long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + Assert.assertEquals(writtenPoint1, sg1Count); + long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); + Assert.assertEquals(writtenPoint2, sg2Count); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + private static class SchemaConfig { + private static final String STORAGE_GROUP_0 = "root.sg.test_0"; + private static final String STORAGE_GROUP_1 = "root.sg.test_1"; + + // device 0, nonaligned, sg 0 + private static final String DEVICE_0 = "root.sg.test_0.d_0"; + private static final MeasurementSchema MEASUREMENT_00 = + new MeasurementSchema("sensor_00", TSDataType.INT32); + private static final MeasurementSchema MEASUREMENT_01 = + new MeasurementSchema("sensor_01", TSDataType.INT64); + private static final MeasurementSchema MEASUREMENT_02 = + new MeasurementSchema("sensor_02", TSDataType.DOUBLE); + private static final MeasurementSchema MEASUREMENT_03 = + new MeasurementSchema("sensor_03", TSDataType.TEXT); + + // device 1, aligned, sg 0 + private static final String DEVICE_1 = "root.sg.test_0.a_1"; + private static final MeasurementSchema MEASUREMENT_10 = + new MeasurementSchema("sensor_10", TSDataType.INT32); + private static final MeasurementSchema MEASUREMENT_11 = + new MeasurementSchema("sensor_11", TSDataType.INT64); + private static final MeasurementSchema MEASUREMENT_12 = + new MeasurementSchema("sensor_12", TSDataType.DOUBLE); + private static final MeasurementSchema MEASUREMENT_13 = + new MeasurementSchema("sensor_13", TSDataType.TEXT); + + // device 2, non aligned, sg 1 + private static final String DEVICE_2 = "root.sg.test_1.d_2"; + private static final MeasurementSchema MEASUREMENT_20 = + new MeasurementSchema("sensor_20", TSDataType.INT32); + + // device 3, non aligned, sg 1 + private static final String DEVICE_3 = "root.sg.test_1.d_3"; + private static final MeasurementSchema MEASUREMENT_30 = + new MeasurementSchema("sensor_30", TSDataType.INT32); + + // device 4, aligned, sg 1 + private static final String DEVICE_4 = "root.sg.test_1.a_4"; + private static final MeasurementSchema MEASUREMENT_40 = + new MeasurementSchema("sensor_40", TSDataType.INT32); + } + + public class TsFileGenerator implements AutoCloseable { + private final File tsFile; + private final TsFileWriter writer; + private final Map device2Number; + private final Map> device2MeasurementSchema; + private Random random; + private long totalNumber; + + public TsFileGenerator(File tsFile) throws IOException { + this.tsFile = tsFile; + this.writer = new TsFileWriter(tsFile); + this.device2Number = new HashMap<>(); + this.device2MeasurementSchema = new HashMap<>(); + this.random = new Random(); + this.totalNumber = 0; + } + + public void resetRandom() { + random = new Random(); + } + + public void resetRandom(long seed) { + random = new Random(seed); + } + + public void registerTimeseries(Path path, List measurementSchemaList) { + if (device2Number.containsKey(path)) { + LOGGER.error(String.format("Register same device %s.", path)); + return; + } + writer.registerTimeseries(path, measurementSchemaList); + device2Number.put(path, 0); + device2MeasurementSchema.put(path, measurementSchemaList); + } + + public void registerAlignedTimeseries(Path path, List measurementSchemaList) + throws WriteProcessException { + if (device2Number.containsKey(path)) { + LOGGER.error(String.format("Register same device %s.", path)); + return; + } + writer.registerAlignedTimeseries(path, measurementSchemaList); + device2Number.put(path, 0); + device2MeasurementSchema.put(path, measurementSchemaList); + } + + public void generateData(Path path, int number, boolean isAligned) + throws IOException, WriteProcessException { + List schemas = device2MeasurementSchema.get(path); + Tablet tablet = new Tablet(path.getFullPath(), schemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + long sensorNum = schemas.size(); + long startTime = device2Number.get(path); + + for (long r = 0; r < number; r++) { + int row = tablet.rowSize++; + timestamps[row] = startTime++; + for (int i = 0; i < sensorNum; i++) { + generateDataPoint(values[i], row, schemas.get(i)); + totalNumber += 1; + } + // write + if (tablet.rowSize == tablet.getMaxRowNumber()) { + if (!isAligned) { + writer.write(tablet); + } else { + writer.writeAligned(tablet); + } + tablet.reset(); + } + } + // write + if (tablet.rowSize != 0) { + if (!isAligned) { + writer.write(tablet); + } else { + writer.writeAligned(tablet); + } + tablet.reset(); + } + device2Number.compute(path, (k, v) -> v + number); + } + + private void generateDataPoint(Object obj, int row, MeasurementSchema schema) { + switch (schema.getType()) { + case INT32: + generateINT32(obj, row); + break; + case INT64: + generateINT64(obj, row); + break; + case FLOAT: + generateFLOAT(obj, row); + break; + case DOUBLE: + generateDOUBLE(obj, row); + break; + case BOOLEAN: + generateBOOLEAN(obj, row); + break; + case TEXT: + generateTEXT(obj, row); + break; + default: + LOGGER.error(String.format("Wrong data type %s.", schema.getType())); + } + } + + private void generateINT32(Object obj, int row) { + int[] ints = (int[]) obj; + ints[row] = random.nextInt(); + } + + private void generateINT64(Object obj, int row) { + long[] longs = (long[]) obj; + longs[row] = random.nextLong(); + } + + private void generateFLOAT(Object obj, int row) { + float[] floats = (float[]) obj; + floats[row] = random.nextFloat(); + } + + private void generateDOUBLE(Object obj, int row) { + double[] doubles = (double[]) obj; + doubles[row] = random.nextDouble(); + } + + private void generateBOOLEAN(Object obj, int row) { + boolean[] booleans = (boolean[]) obj; + booleans[row] = random.nextBoolean(); + } + + private void generateTEXT(Object obj, int row) { + Binary[] binaries = (Binary[]) obj; + binaries[row] = new Binary(String.format("test point %d", random.nextInt())); + } + + public int getNumber(Path path) { + return device2Number.get(path); + } + + public long getTotalNumber() { + return totalNumber; + } + + @Override + public void close() throws Exception { + writer.close(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnvConfig.java b/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnvConfig.java index 30d21973a55e7..fb2b655d8cc4b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnvConfig.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/env/StandaloneEnvConfig.java @@ -60,6 +60,11 @@ public BaseConfig setDataRegionNum(int dataRegionNum) { return this; } + @Override + public boolean isEnablePartition() { + return IoTDBDescriptor.getInstance().getConfig().isEnablePartition(); + } + @Override public BaseConfig setPartitionInterval(long partitionInterval) { IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(partitionInterval); From ccc8bc92e5bea84973dc6c1b8e6427952133c77a Mon Sep 17 00:00:00 2001 From: yschengzi Date: Tue, 30 Aug 2022 10:18:14 +0800 Subject: [PATCH 19/37] reset TsFileSequenceRead --- .../main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index d049bfe3d5e24..aa946f67b7d5e 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "D:\\Pro\\IoTDB\\data\\load\\load\\1661433500850-1-0-0.tsfile"; + String filename = "test.tsfile"; if (args.length >= 1) { filename = args[0]; } From b9ba16b92383c733b6b51af6279453f60a011891 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 31 Aug 2022 09:34:22 +0800 Subject: [PATCH 20/37] rename method and set log --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 3 ++- .../apache/iotdb/db/engine/load/LoadTsFileManager.java | 4 ++-- .../plan/scheduler/load/LoadTsFileDispatcherImpl.java | 10 +++++----- .../thrift/impl/DataNodeInternalRPCServiceImpl.java | 4 ++-- thrift/src/main/thrift/datanode.thrift | 4 ++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 510661caa3c34..9ea03ab4edcf1 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -245,7 +245,8 @@ public class IoTDBConfig { + File.separator + IoTDBConstant.SCHEMA_FOLDER_NAME; - private String loadTsFileDir = systemDir + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME; + private String loadTsFileDir = + IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME; /** Performance tracing directory, stores performance tracing files */ private String tracingDir = diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 7147f5212c8e8..89252280861ba 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -69,7 +69,7 @@ private void clearDir(File dir) { if (dir.delete()) { logger.info(String.format("Delete origin load TsFile dir %s.", dir.getPath())); } - if (dir.mkdirs()) { + if (!dir.mkdirs()) { logger.warn(String.format("load TsFile dir %s can not be created.", dir.getPath())); } } @@ -188,7 +188,7 @@ private void close() throws IOException { entry.getValue().close(); } } - if (taskDir.delete()) { + if (!taskDir.delete()) { logger.warn(String.format("Can not delete load uuid dir %s.", taskDir.getPath())); } dataPartition2Writer = null; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index e8a49a1fccb73..d379cebceac67 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -37,7 +37,7 @@ import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; -import org.apache.iotdb.mpp.rpc.thrift.TLoadTsFileReq; +import org.apache.iotdb.mpp.rpc.thrift.TTsFilePieceReq; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -101,7 +101,7 @@ private void dispatchOneInstance(FragmentInstance instance) if (isDispatchedToLocal(endPoint)) { dispatchLocally(instance); } else { - dispatchRemote(instance, endPoint); + dispatchRemote(instance, endPoint); // TODO: can read only once } } } @@ -114,12 +114,12 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) throws FragmentInstanceDispatchException { try (SyncDataNodeInternalServiceClient client = internalServiceClientManager.borrowClient(endPoint)) { - TLoadTsFileReq loadTsFileReq = - new TLoadTsFileReq( + TTsFilePieceReq loadTsFileReq = + new TTsFilePieceReq( instance.getFragment().getPlanNodeTree().serializeToByteBuffer(), uuid, instance.getRegionReplicaSet().getRegionId()); - TLoadResp loadResp = client.sendLoadNode(loadTsFileReq); + TLoadResp loadResp = client.sendTsFilePieceNode(loadTsFileReq); if (!loadResp.isAccepted()) { logger.error(loadResp.message); throw new FragmentInstanceDispatchException(loadResp.status); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index e981bdc4b1b69..24edcf4edba08 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -97,7 +97,6 @@ import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; -import org.apache.iotdb.mpp.rpc.thrift.TLoadTsFileReq; import org.apache.iotdb.mpp.rpc.thrift.TMigrateRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -107,6 +106,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TSendFragmentInstanceResp; import org.apache.iotdb.mpp.rpc.thrift.TSendPlanNodeReq; import org.apache.iotdb.mpp.rpc.thrift.TSendPlanNodeResp; +import org.apache.iotdb.mpp.rpc.thrift.TTsFilePieceReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateConfigNodeGroupReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTemplateReq; import org.apache.iotdb.rpc.RpcUtils; @@ -303,7 +303,7 @@ public TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) { } @Override - public TLoadResp sendLoadNode(TLoadTsFileReq req) throws TException { + public TLoadResp sendTsFilePieceNode(TTsFilePieceReq req) throws TException { LOGGER.info(String.format("Receive load node from uuid %s.", req.uuid)); ConsensusGroupId groupId = diff --git a/thrift/src/main/thrift/datanode.thrift b/thrift/src/main/thrift/datanode.thrift index 9403aa1ef5f8b..b6db7e28f5903 100644 --- a/thrift/src/main/thrift/datanode.thrift +++ b/thrift/src/main/thrift/datanode.thrift @@ -205,7 +205,7 @@ struct TUpdateTemplateReq{ 2: required binary templateInfo } -struct TLoadTsFileReq{ +struct TTsFilePieceReq{ 1: required binary body 2: required string uuid 3: required common.TConsensusGroupId consensusGroupId @@ -246,7 +246,7 @@ service IDataNodeRPCService { TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) - TLoadResp sendLoadNode(TLoadTsFileReq req); + TLoadResp sendTsFilePieceNode(TTsFilePieceReq req); TLoadResp sendLoadCommand(TLoadCommandReq req); From 2771583eccf8f1e41bfce46ed7d1f669dafec22c Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 1 Sep 2022 16:30:16 +0800 Subject: [PATCH 21/37] finsih auto create schema --- .../confignode/conf/ConfigNodeConfig.java | 2 +- .../iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../commons/partition/DataPartition.java | 8 ++ .../org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +-- .../db/mpp/plan/analyze/SchemaValidator.java | 13 ++ .../plan/analyze/StandaloneSchemaFetcher.java | 2 +- .../plan/node/load/LoadSingleTsFileNode.java | 133 ++++++++++++++++-- .../plan/node/load/LoadTsFileNode.java | 7 +- .../load/LoadTsFileDispatcherImpl.java | 45 ++++-- .../scheduler/load/LoadTsFileScheduler.java | 30 ++++ 10 files changed, 220 insertions(+), 38 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index faef338857a0a..42cc8998b26de 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -105,7 +105,7 @@ public class ConfigNodeConfig { IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME; /** Time partition interval in seconds */ - private long timePartitionInterval = 86400; + private long timePartitionInterval = 10000000; /** Default number of SchemaRegion replicas */ private int schemaReplicationFactor = 1; diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index aa946f67b7d5e..7066eed716690 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "test.tsfile"; + String filename = "D:\\Pro\\IoTDB\\data\\load\\big test\\1-0-0-0.tsfile"; if (args.length >= 1) { filename = args[0]; } diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java index 35e3a98a6f2ea..5008e07cc600f 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java @@ -97,6 +97,14 @@ public List getDataRegionReplicaSetForWriting( .collect(Collectors.toList()); } + public List getAllDataRegionReplicaSetForOneDevice(String deviceName) { + String storageGroup = getStorageGroupByDevice(deviceName); + TSeriesPartitionSlot seriesPartitionSlot = calculateDeviceGroupId(deviceName); + return dataPartitionMap.get(storageGroup).get(seriesPartitionSlot).entrySet().stream() + .flatMap(entry -> entry.getValue().stream()) + .collect(Collectors.toList()); + } + public TRegionReplicaSet getDataRegionReplicaSetForWriting( String deviceName, TTimePartitionSlot timePartitionSlot) { // A list of data region replica sets will store data in a same time partition. diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9ea03ab4edcf1..c7acabdc3c0ba 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -245,9 +245,6 @@ public class IoTDBConfig { + File.separator + IoTDBConstant.SCHEMA_FOLDER_NAME; - private String loadTsFileDir = - IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME; - /** Performance tracing directory, stores performance tracing files */ private String tracingDir = IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.TRACING_FOLDER_NAME; @@ -294,6 +291,9 @@ public class IoTDBConfig { IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.DATA_FOLDER_NAME }; + private String loadTsFileDir = + dataDirs[0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME; + /** Strategy of multiple directories. */ private String multiDirStrategyClassName = null; @@ -391,13 +391,13 @@ public class IoTDBConfig { private int avgSeriesPointNumberThreshold = 100000; /** Enable inner space compaction for sequence files */ - private boolean enableSeqSpaceCompaction = true; + private boolean enableSeqSpaceCompaction = false; /** Enable inner space compaction for unsequence files */ - private boolean enableUnseqSpaceCompaction = true; + private boolean enableUnseqSpaceCompaction = false; /** Compact the unsequence files into the overlapped sequence files */ - private boolean enableCrossSpaceCompaction = true; + private boolean enableCrossSpaceCompaction = false; /** * The strategy of inner space compaction task. There are just one inner space compaction strategy @@ -747,13 +747,13 @@ public class IoTDBConfig { private int primitiveArraySize = 32; /** whether enable data partition. If disabled, all data belongs to partition 0 */ - private boolean enablePartition = false; + private boolean enablePartition = true; /** * Time range for partitioning data inside each storage group, the unit is second. Default time is * a day. */ - private long partitionInterval = 86400; + private long partitionInterval = 10000000; /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */ private long maxPlanNodeSize = 500 * 1048576L; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java index e14fbf1563605..bb34ce306f38c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java @@ -19,11 +19,15 @@ package org.apache.iotdb.db.mpp.plan.analyze; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.BatchInsertNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; + +import java.util.List; public class SchemaValidator { @@ -58,4 +62,13 @@ public static ISchemaTree validate(InsertNode insertNode) { return schemaTree; } + + public static ISchemaTree validate( + List devicePaths, + List measurements, + List dataTypes, + List isAlignedList) { + return SCHEMA_FETCHER.fetchSchemaListWithAutoCreate( + devicePaths, measurements, dataTypes, isAlignedList); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java index 4e9cc1cc67af0..924a7517d22b1 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java @@ -125,7 +125,7 @@ public ISchemaTree fetchSchemaListWithAutoCreate( int totalSize = 0; boolean isAligned = isAlignedList.get(entry.getValue().get(0)); for (int index : entry.getValue()) { - if (isAlignedList.get(entry.getValue().get(index)) != isAligned) { + if (isAlignedList.get(index) != isAligned) { throw new StatementAnalyzeException( String.format("Inconsistent device alignment of %s in insert plan.", entry.getKey())); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index e23f02adc51e5..fe732c59198c5 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -19,16 +19,24 @@ package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.load.AlignedChunkData; import org.apache.iotdb.db.engine.load.ChunkData; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; +import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -66,31 +74,120 @@ public class LoadSingleTsFileNode extends WritePlanNode { private static final Logger logger = LoggerFactory.getLogger(LoadSingleTsFileNode.class); private File tsFile; + private boolean needDecodeTsFile; + private Map> replicaSet2Pieces; + private TsFileResource resource; + private TRegionReplicaSet localRegionReplicaSet; + public LoadSingleTsFileNode(PlanNodeId id) { super(id); } - public LoadSingleTsFileNode(PlanNodeId id, File tsFile) { + public LoadSingleTsFileNode(PlanNodeId id, File tsFile) throws IOException { super(id); this.tsFile = tsFile; + this.resource = new TsFileResource(tsFile); + + FileLoaderUtils.loadOrGenerateResource(resource); } - public Map> getReplicaSet2Pieces() { - return replicaSet2Pieces; + public void checkIfNeedDecodeTsFile(DataPartition dataPartition) { + Set allRegionReplicaSet = new HashSet<>(); + needDecodeTsFile = false; + for (String device : resource.getDevices()) { + if (!StorageEngineV2.getTimePartitionSlot(resource.getStartTime(device)) + .equals(StorageEngineV2.getTimePartitionSlot(resource.getEndTime(device)))) { + needDecodeTsFile = true; + } + allRegionReplicaSet.addAll(dataPartition.getAllDataRegionReplicaSetForOneDevice(device)); + } + needDecodeTsFile = !isDispatchedToLocal(allRegionReplicaSet); } - private LoadTsFilePieceNode getPieceNode( - String device, TTimePartitionSlot timePartitionSlot, DataPartition dataPartition) { - TRegionReplicaSet replicaSet = - dataPartition.getDataRegionReplicaSetForWriting(device, timePartitionSlot); - List pieceNodes = - replicaSet2Pieces.computeIfAbsent(replicaSet, o -> new ArrayList<>()); - if (pieceNodes.isEmpty() || pieceNodes.get(pieceNodes.size() - 1).exceedSize()) { - pieceNodes.add(new LoadTsFilePieceNode(getPlanNodeId(), tsFile)); + private boolean isDispatchedToLocal(Set replicaSets) { + if (replicaSets.size() > 1) { + return false; } - return pieceNodes.get(pieceNodes.size() - 1); + for (TRegionReplicaSet replicaSet : replicaSets) { + List dataNodeLocationList = replicaSet.getDataNodeLocations(); + if (dataNodeLocationList.size() > 1) { + return false; + } + localRegionReplicaSet = replicaSet; + return isDispatchedToLocal(dataNodeLocationList.get(0).getInternalEndPoint()); + } + return true; + } + + private boolean isDispatchedToLocal(TEndPoint endPoint) { + return IoTDBDescriptor.getInstance().getConfig().getInternalAddress().equals(endPoint.getIp()) + && IoTDBDescriptor.getInstance().getConfig().getInternalPort() == endPoint.port; + } + + public void autoRegisterSchema() + throws IOException, IllegalPathException { // TODO: only support sg level=1 + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + List deviceList = new ArrayList<>(); + List measurementList = new ArrayList<>(); + List dataTypeList = new ArrayList<>(); + List isAlignedList = new ArrayList<>(); + + Map> device2Metadata = reader.getAllTimeseriesMetadata(true); + for (Map.Entry> entry : device2Metadata.entrySet()) { + deviceList.add(new PartialPath(entry.getKey())); + + List timeseriesMetadataList = entry.getValue(); + boolean isAligned = + timeseriesMetadataList.stream() + .mapToInt(o -> o.getTSDataType().equals(TSDataType.VECTOR) ? 1 : 0) + .sum() + != 0; + int measurementSize = timeseriesMetadataList.size() - (isAligned ? 1 : 0); + String[] measurements = new String[measurementSize]; + TSDataType[] tsDataTypes = new TSDataType[measurementSize]; + + int index = 0; + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + TSDataType dataType = timeseriesMetadata.getTSDataType(); + if (!dataType.equals(TSDataType.VECTOR)) { + measurements[index] = timeseriesMetadata.getMeasurementId(); + tsDataTypes[index++] = dataType; + } + } + measurementList.add(measurements); + dataTypeList.add(tsDataTypes); + isAlignedList.add(isAligned); + } + + logger.info(String.format("deviceList: %s", deviceList)); + logger.info(String.format("measurementList: %s", measurementList)); + logger.info(String.format("dataTypeList: %s", dataTypeList)); + logger.info(String.format("isAlignedList: %s", isAlignedList)); + SchemaValidator.validate(deviceList, measurementList, dataTypeList, isAlignedList); + } + } + + public boolean needDecodeTsFile() { + return needDecodeTsFile; + } + + /** + * only used for load locally. + * + * @return local TRegionReplicaSet + */ + public TRegionReplicaSet getLocalRegionReplicaSet() { + return localRegionReplicaSet; + } + + public TsFileResource getTsFileResource() { + return resource; + } + + public Map> getReplicaSet2Pieces() { + return replicaSet2Pieces; } @Override @@ -425,4 +522,16 @@ private void handleEmptyValueChunk( } } } + + private LoadTsFilePieceNode getPieceNode( + String device, TTimePartitionSlot timePartitionSlot, DataPartition dataPartition) { + TRegionReplicaSet replicaSet = + dataPartition.getDataRegionReplicaSetForWriting(device, timePartitionSlot); + List pieceNodes = + replicaSet2Pieces.computeIfAbsent(replicaSet, o -> new ArrayList<>()); + if (pieceNodes.isEmpty() || pieceNodes.get(pieceNodes.size() - 1).exceedSize()) { + pieceNodes.add(new LoadTsFilePieceNode(getPlanNodeId(), tsFile)); + } + return pieceNodes.get(pieceNodes.size() - 1); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index a6878ba921a3b..c991a9c4768b8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -90,7 +90,12 @@ public List splitByPartition(Analysis analysis) { for (File file : tsFiles) { try { LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), file); - singleTsFileNode.splitTsFileByDataPartition(analysis.getDataPartitionInfo()); + singleTsFileNode.checkIfNeedDecodeTsFile(analysis.getDataPartitionInfo()); + singleTsFileNode.autoRegisterSchema(); + + if (singleTsFileNode.needDecodeTsFile()) { + singleTsFileNode.splitTsFileByDataPartition(analysis.getDataPartitionInfo()); + } res.add(singleTsFileNode); } catch (Exception e) { logger.error(String.format("Parse TsFile %s error", file.getPath()), e); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index d379cebceac67..f3f5b249f5252 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -29,9 +29,12 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadSingleTsFileNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.mpp.plan.scheduler.FragInstanceDispatchResult; import org.apache.iotdb.db.mpp.plan.scheduler.IFragInstanceDispatcher; @@ -133,26 +136,40 @@ private void dispatchRemote(FragmentInstance instance, TEndPoint endPoint) } } - private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDispatchException { + public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDispatchException { logger.info(String.format("Receive load node from uuid %s.", uuid)); ConsensusGroupId groupId = ConsensusGroupId.Factory.createFromTConsensusGroupId( instance.getRegionReplicaSet().getRegionId()); - LoadTsFilePieceNode pieceNode = - (LoadTsFilePieceNode) - PlanNodeType.deserialize( - instance.getFragment().getPlanNodeTree().serializeToByteBuffer()); - if (pieceNode == null) { - throw new FragmentInstanceDispatchException( - new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); - } - - TSStatus resultStatus = - StorageEngineV2.getInstance().writeLoadTsFileNode((DataRegionId) groupId, pieceNode, uuid); + PlanNode planNode = instance.getFragment().getPlanNodeTree(); + + if (planNode instanceof LoadTsFilePieceNode) { + LoadTsFilePieceNode pieceNode = + (LoadTsFilePieceNode) PlanNodeType.deserialize(planNode.serializeToByteBuffer()); + if (pieceNode == null) { + throw new FragmentInstanceDispatchException( + new TSStatus(TSStatusCode.NODE_DESERIALIZE_ERROR.getStatusCode())); + } + TSStatus resultStatus = + StorageEngineV2.getInstance() + .writeLoadTsFileNode((DataRegionId) groupId, pieceNode, uuid); - if (!RpcUtils.SUCCESS_STATUS.equals(resultStatus)) { - throw new FragmentInstanceDispatchException(resultStatus); + if (!RpcUtils.SUCCESS_STATUS.equals(resultStatus)) { + throw new FragmentInstanceDispatchException(resultStatus); + } + } else if (planNode instanceof LoadSingleTsFileNode) { + try { + StorageEngineV2.getInstance() + .getDataRegion((DataRegionId) groupId) + .loadNewTsFile(((LoadSingleTsFileNode) planNode).getTsFileResource(), true); + } catch (LoadFileException e) { + logger.error(String.format("Load TsFile Node %s error.", planNode), e); + TSStatus resultStatus = new TSStatus(); + resultStatus.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + resultStatus.setMessage(e.getMessage()); + throw new FragmentInstanceDispatchException(resultStatus); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 3cb08e8d4cf60..1e9f8d33f6594 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.common.FragmentInstanceId; import org.apache.iotdb.db.mpp.common.MPPQueryContext; import org.apache.iotdb.db.mpp.common.PlanFragmentId; @@ -85,6 +86,14 @@ public LoadTsFileScheduler( public void start() { stateMachine.transitionToRunning(); for (LoadSingleTsFileNode node : tsFileNodeList) { + if (!node.needDecodeTsFile()) { + boolean isLoadLocallySuccess = loadLocally(node); + if (!isLoadLocallySuccess) { + return; + } + continue; + } + String uuid = UUID.randomUUID().toString(); dispatcher.setUuid(uuid); allReplicaSets.clear(); @@ -189,6 +198,27 @@ private boolean secondPhase(boolean isFirstPhaseSuccess, String uuid) { return true; } + private boolean loadLocally(LoadSingleTsFileNode node) { + try { + FragmentInstance instance = + new FragmentInstance( + new PlanFragment(fragmentId, node), + fragmentId.genFragmentInstanceId(), + null, + queryContext.getQueryType(), + queryContext.getTimeOut()); + instance.setDataRegionAndHost(node.getLocalRegionReplicaSet()); + dispatcher.dispatchLocally(instance); + } catch (FragmentInstanceDispatchException e) { + logger.error("Dispatch LoadCommand error to local error."); + logger.error(String.format("Result status code %s.", e.getFailureStatus().getCode())); + logger.error(String.format("Result status message %s.", e.getFailureStatus().getMessage())); + stateMachine.transitionToFailed(e.getFailureStatus()); + return false; + } + return true; + } + @Override public void stop() {} From 8ed9e704efdbd94bfa931ad7de7f534f9a27baba Mon Sep 17 00:00:00 2001 From: yschengzi Date: Sun, 4 Sep 2022 20:52:33 +0800 Subject: [PATCH 22/37] finish debug --- .../confignode/conf/ConfigNodeConfig.java | 2 +- .../iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../iotdb/it/env/AbstractNodeWrapper.java | 4 +++- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++----- .../db/engine/load/AlignedChunkData.java | 21 ++++++++++++------- .../plan/node/load/LoadSingleTsFileNode.java | 9 +++----- .../load/LoadTsFileDispatcherImpl.java | 4 ++-- .../scheduler/load/LoadTsFileScheduler.java | 2 +- .../thrift/impl/ClientRPCServiceImpl.java | 1 + .../db/service/thrift/impl/TSServiceImpl.java | 5 ++++- 10 files changed, 34 insertions(+), 26 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 42cc8998b26de..faef338857a0a 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -105,7 +105,7 @@ public class ConfigNodeConfig { IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME; /** Time partition interval in seconds */ - private long timePartitionInterval = 10000000; + private long timePartitionInterval = 86400; /** Default number of SchemaRegion replicas */ private int schemaReplicationFactor = 1; diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index 7066eed716690..aa946f67b7d5e 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -53,7 +53,7 @@ public class TsFileSequenceRead { "squid:S106" }) // Suppress high Cognitive Complexity and Standard outputs warning public static void main(String[] args) throws IOException { - String filename = "D:\\Pro\\IoTDB\\data\\load\\big test\\1-0-0-0.tsfile"; + String filename = "test.tsfile"; if (args.length >= 1) { filename = args[0]; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java index a0ebb1a757170..a55777c7d511d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/AbstractNodeWrapper.java @@ -174,7 +174,9 @@ public void start() { "-Xms200m", "-Xmx200m", "-XX:MaxDirectMemorySize=200m", - "-Djdk.nio.maxCachedBufferSize=262144")); + "-Djdk.nio.maxCachedBufferSize=262144", + "-cp", + templateNodeLibPath)); addStartCmdParams(startCmd); FileUtils.write( stdoutFile, String.join(" ", startCmd) + "\n\n", StandardCharsets.UTF_8, true); diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c7acabdc3c0ba..2c5250deb782d 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -391,13 +391,13 @@ public class IoTDBConfig { private int avgSeriesPointNumberThreshold = 100000; /** Enable inner space compaction for sequence files */ - private boolean enableSeqSpaceCompaction = false; + private boolean enableSeqSpaceCompaction = true; /** Enable inner space compaction for unsequence files */ - private boolean enableUnseqSpaceCompaction = false; + private boolean enableUnseqSpaceCompaction = true; /** Compact the unsequence files into the overlapped sequence files */ - private boolean enableCrossSpaceCompaction = false; + private boolean enableCrossSpaceCompaction = true; /** * The strategy of inner space compaction task. There are just one inner space compaction strategy @@ -747,13 +747,13 @@ public class IoTDBConfig { private int primitiveArraySize = 32; /** whether enable data partition. If disabled, all data belongs to partition 0 */ - private boolean enablePartition = true; + private boolean enablePartition = false; /** * Time range for partitioning data inside each storage group, the unit is second. Default time is * a day. */ - private long partitionInterval = 10000000; + private long partitionInterval = 86400; /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */ private long maxPlanNodeSize = 500 * 1048576L; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java index fbefa3fa7bab0..7a2a3eb77683f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java @@ -72,7 +72,7 @@ public class AlignedChunkData implements ChunkData { private List chunkMetadataList; private List timeBatch; - private int satisfiedLength; + private List satisfiedTimeBatchLength; private AlignedChunkWriterImpl chunkWriter; private List chunkList; @@ -184,6 +184,7 @@ private void serializeAttr(DataOutputStream stream) throws IOException { private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IOException { timeBatch = new ArrayList<>(); + satisfiedTimeBatchLength = new ArrayList<>(); ReadWriteIOUtils.write(needDecodeChunk(), stream); try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { int chunkSize = offset.size(); @@ -196,6 +197,7 @@ private void serializeTsFileData(DataOutputStream stream, File tsFile) throws IO } } timeBatch = null; + satisfiedTimeBatchLength = null; } private void serializeEntireChunk( @@ -223,7 +225,7 @@ private void serializeDecodeChunk( Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); reader.position(offset.get(chunkIndex)); - int pageIndex = 0; + int decodePageIndex = 0; // should be 0,1 or 2 long dataSize = this.dataSize.get(chunkIndex); while (dataSize > 0) { boolean hasStatistic = (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER; @@ -235,15 +237,15 @@ private void serializeDecodeChunk( if (chunkIndex == 0) { decodeTimePage(reader, chunkHeader, pageHeader, defaultTimeDecoder, valueDecoder, stream); } else { - decodeValuePage(reader, chunkHeader, pageHeader, pageIndex, valueDecoder, stream); + decodeValuePage(reader, chunkHeader, pageHeader, decodePageIndex, valueDecoder, stream); } + decodePageIndex += 1; } else { // entire page ReadWriteIOUtils.write(false, stream); // don't decode pageHeader.serializeTo(stream); ByteBuffer pageData = reader.readCompressedPage(pageHeader); ReadWriteIOUtils.write(pageData, stream); } - pageIndex += 1; dataSize -= pageDataSize; } @@ -263,7 +265,7 @@ private void decodeTimePage( ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); TimePageReader timePageReader = new TimePageReader(pageHeader, pageData, timeDecoder); long[] decodeTime = timePageReader.getNextTimeBatch(); - satisfiedLength = 0; + int satisfiedLength = 0; long[] time = new long[decodeTime.length]; for (int i = 0; i < decodeTime.length; i++) { if (decodeTime[i] < timePartitionSlot.getStartTime()) { @@ -273,11 +275,12 @@ private void decodeTimePage( } time[satisfiedLength++] = decodeTime[i]; } - timeBatch.add(decodeTime); ReadWriteIOUtils.write(satisfiedLength, stream); for (int i = 0; i < satisfiedLength; i++) { ReadWriteIOUtils.write(time[i], stream); } + timeBatch.add(decodeTime); + satisfiedTimeBatchLength.add(satisfiedLength); } private void decodeValuePage( @@ -293,8 +296,10 @@ private void decodeValuePage( ValuePageReader valuePageReader = new ValuePageReader(pageHeader, pageData, chunkHeader.getDataType(), valueDecoder); long[] time = timeBatch.get(pageIndex); - TsPrimitiveType[] valueBatch = valuePageReader.nextValueBatch(time); - ReadWriteIOUtils.write(satisfiedLength, stream); + TsPrimitiveType[] valueBatch = + valuePageReader.nextValueBatch( + time); // should be origin time, so recording satisfied length is necessary + ReadWriteIOUtils.write(satisfiedTimeBatchLength.get(pageIndex), stream); for (int i = 0; i < valueBatch.length; i++) { if (time[i] < timePartitionSlot.getStartTime()) { continue; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index fe732c59198c5..de773bc3702af 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -100,6 +100,7 @@ public void checkIfNeedDecodeTsFile(DataPartition dataPartition) { if (!StorageEngineV2.getTimePartitionSlot(resource.getStartTime(device)) .equals(StorageEngineV2.getTimePartitionSlot(resource.getEndTime(device)))) { needDecodeTsFile = true; + return; } allRegionReplicaSet.addAll(dataPartition.getAllDataRegionReplicaSetForOneDevice(device)); } @@ -161,10 +162,6 @@ public void autoRegisterSchema() isAlignedList.add(isAligned); } - logger.info(String.format("deviceList: %s", deviceList)); - logger.info(String.format("measurementList: %s", measurementList)); - logger.info(String.format("dataTypeList: %s", dataTypeList)); - logger.info(String.format("isAlignedList: %s", isAlignedList)); SchemaValidator.validate(deviceList, measurementList, dataTypeList, isAlignedList); } } @@ -234,8 +231,8 @@ public String toString() { return "LoadSingleTsFileNode{" + "tsFile=" + tsFile - + ", replicaSets=" - + replicaSet2Pieces.keySet() + + ", needDecodeTsFile=" + + needDecodeTsFile + '}'; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java index f3f5b249f5252..69733b2443cb6 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -144,7 +144,7 @@ public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDi instance.getRegionReplicaSet().getRegionId()); PlanNode planNode = instance.getFragment().getPlanNodeTree(); - if (planNode instanceof LoadTsFilePieceNode) { + if (planNode instanceof LoadTsFilePieceNode) { // split LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) PlanNodeType.deserialize(planNode.serializeToByteBuffer()); if (pieceNode == null) { @@ -158,7 +158,7 @@ public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDi if (!RpcUtils.SUCCESS_STATUS.equals(resultStatus)) { throw new FragmentInstanceDispatchException(resultStatus); } - } else if (planNode instanceof LoadSingleTsFileNode) { + } else if (planNode instanceof LoadSingleTsFileNode) { // do not need split try { StorageEngineV2.getInstance() .getDataRegion((DataRegionId) groupId) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 1e9f8d33f6594..6cd9df2d17aed 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -143,7 +143,7 @@ private boolean dispatchOneTsFile(LoadSingleTsFileNode node) { TSStatusCode.representOf(result.getFailureStatus().getCode()).name())); logger.error( String.format("Result status message %s.", result.getFailureStatus().getMessage())); - if (result.getFailureStatus().getSubStatus().size() > 0) { + if (result.getFailureStatus().getSubStatus() != null) { for (TSStatus status : result.getFailureStatus().getSubStatus()) { logger.error( String.format( diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java index 21fed2010e007..23a86b85dece5 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java @@ -598,6 +598,7 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { if (costTime >= CONFIG.getSlowQueryThreshold()) { SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement); } + LOGGER.info("result sql {} cost {} ms", statement, costTime); } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java index 0654444a799e2..6fccab1cd8ce3 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java @@ -607,12 +607,12 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { @Override public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { String statement = req.getStatement(); + long startTime = System.currentTimeMillis(); try { if (!SESSION_MANAGER.checkLogin(req.getSessionId())) { return RpcUtils.getTSExecuteStatementResp(getNotLoggedInStatus()); } - long startTime = System.currentTimeMillis(); PhysicalPlan physicalPlan = serviceProvider .getPlanner() @@ -640,6 +640,9 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { } catch (Exception e) { return RpcUtils.getTSExecuteStatementResp( onQueryException(e, "\"" + statement + "\". " + OperationType.EXECUTE_STATEMENT)); + } finally { + LOGGER.info( + "result sql {}, cost time {} ms.", statement, System.currentTimeMillis() - startTime); } } From 010f0c693ac73062e2ec699b1b2bdb2a83d67c06 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 7 Sep 2022 10:53:02 +0800 Subject: [PATCH 23/37] finish test --- .../plan/planner/plan/node/load/LoadSingleTsFileNode.java | 2 +- .../iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java | 1 - .../apache/iotdb/db/service/thrift/impl/TSServiceImpl.java | 5 +---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index de773bc3702af..91519fe649769 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -336,7 +336,7 @@ public void splitTsFileByDataPartition(DataPartition dataPartition) throws IOExc boolean isFirstData = true; for (long currentTime : timeBatch) { TTimePartitionSlot currentTimePartitionSlot = - StorageEngineV2.getTimePartitionSlot(currentTime); + StorageEngineV2.getTimePartitionSlot(currentTime); // TODO: can speed up if (!timePartitionSlot.equals(currentTimePartitionSlot)) { if (!isFirstData) { chunkData.setTailPageNeedDecode(true); // close last chunk data diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java index 23a86b85dece5..21fed2010e007 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java @@ -598,7 +598,6 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { if (costTime >= CONFIG.getSlowQueryThreshold()) { SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement); } - LOGGER.info("result sql {} cost {} ms", statement, costTime); } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java index 6fccab1cd8ce3..0654444a799e2 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java @@ -607,12 +607,12 @@ public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) { @Override public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { String statement = req.getStatement(); - long startTime = System.currentTimeMillis(); try { if (!SESSION_MANAGER.checkLogin(req.getSessionId())) { return RpcUtils.getTSExecuteStatementResp(getNotLoggedInStatus()); } + long startTime = System.currentTimeMillis(); PhysicalPlan physicalPlan = serviceProvider .getPlanner() @@ -640,9 +640,6 @@ public TSExecuteStatementResp executeStatement(TSExecuteStatementReq req) { } catch (Exception e) { return RpcUtils.getTSExecuteStatementResp( onQueryException(e, "\"" + statement + "\". " + OperationType.EXECUTE_STATEMENT)); - } finally { - LOGGER.info( - "result sql {}, cost time {} ms.", statement, System.currentTimeMillis() - startTime); } } From 0c111b78cd965c2e738bb991a4eb2bdeffd2832d Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 9 Sep 2022 15:33:00 +0800 Subject: [PATCH 24/37] finish device and measurements --- .../schemaregion/rocksdb/RSchemaRegion.java | 2 + .../metadata/schemaregion/ISchemaRegion.java | 4 + .../schemaregion/SchemaRegionMemoryImpl.java | 38 +++++- .../SchemaRegionSchemaFileImpl.java | 2 + .../plan/analyze/ClusterSchemaFetcher.java | 48 ++++++-- .../plan/analyze/FakeSchemaFetcherImpl.java | 13 +++ .../db/mpp/plan/analyze/ISchemaFetcher.java | 10 ++ .../db/mpp/plan/analyze/SchemaValidator.java | 6 +- .../plan/analyze/StandaloneSchemaFetcher.java | 47 +++++++- .../plan/node/load/LoadSingleTsFileNode.java | 108 +++++++++++++++++- .../plan/node/load/LoadTsFileNode.java | 6 +- .../statement/crud/LoadTsFileStatement.java | 8 ++ 12 files changed, 272 insertions(+), 20 deletions(-) diff --git a/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java index 5bde0ccce4b90..b396279d97883 100644 --- a/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java +++ b/schema-engine-rocksdb/src/main/java/org/apache/iotdb/db/metadata/schemaregion/rocksdb/RSchemaRegion.java @@ -1926,6 +1926,8 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/ISchemaRegion.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/ISchemaRegion.java index 4ff65c3117208..1f55881886a85 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/ISchemaRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/ISchemaRegion.java @@ -43,7 +43,9 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.dataset.ShowDevicesResult; import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; import java.io.File; @@ -362,6 +364,8 @@ DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException; // endregion diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java index 31208132e3cc7..ab3f7961983ad 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java @@ -1770,6 +1770,33 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) Collections.emptyMap()); } + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + int measurementsSize = measurements.size(); + for (int i = 0; i < measurementsSize; i++) { + if (encodings.get(i) == null) { + encodings.set(i, getDefaultEncoding(dataTypes.get(i))); + } + if (compressors.get(i) == null) { + compressors.set(i, TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + /** create aligned timeseries ignoring PathAlreadyExistException */ private void internalAlignedCreateTimeseries( PartialPath prefixPath, List measurements, List dataTypes) @@ -1788,6 +1815,8 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException { try { @@ -1802,11 +1831,16 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( internalAlignedCreateTimeseries( devicePath, Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i))); + Collections.singletonList(getDataType.apply(i)), + Collections.singletonList(encodings[i]), + Collections.singletonList(compressionTypes[i])); } else { internalCreateTimeseries( - devicePath.concatNode(measurements[i]), getDataType.apply(i)); + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); } // after creating timeseries, the deviceMNode has been replaced by a new entityMNode deviceMNode = mtree.getNodeByPath(devicePath); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java index 68fa31a77e21a..eae3856a96c40 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java @@ -1645,6 +1645,8 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java index 77ff8e9abb21e..06b151df05474 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java @@ -207,6 +207,8 @@ public ISchemaTree fetchSchemaWithAutoCreate( indexOfMissingMeasurements, measurements, getDataType, + null, + null, isAligned); schemaTree.mergeSchemaTree(missingSchemaTree); @@ -221,6 +223,18 @@ public ISchemaTree fetchSchemaListWithAutoCreate( List measurementsList, List tsDataTypesList, List isAlignedList) { + return fetchSchemaListWithAutoCreate( + devicePathList, measurementsList, tsDataTypesList, null, null, isAlignedList); + } + + @Override + public ISchemaTree fetchSchemaListWithAutoCreate( + List devicePathList, + List measurementsList, + List tsDataTypesList, + List encodingsList, + List compressionTypesList, + List isAlignedList) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); PathPatternTree patternTree = new PathPatternTree(); @@ -259,6 +273,8 @@ public ISchemaTree fetchSchemaListWithAutoCreate( indexOfMissingMeasurementsList.get(i), measurementsList.get(i), index -> tsDataTypesList.get(finalI)[index], + encodingsList == null ? null : encodingsList.get(i), + compressionTypesList == null ? null : compressionTypesList.get(i), isAlignedList.get(i)); schemaTree.mergeSchemaTree(missingSchemaTree); schemaCache.put(missingSchemaTree); @@ -287,6 +303,8 @@ private ClusterSchemaTree checkAndAutoCreateMissingMeasurements( List indexOfMissingMeasurements, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean isAligned) { DeviceSchemaInfo deviceSchemaInfo = schemaTree.searchDeviceSchemaInfo( @@ -348,15 +366,31 @@ private ClusterSchemaTree checkAndAutoCreateMissingMeasurements( List missingMeasurements = new ArrayList<>(indexOfMissingMeasurements.size()); List dataTypesOfMissingMeasurement = new ArrayList<>(indexOfMissingMeasurements.size()); + List encodingsOfMissingMeasurement = + new ArrayList<>(indexOfMissingMeasurements.size()); + List compressionTypesOfMissingMeasurement = + new ArrayList<>(indexOfMissingMeasurements.size()); indexOfMissingMeasurements.forEach( index -> { + TSDataType tsDataType = getDataType.apply(index); missingMeasurements.add(measurements[index]); - dataTypesOfMissingMeasurement.add(getDataType.apply(index)); + dataTypesOfMissingMeasurement.add(tsDataType); + encodingsOfMissingMeasurement.add( + encodings == null ? getDefaultEncoding(tsDataType) : encodings[index]); + compressionTypesOfMissingMeasurement.add( + compressionTypes == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[index]); }); schemaTree.mergeSchemaTree( internalCreateTimeseries( - devicePath, missingMeasurements, dataTypesOfMissingMeasurement, isAligned)); + devicePath, + missingMeasurements, + dataTypesOfMissingMeasurement, + encodingsOfMissingMeasurement, + compressionTypesOfMissingMeasurement, + isAligned)); return schemaTree; } @@ -384,15 +418,9 @@ private ClusterSchemaTree internalCreateTimeseries( PartialPath devicePath, List measurements, List tsDataTypes, + List encodings, + List compressors, boolean isAligned) { - - List encodings = new ArrayList<>(); - List compressors = new ArrayList<>(); - for (TSDataType dataType : tsDataTypes) { - encodings.add(getDefaultEncoding(dataType)); - compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); - } - List measurementPathList = executeInternalCreateTimeseriesStatement( new InternalCreateTimeSeriesStatement( diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakeSchemaFetcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakeSchemaFetcherImpl.java index 7c620a43e3f12..bccca336521ab 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakeSchemaFetcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakeSchemaFetcherImpl.java @@ -28,7 +28,9 @@ import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -111,6 +113,17 @@ public ISchemaTree fetchSchemaListWithAutoCreate( return null; } + @Override + public ISchemaTree fetchSchemaListWithAutoCreate( + List devicePath, + List measurements, + List tsDataTypes, + List encodings, + List compressionTypes, + List aligned) { + return null; + } + @Override public Pair checkTemplateSetInfo(PartialPath path) { return null; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java index e175f5917466b..f17b6dc43b90e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java @@ -23,7 +23,9 @@ import org.apache.iotdb.db.metadata.template.Template; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; import java.util.List; @@ -49,6 +51,14 @@ ISchemaTree fetchSchemaListWithAutoCreate( List tsDataTypes, List aligned); + ISchemaTree fetchSchemaListWithAutoCreate( + List devicePath, + List measurements, + List tsDataTypes, + List encodings, + List compressionTypes, + List aligned); + Pair checkTemplateSetInfo(PartialPath path); Map checkAllRelatedTemplate(PartialPath pathPattern); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java index bb34ce306f38c..a3b41f570362b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java @@ -25,7 +25,9 @@ import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.BatchInsertNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import java.util.List; @@ -67,8 +69,10 @@ public static ISchemaTree validate( List devicePaths, List measurements, List dataTypes, + List encodings, + List compressionTypes, List isAlignedList) { return SCHEMA_FETCHER.fetchSchemaListWithAutoCreate( - devicePaths, measurements, dataTypes, isAlignedList); + devicePaths, measurements, dataTypes, encodings,compressionTypes, isAlignedList); } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java index 924a7517d22b1..1d0578b0f9fa9 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java @@ -33,7 +33,9 @@ import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; import java.util.ArrayList; @@ -87,7 +89,13 @@ public ISchemaTree fetchSchemaWithAutoCreate( Function getDataType, boolean aligned) { DeviceSchemaInfo deviceSchemaInfo = - getDeviceSchemaInfoWithAutoCreate(devicePath, measurements, getDataType, aligned); + getDeviceSchemaInfoWithAutoCreate( + devicePath, + measurements, + getDataType, + new TSEncoding[measurements.length], + new CompressionType[measurements.length], + aligned); DeviceGroupSchemaTree schemaTree = new DeviceGroupSchemaTree(); schemaTree.addDeviceInfo(deviceSchemaInfo); return schemaTree; @@ -97,12 +105,14 @@ private DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) { try { SchemaRegionId schemaRegionId = localConfigNode.getBelongedSchemaRegionId(devicePath); ISchemaRegion schemaRegion = schemaEngine.getSchemaRegion(schemaRegionId); return schemaRegion.getDeviceSchemaInfoWithAutoCreate( - devicePath, measurements, getDataType, aligned); + devicePath, measurements, getDataType, encodings, compressionTypes, aligned); } catch (MetadataException e) { throw new RuntimeException(e); } @@ -114,6 +124,18 @@ public ISchemaTree fetchSchemaListWithAutoCreate( List measurementsList, List tsDataTypesList, List isAlignedList) { + return fetchSchemaListWithAutoCreate( + devicePathList, measurementsList, tsDataTypesList, null, null, isAlignedList); + } + + @Override + public ISchemaTree fetchSchemaListWithAutoCreate( + List devicePathList, + List measurementsList, + List tsDataTypesList, + List encodingsList, + List compressionTypesList, + List isAlignedList) { Map> deviceMap = new HashMap<>(); for (int i = 0, size = devicePathList.size(); i < size; i++) { deviceMap.computeIfAbsent(devicePathList.get(i), k -> new ArrayList<>()).add(i); @@ -134,6 +156,8 @@ public ISchemaTree fetchSchemaListWithAutoCreate( String[] measurements = new String[totalSize]; TSDataType[] tsDataTypes = new TSDataType[totalSize]; + TSEncoding[] encodings = new TSEncoding[totalSize]; + CompressionType[] compressionTypes = new CompressionType[totalSize]; int curPos = 0; for (int index : entry.getValue()) { @@ -145,12 +169,29 @@ public ISchemaTree fetchSchemaListWithAutoCreate( measurementsList.get(index).length); System.arraycopy( tsDataTypesList.get(index), 0, tsDataTypes, curPos, tsDataTypesList.get(index).length); + if (encodingsList != null) { + System.arraycopy( + encodingsList.get(index), 0, encodings, curPos, encodingsList.get(index).length); + } + if (compressionTypesList != null) { + System.arraycopy( + compressionTypesList.get(index), + 0, + compressionTypes, + curPos, + compressionTypesList.get(index).length); + } curPos += measurementsList.get(index).length; } schemaTree.addDeviceInfo( getDeviceSchemaInfoWithAutoCreate( - entry.getKey(), measurements, index -> tsDataTypes[index], isAligned)); + entry.getKey(), + measurements, + index -> tsDataTypes[index], + encodings, + compressionTypes, + isAligned)); } return schemaTree; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 91519fe649769..4b0439bbdf56c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -31,6 +31,9 @@ import org.apache.iotdb.db.engine.load.AlignedChunkData; import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; @@ -47,8 +50,10 @@ import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -56,6 +61,7 @@ import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +70,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -127,12 +134,14 @@ private boolean isDispatchedToLocal(TEndPoint endPoint) { && IoTDBDescriptor.getInstance().getConfig().getInternalPort() == endPoint.port; } - public void autoRegisterSchema() + public void autoRegisterSchema(boolean needVerifySchema) throws IOException, IllegalPathException { // TODO: only support sg level=1 try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { List deviceList = new ArrayList<>(); List measurementList = new ArrayList<>(); List dataTypeList = new ArrayList<>(); + List encodingsList = new ArrayList<>(); + List compressionTypesList = new ArrayList<>(); List isAlignedList = new ArrayList<>(); Map> device2Metadata = reader.getAllTimeseriesMetadata(true); @@ -148,24 +157,117 @@ public void autoRegisterSchema() int measurementSize = timeseriesMetadataList.size() - (isAligned ? 1 : 0); String[] measurements = new String[measurementSize]; TSDataType[] tsDataTypes = new TSDataType[measurementSize]; + TSEncoding[] encodings = new TSEncoding[measurementSize]; + CompressionType[] compressionTypes = new CompressionType[measurementSize]; int index = 0; for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { TSDataType dataType = timeseriesMetadata.getTSDataType(); if (!dataType.equals(TSDataType.VECTOR)) { measurements[index] = timeseriesMetadata.getMeasurementId(); - tsDataTypes[index++] = dataType; + tsDataTypes[index] = dataType; + ChunkHeader chunkHeader = + getChunkHeaderByTimeseriesMetadata(reader, timeseriesMetadata); + encodings[index] = chunkHeader.getEncodingType(); + compressionTypes[index++] = chunkHeader.getCompressionType(); } } measurementList.add(measurements); dataTypeList.add(tsDataTypes); + encodingsList.add(encodings); + compressionTypesList.add(compressionTypes); isAlignedList.add(isAligned); } - SchemaValidator.validate(deviceList, measurementList, dataTypeList, isAlignedList); + ISchemaTree schemaTree = + SchemaValidator.validate( + deviceList, + measurementList, + dataTypeList, + encodingsList, + compressionTypesList, + isAlignedList); + if (needVerifySchema) { + verifySchema( + schemaTree, + deviceList, + measurementList, + dataTypeList, + encodingsList, + compressionTypesList, + isAlignedList); + } } } + private ChunkHeader getChunkHeaderByTimeseriesMetadata( + TsFileSequenceReader reader, TimeseriesMetadata timeseriesMetadata) throws IOException { + IChunkMetadata chunkMetadata = timeseriesMetadata.getChunkMetadataList().get(0); + reader.position(chunkMetadata.getOffsetOfChunkHeader()); + return reader.readChunkHeader(reader.readMarker()); + } + + private void verifySchema( + ISchemaTree schemaTree, + List deviceList, + List measurementsList, + List tsDataTypesList, + List encodingsList, + List compressionTypeList, + List isAlignedList) { + int deviceSize = deviceList.size(); + for (int i = 0; i < deviceSize; i++) { + DeviceSchemaInfo schemaInfo = + schemaTree.searchDeviceSchemaInfo( + deviceList.get(i), Arrays.asList(measurementsList.get(i))); + String device = deviceList.get(i).toString(); + if (schemaInfo.isAligned() != isAlignedList.get(i)) { + throw new SemanticException( + makeMismatchMessage( + device, + "Is aligned", + isAlignedList.get(i).toString(), + String.valueOf(schemaInfo.isAligned()))); + } + List measurementSchemaList = schemaInfo.getMeasurementSchemaList(); + int measurementSize = measurementSchemaList.size(); + for (int j = 0; j < measurementSize; j++) { + MeasurementSchema measurementSchema = measurementSchemaList.get(j); + if (tsDataTypesList.get(i)[j] != measurementSchema.getType()) { + throw new SemanticException( + makeMismatchMessage( + device + measurementsList.get(i)[j], + "Datatype", + tsDataTypesList.get(i)[j].name(), + measurementSchema.getType().name())); + } + if (!encodingsList.get(i)[j].equals(measurementSchema.getEncodingType())) { + throw new SemanticException( + makeMismatchMessage( + device + measurementsList.get(i)[j], + "Encoding", + encodingsList.get(i)[j].name(), + measurementSchema.getEncodingType().name())); + } + if (!compressionTypeList.get(i)[j].equals(measurementSchema.getCompressor())) { + throw new SemanticException( + makeMismatchMessage( + device + measurementsList.get(i)[j], + "Compress type", + compressionTypeList.get(i)[j].name(), + measurementSchema.getCompressor().name())); + } + } + } + } + + private String makeMismatchMessage( + String path, String compareInfo, String tsFileInfo, String IoTDBInfo) { + return String.format( + "%s %s mismatch, %s in tsfile %s, but %s in IoTDb.", + path, compareInfo, tsFileInfo, tsFile.getAbsoluteFile(), IoTDBInfo); + } + public boolean needDecodeTsFile() { return needDecodeTsFile; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index c991a9c4768b8..dab6a1cda4a79 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.slf4j.Logger; @@ -86,12 +87,15 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { @Override public List splitByPartition(Analysis analysis) { + LoadTsFileStatement statement = (LoadTsFileStatement) analysis.getStatement(); List res = new ArrayList<>(); for (File file : tsFiles) { try { LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), file); singleTsFileNode.checkIfNeedDecodeTsFile(analysis.getDataPartitionInfo()); - singleTsFileNode.autoRegisterSchema(); + if (statement.isAutoCreateSchema() || statement.isVerifySchema()) { + singleTsFileNode.autoRegisterSchema(statement.isVerifySchema()); + } if (singleTsFileNode.needDecodeTsFile()) { singleTsFileNode.splitTsFileByDataPartition(analysis.getDataPartitionInfo()); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java index 4fb39cad1a71b..6f7a30fa444d8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java @@ -98,6 +98,14 @@ public void setVerifySchema(boolean verifySchema) { this.verifySchema = verifySchema; } + public boolean isVerifySchema() { + return verifySchema; + } + + public boolean isAutoCreateSchema() { + return autoCreateSchema; + } + public List getTsFiles() { return tsFiles; } From 517fbd240284cc178edf33faf715faa24e737794 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 9 Sep 2022 17:22:16 +0800 Subject: [PATCH 25/37] finsih auto create --- .../db/exception/VerifyMetadataException.java | 38 +++++ .../db/mpp/plan/analyze/ISchemaFetcher.java | 12 +- .../db/mpp/plan/analyze/SchemaValidator.java | 2 +- .../plan/node/load/LoadSingleTsFileNode.java | 145 +++++++++++++----- .../plan/node/load/LoadTsFileNode.java | 6 +- .../statement/crud/LoadTsFileStatement.java | 4 + .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + 7 files changed, 161 insertions(+), 47 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java diff --git a/server/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java b/server/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java new file mode 100644 index 0000000000000..1603c9872f4cb --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception; + +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.rpc.TSStatusCode; + +public class VerifyMetadataException extends IoTDBException { + public VerifyMetadataException( + String path, String compareInfo, String tsFileInfo, String tsFilePath, String IoTDBInfo) { + super( + String.format( + "%s %s mismatch, %s in tsfile %s, but %s in IoTDB.", + path, compareInfo, tsFileInfo, tsFilePath, IoTDBInfo), + TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); + } + + public VerifyMetadataException(String message) { + super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java index f17b6dc43b90e..e6ac3a4befafe 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java @@ -52,12 +52,12 @@ ISchemaTree fetchSchemaListWithAutoCreate( List aligned); ISchemaTree fetchSchemaListWithAutoCreate( - List devicePath, - List measurements, - List tsDataTypes, - List encodings, - List compressionTypes, - List aligned); + List devicePath, + List measurements, + List tsDataTypes, + List encodings, + List compressionTypes, + List aligned); Pair checkTemplateSetInfo(PartialPath path); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java index a3b41f570362b..f40e82abb4f03 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/SchemaValidator.java @@ -73,6 +73,6 @@ public static ISchemaTree validate( List compressionTypes, List isAlignedList) { return SCHEMA_FETCHER.fetchSchemaListWithAutoCreate( - devicePaths, measurements, dataTypes, encodings,compressionTypes, isAlignedList); + devicePaths, measurements, dataTypes, encodings, compressionTypes, isAlignedList); } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 4b0439bbdf56c..872089d026f74 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -26,20 +26,33 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.load.AlignedChunkData; import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.exception.VerifyMetadataException; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; +import org.apache.iotdb.db.mpp.plan.Coordinator; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; +import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher; +import org.apache.iotdb.db.mpp.plan.analyze.ClusterSchemaFetcher; +import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher; +import org.apache.iotdb.db.mpp.plan.analyze.ISchemaFetcher; import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; +import org.apache.iotdb.db.mpp.plan.analyze.StandalonePartitionFetcher; +import org.apache.iotdb.db.mpp.plan.analyze.StandaloneSchemaFetcher; +import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.mpp.plan.statement.Statement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; +import org.apache.iotdb.db.query.control.SessionManager; import org.apache.iotdb.db.utils.FileLoaderUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -50,7 +63,6 @@ import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -60,8 +72,8 @@ import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; - import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,6 +91,21 @@ public class LoadSingleTsFileNode extends WritePlanNode { private static final Logger logger = LoggerFactory.getLogger(LoadSingleTsFileNode.class); + private static final Coordinator COORDINATOR = Coordinator.getInstance(); + private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); + private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + private final IPartitionFetcher PARTITION_FETCHER; + private final ISchemaFetcher SCHEMA_FETCHER; + + { + if (CONFIG.isClusterMode()) { + PARTITION_FETCHER = ClusterPartitionFetcher.getInstance(); + SCHEMA_FETCHER = ClusterSchemaFetcher.getInstance(); + } else { + PARTITION_FETCHER = StandalonePartitionFetcher.getInstance(); + SCHEMA_FETCHER = StandaloneSchemaFetcher.getInstance(); + } + } private File tsFile; private boolean needDecodeTsFile; @@ -134,8 +161,10 @@ private boolean isDispatchedToLocal(TEndPoint endPoint) { && IoTDBDescriptor.getInstance().getConfig().getInternalPort() == endPoint.port; } - public void autoRegisterSchema(boolean needVerifySchema) - throws IOException, IllegalPathException { // TODO: only support sg level=1 + public void autoRegisterSchema( + boolean needVerifySchema, int sgLevel, Set registeredDevices) + throws IOException, IllegalPathException, + VerifyMetadataException { // TODO: only support sg level=1 try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { List deviceList = new ArrayList<>(); List measurementList = new ArrayList<>(); @@ -145,9 +174,12 @@ public void autoRegisterSchema(boolean needVerifySchema) List isAlignedList = new ArrayList<>(); Map> device2Metadata = reader.getAllTimeseriesMetadata(true); - for (Map.Entry> entry : device2Metadata.entrySet()) { - deviceList.add(new PartialPath(entry.getKey())); + for (String device : device2Metadata.keySet()) { + deviceList.add(new PartialPath(device)); + } + autoCreateSg(deviceList, sgLevel, registeredDevices); + for (Map.Entry> entry : device2Metadata.entrySet()) { List timeseriesMetadataList = entry.getValue(); boolean isAligned = timeseriesMetadataList.stream() @@ -200,6 +232,47 @@ public void autoRegisterSchema(boolean needVerifySchema) } } + private void autoCreateSg( + List devicePathList, int sgLevel, Set registeredDevices) + throws VerifyMetadataException { + for (PartialPath device : devicePathList) { + if (!registeredDevices.contains(device.toString())) { + String[] nodes = device.getNodes(); + String[] sgNodes = new String[sgLevel]; + if (nodes.length < sgLevel) { + throw new VerifyMetadataException( + String.format("Sg level %d is longer than device %s.", sgLevel, device)); + } + for (int i = 0; i < sgLevel; i++) { + sgNodes[i] = nodes[i]; + } + + SetStorageGroupStatement statement = new SetStorageGroupStatement(); + statement.setStorageGroupPath(new PartialPath(sgNodes)); + executeSetStorageGroupStatement(statement); + + registeredDevices.add(device.toString()); + } + } + } + + private void executeSetStorageGroupStatement(Statement statement) { + long queryId = SESSION_MANAGER.requestQueryId(false); + ExecutionResult result = + COORDINATOR.execute( + statement, + queryId, + null, + "", + PARTITION_FETCHER, + SCHEMA_FETCHER, + CONFIG.getQueryTimeoutThreshold()); + if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + logger.warn(String.format("Set Storage group error, statement: %s.", statement)); + logger.warn(String.format("Set storage group result status : %s.", result.status)); + } + } + private ChunkHeader getChunkHeaderByTimeseriesMetadata( TsFileSequenceReader reader, TimeseriesMetadata timeseriesMetadata) throws IOException { IChunkMetadata chunkMetadata = timeseriesMetadata.getChunkMetadataList().get(0); @@ -214,7 +287,8 @@ private void verifySchema( List tsDataTypesList, List encodingsList, List compressionTypeList, - List isAlignedList) { + List isAlignedList) + throws VerifyMetadataException { int deviceSize = deviceList.size(); for (int i = 0; i < deviceSize; i++) { DeviceSchemaInfo schemaInfo = @@ -222,52 +296,45 @@ private void verifySchema( deviceList.get(i), Arrays.asList(measurementsList.get(i))); String device = deviceList.get(i).toString(); if (schemaInfo.isAligned() != isAlignedList.get(i)) { - throw new SemanticException( - makeMismatchMessage( - device, - "Is aligned", - isAlignedList.get(i).toString(), - String.valueOf(schemaInfo.isAligned()))); + throw new VerifyMetadataException( + device, + "Is aligned", + isAlignedList.get(i).toString(), + tsFile.getPath(), + String.valueOf(schemaInfo.isAligned())); } List measurementSchemaList = schemaInfo.getMeasurementSchemaList(); int measurementSize = measurementSchemaList.size(); for (int j = 0; j < measurementSize; j++) { MeasurementSchema measurementSchema = measurementSchemaList.get(j); if (tsDataTypesList.get(i)[j] != measurementSchema.getType()) { - throw new SemanticException( - makeMismatchMessage( - device + measurementsList.get(i)[j], - "Datatype", - tsDataTypesList.get(i)[j].name(), - measurementSchema.getType().name())); + throw new VerifyMetadataException( + device + measurementsList.get(i)[j], + "Datatype", + tsDataTypesList.get(i)[j].name(), + tsFile.getPath(), + measurementSchema.getType().name()); } if (!encodingsList.get(i)[j].equals(measurementSchema.getEncodingType())) { - throw new SemanticException( - makeMismatchMessage( - device + measurementsList.get(i)[j], - "Encoding", - encodingsList.get(i)[j].name(), - measurementSchema.getEncodingType().name())); + throw new VerifyMetadataException( + device + measurementsList.get(i)[j], + "Encoding", + encodingsList.get(i)[j].name(), + tsFile.getPath(), + measurementSchema.getEncodingType().name()); } if (!compressionTypeList.get(i)[j].equals(measurementSchema.getCompressor())) { - throw new SemanticException( - makeMismatchMessage( - device + measurementsList.get(i)[j], - "Compress type", - compressionTypeList.get(i)[j].name(), - measurementSchema.getCompressor().name())); + throw new VerifyMetadataException( + device + measurementsList.get(i)[j], + "Compress type", + compressionTypeList.get(i)[j].name(), + tsFile.getPath(), + measurementSchema.getCompressor().name()); } } } } - private String makeMismatchMessage( - String path, String compareInfo, String tsFileInfo, String IoTDBInfo) { - return String.format( - "%s %s mismatch, %s in tsfile %s, but %s in IoTDb.", - path, compareInfo, tsFileInfo, tsFile.getAbsoluteFile(), IoTDBInfo); - } - public boolean needDecodeTsFile() { return needDecodeTsFile; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index dab6a1cda4a79..9f15623531075 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -35,7 +35,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class LoadTsFileNode extends WritePlanNode { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileNode.class); @@ -89,12 +91,14 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { public List splitByPartition(Analysis analysis) { LoadTsFileStatement statement = (LoadTsFileStatement) analysis.getStatement(); List res = new ArrayList<>(); + Set registeredDevices = new HashSet<>(); for (File file : tsFiles) { try { LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), file); singleTsFileNode.checkIfNeedDecodeTsFile(analysis.getDataPartitionInfo()); if (statement.isAutoCreateSchema() || statement.isVerifySchema()) { - singleTsFileNode.autoRegisterSchema(statement.isVerifySchema()); + singleTsFileNode.autoRegisterSchema( + statement.isVerifySchema(), statement.getSgLevel(), registeredDevices); } if (singleTsFileNode.needDecodeTsFile()) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java index 6f7a30fa444d8..e1a496b68540d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java @@ -106,6 +106,10 @@ public boolean isAutoCreateSchema() { return autoCreateSchema; } + public int getSgLevel() { + return sgLevel; + } + public List getTsFiles() { return tsFiles; } diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index e0304251d5740..4795b9b94fe91 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -72,6 +72,7 @@ public enum TSStatusCode { CREATE_TEMPLATE_ERROR(340), SYNC_FILE_REBASE(341), SYNC_FILE_ERROR(342), + VERIFY_METADATA_ERROR(343), EXECUTE_STATEMENT_ERROR(400), SQL_PARSE_ERROR(401), From 67aeecfa233cac8c7874968739c07b60073174b9 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Tue, 13 Sep 2022 10:28:03 +0800 Subject: [PATCH 26/37] merge to master --- .../java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java | 5 +++++ .../service/thrift/impl/DataNodeInternalRPCServiceImpl.java | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index 522b303bb1927..cdbab30564408 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -2106,6 +2106,11 @@ private TimeRange parseTimeRange( } } + /** function for parsing file path used by LOAD statement. */ + public String parseFilePath(String src) { + return src.substring(1, src.length() - 1); + } + // Expression & Predicate ======================================================================== private Expression parseExpression( diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index 60ecab10de52a..2289965b9385c 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -104,7 +104,6 @@ import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; -import org.apache.iotdb.mpp.rpc.thrift.TMigrateRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; From 2edab4a2bc8b15d7f0a852981c5ee94c57146430 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 14 Sep 2022 15:21:42 +0800 Subject: [PATCH 27/37] finsih auto register --- .../db/mpp/plan/analyze/AnalyzeVisitor.java | 322 +++++++++++++++++- .../mpp/plan/planner/LogicalPlanVisitor.java | 2 +- .../plan/node/load/LoadSingleTsFileNode.java | 196 +---------- .../plan/node/load/LoadTsFileNode.java | 24 +- .../statement/crud/LoadTsFileStatement.java | 13 +- .../metadata/SetStorageGroupStatement.java | 16 + .../iotdb/db/utils/FileLoaderUtils.java | 13 +- 7 files changed, 355 insertions(+), 231 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index cba594a546c91..84f08773ce642 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -20,14 +20,19 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.LoadFileException; +import org.apache.iotdb.db.exception.VerifyMetadataException; import org.apache.iotdb.db.exception.metadata.template.TemplateImcompatibeException; import org.apache.iotdb.db.exception.sql.MeasurementNotExistException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -42,6 +47,8 @@ import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; +import org.apache.iotdb.db.mpp.plan.Coordinator; +import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult; import org.apache.iotdb.db.mpp.plan.expression.Expression; import org.apache.iotdb.db.mpp.plan.expression.ExpressionType; import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand; @@ -79,6 +86,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateMultiTimeSeriesStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildPathsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement; @@ -96,13 +104,22 @@ import org.apache.iotdb.db.mpp.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.mpp.plan.statement.sys.sync.ShowPipeSinkTypeStatement; +import org.apache.iotdb.db.query.control.SessionManager; import org.apache.iotdb.db.utils.FileLoaderUtils; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.filter.GroupByFilter; import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,6 +127,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -133,6 +151,9 @@ public class AnalyzeVisitor extends StatementVisitor private static final Logger logger = LoggerFactory.getLogger(Analyzer.class); + private final Coordinator coordinator = Coordinator.getInstance(); + private final SessionManager sessionManager = SessionManager.getInstance(); + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private final IPartitionFetcher partitionFetcher; private final ISchemaFetcher schemaFetcher; private final MPPQueryContext context; @@ -1350,29 +1371,49 @@ public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryC Map device2MinTime = new HashMap<>(); Map device2MaxTime = new HashMap<>(); + Map> device2Schemas = new HashMap<>(); + Map> device2IsAligned = new HashMap<>(); + + // analyze tsfile metadata for (File tsFile : loadTsFileStatement.getTsFiles()) { try { - TsFileResource resource = new TsFileResource(tsFile); - FileLoaderUtils.loadOrGenerateResource(resource); - for (String device : resource.getDevices()) { - device2MinTime.put( - device, - Math.min( - device2MinTime.getOrDefault(device, Long.MAX_VALUE), - resource.getStartTime(device))); - device2MaxTime.put( - device, - Math.max( - device2MaxTime.getOrDefault(device, Long.MIN_VALUE), - resource.getEndTime(device))); - } - } catch (IOException e) { + TsFileResource resource = + analyzeTsFile( + loadTsFileStatement, + tsFile, + device2MinTime, + device2MaxTime, + device2Schemas, + device2IsAligned); + loadTsFileStatement.addTsFileResource(resource); + } catch (Exception e) { logger.error(String.format("Parse file %s to resource error.", tsFile.getPath()), e); throw new SemanticException( String.format("Parse file %s to resource error", tsFile.getPath())); } } + // auto create and verify schema + if (loadTsFileStatement.isVerifySchema() || loadTsFileStatement.isAutoCreateSchema()) { + try { + if (loadTsFileStatement.isVerifySchema()) { + verifyLoadingMeasurements(device2Schemas); + } + autoCreateSg(loadTsFileStatement.getSgLevel(), device2Schemas); + ISchemaTree schemaTree = autoCreateSchema(device2Schemas, device2IsAligned); + if (loadTsFileStatement.isVerifySchema()) { + verifySchema(schemaTree, device2Schemas, device2IsAligned); + } + } catch (Exception e) { + logger.error("Auto create or verify schema error.", e); + throw new SemanticException( + String.format( + "Auto create or verify schema error when executing statement %s.", + loadTsFileStatement)); + } + } + + // construct partition info List params = new ArrayList<>(); for (Map.Entry entry : device2MinTime.entrySet()) { List timePartitionSlots = new ArrayList<>(); @@ -1399,6 +1440,257 @@ public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryC return analysis; } + private TsFileResource analyzeTsFile( + LoadTsFileStatement statement, + File tsFile, + Map device2MinTime, + Map device2MaxTime, + Map> device2Schemas, + Map> device2IsAligned) + throws IOException, VerifyMetadataException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + Map> device2Metadata = reader.getAllTimeseriesMetadata(true); + + if (statement.isAutoCreateSchema() || statement.isVerifySchema()) { + // construct schema + for (Map.Entry> entry : device2Metadata.entrySet()) { + String device = entry.getKey(); + List timeseriesMetadataList = entry.getValue(); + boolean isAligned = false; + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + TSDataType dataType = timeseriesMetadata.getTSDataType(); + if (!dataType.equals(TSDataType.VECTOR)) { + ChunkHeader chunkHeader = + getChunkHeaderByTimeseriesMetadata(reader, timeseriesMetadata); + MeasurementSchema measurementSchema = + new MeasurementSchema( + timeseriesMetadata.getMeasurementId(), + dataType, + chunkHeader.getEncodingType(), + chunkHeader.getCompressionType()); + device2Schemas + .computeIfAbsent(device, o -> new HashMap<>()) + .put(measurementSchema, tsFile); + } else { + isAligned = true; + } + } + boolean finalIsAligned = isAligned; + if (!device2IsAligned + .computeIfAbsent(device, o -> new Pair<>(finalIsAligned, tsFile)) + .left + .equals(isAligned)) { + throw new VerifyMetadataException( + String.format( + "Device %s has different aligned definition in tsFile %s and other TsFile.", + device, tsFile.getParentFile())); + } + } + } + + // construct TsFileResource + TsFileResource resource = new TsFileResource(tsFile); + FileLoaderUtils.updateTsFileResource(device2Metadata, resource); + resource.updatePlanIndexes(reader.getMinPlanIndex()); + resource.updatePlanIndexes(reader.getMaxPlanIndex()); + + // construct device time range + for (String device : resource.getDevices()) { + device2MinTime.put( + device, + Math.min( + device2MinTime.getOrDefault(device, Long.MAX_VALUE), + resource.getStartTime(device))); + device2MaxTime.put( + device, + Math.max( + device2MaxTime.getOrDefault(device, Long.MIN_VALUE), resource.getEndTime(device))); + } + + return resource; + } + } + + private ChunkHeader getChunkHeaderByTimeseriesMetadata( + TsFileSequenceReader reader, TimeseriesMetadata timeseriesMetadata) throws IOException { + IChunkMetadata chunkMetadata = timeseriesMetadata.getChunkMetadataList().get(0); + reader.position(chunkMetadata.getOffsetOfChunkHeader()); + return reader.readChunkHeader(reader.readMarker()); + } + + private void autoCreateSg(int sgLevel, Map> device2Schemas) + throws VerifyMetadataException, LoadFileException, IllegalPathException { + sgLevel += 1; // e.g. "root.sg" means sgLevel = 1, "root.sg.test" means sgLevel=2 + Set sgSet = new HashSet<>(); + for (String device : device2Schemas.keySet()) { + PartialPath devicePath = new PartialPath(device); + + String[] nodes = devicePath.getNodes(); + String[] sgNodes = new String[sgLevel]; + if (nodes.length < sgLevel) { + throw new VerifyMetadataException( + String.format("Sg level %d is longer than device %s.", sgLevel, device)); + } + for (int i = 0; i < sgLevel; i++) { + sgNodes[i] = nodes[i]; + } + PartialPath sgPath = new PartialPath(sgNodes); + sgSet.add(sgPath); + } + + for (PartialPath sgPath : sgSet) { + SetStorageGroupStatement statement = new SetStorageGroupStatement(); + statement.setStorageGroupPath(sgPath); + executeSetStorageGroupStatement(statement); + } + } + + private void executeSetStorageGroupStatement(Statement statement) throws LoadFileException { + long queryId = sessionManager.requestQueryId(false); + ExecutionResult result = + coordinator.execute( + statement, + queryId, + null, + "", + partitionFetcher, + schemaFetcher, + config.getQueryTimeoutThreshold()); + if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && result.status.code != TSStatusCode.STORAGE_GROUP_ALREADY_EXISTS.getStatusCode()) { + logger.error(String.format("Set Storage group error, statement: %s.", statement)); + logger.error(String.format("Set storage group result status : %s.", result.status)); + throw new LoadFileException( + String.format("Can not execute set storage group statement: %s", statement)); + } + } + + private ISchemaTree autoCreateSchema( + Map> device2Schemas, + Map> device2IsAligned) + throws IllegalPathException { + List deviceList = new ArrayList<>(); + List measurementList = new ArrayList<>(); + List dataTypeList = new ArrayList<>(); + List encodingsList = new ArrayList<>(); + List compressionTypesList = new ArrayList<>(); + List isAlignedList = new ArrayList<>(); + + for (Map.Entry> entry : device2Schemas.entrySet()) { + int measurementSize = entry.getValue().size(); + String[] measurements = new String[measurementSize]; + TSDataType[] tsDataTypes = new TSDataType[measurementSize]; + TSEncoding[] encodings = new TSEncoding[measurementSize]; + CompressionType[] compressionTypes = new CompressionType[measurementSize]; + + int index = 0; + for (MeasurementSchema measurementSchema : entry.getValue().keySet()) { + measurements[index] = measurementSchema.getMeasurementId(); + tsDataTypes[index] = measurementSchema.getType(); + encodings[index] = measurementSchema.getEncodingType(); + compressionTypes[index++] = measurementSchema.getCompressor(); + } + + deviceList.add(new PartialPath(entry.getKey())); + measurementList.add(measurements); + dataTypeList.add(tsDataTypes); + encodingsList.add(encodings); + compressionTypesList.add(compressionTypes); + isAlignedList.add(device2IsAligned.get(entry.getKey()).left); + } + + return SchemaValidator.validate( + deviceList, + measurementList, + dataTypeList, + encodingsList, + compressionTypesList, + isAlignedList); + } + + private void verifyLoadingMeasurements(Map> device2Schemas) + throws VerifyMetadataException { + for (Map.Entry> deviceEntry : device2Schemas.entrySet()) { + Map id2Schema = new HashMap<>(); + Map schema2TsFile = deviceEntry.getValue(); + for (Map.Entry entry : schema2TsFile.entrySet()) { + String measurementId = entry.getKey().getMeasurementId(); + if (!id2Schema.containsKey(measurementId)) { + id2Schema.put(measurementId, entry.getKey()); + } else { + MeasurementSchema conflictSchema = id2Schema.get(measurementId); + String msg = + String.format( + "Measurement %s Conflict, TsFile %s has measurement: %s, TsFile %s has measurement %s.", + deviceEntry.getKey() + measurementId, + entry.getValue().getPath(), + entry.getKey(), + schema2TsFile.get(conflictSchema).getPath(), + conflictSchema); + logger.error(msg); + throw new VerifyMetadataException(msg); + } + } + } + } + + private void verifySchema( + ISchemaTree schemaTree, + Map> device2Schemas, + Map> device2IsAligned) + throws VerifyMetadataException, IllegalPathException { + for (Map.Entry> entry : device2Schemas.entrySet()) { + String device = entry.getKey(); + MeasurementSchema[] tsFileSchemas = + entry.getValue().keySet().toArray(new MeasurementSchema[0]); + DeviceSchemaInfo schemaInfo = + schemaTree.searchDeviceSchemaInfo( + new PartialPath(device), + Arrays.stream(tsFileSchemas) + .map(MeasurementSchema::getMeasurementId) + .collect(Collectors.toList())); + if (schemaInfo.isAligned() != device2IsAligned.get(device).left) { + throw new VerifyMetadataException( + device, + "Is aligned", + device2IsAligned.get(device).left.toString(), + device2IsAligned.get(device).right.getPath(), + String.valueOf(schemaInfo.isAligned())); + } + List originSchemaList = schemaInfo.getMeasurementSchemaList(); + int measurementSize = originSchemaList.size(); + for (int j = 0; j < measurementSize; j++) { + MeasurementSchema originSchema = originSchemaList.get(j); + MeasurementSchema tsFileSchema = tsFileSchemas[j]; + String measurementPath = device + originSchema.getMeasurementId(); + if (!tsFileSchema.getType().equals(originSchema.getType())) { + throw new VerifyMetadataException( + measurementPath, + "Datatype", + tsFileSchema.getType().name(), + entry.getValue().get(tsFileSchema).getPath(), + originSchema.getType().name()); + } + if (!tsFileSchema.getEncodingType().equals(originSchema.getEncodingType())) { + throw new VerifyMetadataException( + measurementPath, + "Encoding", + tsFileSchema.getEncodingType().name(), + entry.getValue().get(tsFileSchema).getPath(), + originSchema.getEncodingType().name()); + } + if (!tsFileSchema.getCompressor().equals(originSchema.getCompressor())) { + throw new VerifyMetadataException( + measurementPath, + "Compress type", + tsFileSchema.getCompressor().name(), + entry.getValue().get(tsFileSchema).getPath(), + originSchema.getCompressor().name()); + } + } + } + } + @Override public Analysis visitShowTimeSeries( ShowTimeSeriesStatement showTimeSeriesStatement, MPPQueryContext context) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java index be44c14eee6f4..4cf5e55c0f18d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java @@ -481,7 +481,7 @@ public PlanNode visitInsertRow(InsertRowStatement insertRowStatement, MPPQueryCo @Override public PlanNode visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { return new LoadTsFileNode( - context.getQueryId().genPlanNodeId(), loadTsFileStatement.getTsFiles()); + context.getQueryId().genPlanNodeId(), loadTsFileStatement.getResources()); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 872089d026f74..bed28ef76589d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -23,36 +23,25 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.partition.DataPartition; -import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.load.AlignedChunkData; import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.Coordinator; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.mpp.plan.analyze.ClusterSchemaFetcher; import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.mpp.plan.analyze.ISchemaFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.analyze.StandalonePartitionFetcher; import org.apache.iotdb.db.mpp.plan.analyze.StandaloneSchemaFetcher; -import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.mpp.plan.statement.Statement; -import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; import org.apache.iotdb.db.query.control.SessionManager; -import org.apache.iotdb.db.utils.FileLoaderUtils; -import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -65,14 +54,12 @@ import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import org.apache.iotdb.tsfile.read.reader.page.TimePageReader; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,7 +69,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -119,12 +105,10 @@ public LoadSingleTsFileNode(PlanNodeId id) { super(id); } - public LoadSingleTsFileNode(PlanNodeId id, File tsFile) throws IOException { + public LoadSingleTsFileNode(PlanNodeId id, TsFileResource resource) throws IOException { super(id); - this.tsFile = tsFile; - this.resource = new TsFileResource(tsFile); - - FileLoaderUtils.loadOrGenerateResource(resource); + this.tsFile = resource.getTsFile(); + this.resource = resource; } public void checkIfNeedDecodeTsFile(DataPartition dataPartition) { @@ -161,180 +145,6 @@ private boolean isDispatchedToLocal(TEndPoint endPoint) { && IoTDBDescriptor.getInstance().getConfig().getInternalPort() == endPoint.port; } - public void autoRegisterSchema( - boolean needVerifySchema, int sgLevel, Set registeredDevices) - throws IOException, IllegalPathException, - VerifyMetadataException { // TODO: only support sg level=1 - try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { - List deviceList = new ArrayList<>(); - List measurementList = new ArrayList<>(); - List dataTypeList = new ArrayList<>(); - List encodingsList = new ArrayList<>(); - List compressionTypesList = new ArrayList<>(); - List isAlignedList = new ArrayList<>(); - - Map> device2Metadata = reader.getAllTimeseriesMetadata(true); - for (String device : device2Metadata.keySet()) { - deviceList.add(new PartialPath(device)); - } - autoCreateSg(deviceList, sgLevel, registeredDevices); - - for (Map.Entry> entry : device2Metadata.entrySet()) { - List timeseriesMetadataList = entry.getValue(); - boolean isAligned = - timeseriesMetadataList.stream() - .mapToInt(o -> o.getTSDataType().equals(TSDataType.VECTOR) ? 1 : 0) - .sum() - != 0; - int measurementSize = timeseriesMetadataList.size() - (isAligned ? 1 : 0); - String[] measurements = new String[measurementSize]; - TSDataType[] tsDataTypes = new TSDataType[measurementSize]; - TSEncoding[] encodings = new TSEncoding[measurementSize]; - CompressionType[] compressionTypes = new CompressionType[measurementSize]; - - int index = 0; - for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { - TSDataType dataType = timeseriesMetadata.getTSDataType(); - if (!dataType.equals(TSDataType.VECTOR)) { - measurements[index] = timeseriesMetadata.getMeasurementId(); - tsDataTypes[index] = dataType; - ChunkHeader chunkHeader = - getChunkHeaderByTimeseriesMetadata(reader, timeseriesMetadata); - encodings[index] = chunkHeader.getEncodingType(); - compressionTypes[index++] = chunkHeader.getCompressionType(); - } - } - measurementList.add(measurements); - dataTypeList.add(tsDataTypes); - encodingsList.add(encodings); - compressionTypesList.add(compressionTypes); - isAlignedList.add(isAligned); - } - - ISchemaTree schemaTree = - SchemaValidator.validate( - deviceList, - measurementList, - dataTypeList, - encodingsList, - compressionTypesList, - isAlignedList); - if (needVerifySchema) { - verifySchema( - schemaTree, - deviceList, - measurementList, - dataTypeList, - encodingsList, - compressionTypesList, - isAlignedList); - } - } - } - - private void autoCreateSg( - List devicePathList, int sgLevel, Set registeredDevices) - throws VerifyMetadataException { - for (PartialPath device : devicePathList) { - if (!registeredDevices.contains(device.toString())) { - String[] nodes = device.getNodes(); - String[] sgNodes = new String[sgLevel]; - if (nodes.length < sgLevel) { - throw new VerifyMetadataException( - String.format("Sg level %d is longer than device %s.", sgLevel, device)); - } - for (int i = 0; i < sgLevel; i++) { - sgNodes[i] = nodes[i]; - } - - SetStorageGroupStatement statement = new SetStorageGroupStatement(); - statement.setStorageGroupPath(new PartialPath(sgNodes)); - executeSetStorageGroupStatement(statement); - - registeredDevices.add(device.toString()); - } - } - } - - private void executeSetStorageGroupStatement(Statement statement) { - long queryId = SESSION_MANAGER.requestQueryId(false); - ExecutionResult result = - COORDINATOR.execute( - statement, - queryId, - null, - "", - PARTITION_FETCHER, - SCHEMA_FETCHER, - CONFIG.getQueryTimeoutThreshold()); - if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - logger.warn(String.format("Set Storage group error, statement: %s.", statement)); - logger.warn(String.format("Set storage group result status : %s.", result.status)); - } - } - - private ChunkHeader getChunkHeaderByTimeseriesMetadata( - TsFileSequenceReader reader, TimeseriesMetadata timeseriesMetadata) throws IOException { - IChunkMetadata chunkMetadata = timeseriesMetadata.getChunkMetadataList().get(0); - reader.position(chunkMetadata.getOffsetOfChunkHeader()); - return reader.readChunkHeader(reader.readMarker()); - } - - private void verifySchema( - ISchemaTree schemaTree, - List deviceList, - List measurementsList, - List tsDataTypesList, - List encodingsList, - List compressionTypeList, - List isAlignedList) - throws VerifyMetadataException { - int deviceSize = deviceList.size(); - for (int i = 0; i < deviceSize; i++) { - DeviceSchemaInfo schemaInfo = - schemaTree.searchDeviceSchemaInfo( - deviceList.get(i), Arrays.asList(measurementsList.get(i))); - String device = deviceList.get(i).toString(); - if (schemaInfo.isAligned() != isAlignedList.get(i)) { - throw new VerifyMetadataException( - device, - "Is aligned", - isAlignedList.get(i).toString(), - tsFile.getPath(), - String.valueOf(schemaInfo.isAligned())); - } - List measurementSchemaList = schemaInfo.getMeasurementSchemaList(); - int measurementSize = measurementSchemaList.size(); - for (int j = 0; j < measurementSize; j++) { - MeasurementSchema measurementSchema = measurementSchemaList.get(j); - if (tsDataTypesList.get(i)[j] != measurementSchema.getType()) { - throw new VerifyMetadataException( - device + measurementsList.get(i)[j], - "Datatype", - tsDataTypesList.get(i)[j].name(), - tsFile.getPath(), - measurementSchema.getType().name()); - } - if (!encodingsList.get(i)[j].equals(measurementSchema.getEncodingType())) { - throw new VerifyMetadataException( - device + measurementsList.get(i)[j], - "Encoding", - encodingsList.get(i)[j].name(), - tsFile.getPath(), - measurementSchema.getEncodingType().name()); - } - if (!compressionTypeList.get(i)[j].equals(measurementSchema.getCompressor())) { - throw new VerifyMetadataException( - device + measurementsList.get(i)[j], - "Compress type", - compressionTypeList.get(i)[j].name(), - tsFile.getPath(), - measurementSchema.getCompressor().name()); - } - } - } - } - public boolean needDecodeTsFile() { return needDecodeTsFile; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java index 9f15623531075..c0f6f8b8b540f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadTsFileNode.java @@ -20,37 +20,34 @@ package org.apache.iotdb.db.mpp.plan.planner.plan.node.load; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.tsfile.exception.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.DataOutputStream; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; public class LoadTsFileNode extends WritePlanNode { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileNode.class); - private final List tsFiles; + private final List resources; public LoadTsFileNode(PlanNodeId id) { this(id, new ArrayList<>()); } - public LoadTsFileNode(PlanNodeId id, List tsFiles) { + public LoadTsFileNode(PlanNodeId id, List resources) { super(id); - this.tsFiles = tsFiles; + this.resources = resources; } @Override @@ -89,24 +86,17 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { @Override public List splitByPartition(Analysis analysis) { - LoadTsFileStatement statement = (LoadTsFileStatement) analysis.getStatement(); List res = new ArrayList<>(); - Set registeredDevices = new HashSet<>(); - for (File file : tsFiles) { + for (TsFileResource resource : resources) { try { - LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), file); + LoadSingleTsFileNode singleTsFileNode = new LoadSingleTsFileNode(getPlanNodeId(), resource); singleTsFileNode.checkIfNeedDecodeTsFile(analysis.getDataPartitionInfo()); - if (statement.isAutoCreateSchema() || statement.isVerifySchema()) { - singleTsFileNode.autoRegisterSchema( - statement.isVerifySchema(), statement.getSgLevel(), registeredDevices); - } - if (singleTsFileNode.needDecodeTsFile()) { singleTsFileNode.splitTsFileByDataPartition(analysis.getDataPartitionInfo()); } res.add(singleTsFileNode); } catch (Exception e) { - logger.error(String.format("Parse TsFile %s error", file.getPath()), e); + logger.error(String.format("Parse TsFile %s error", resource.getTsFile().getPath()), e); } } return res; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java index e1a496b68540d..f5cd7e90cf8e4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/LoadTsFileStatement.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -40,14 +41,16 @@ public class LoadTsFileStatement extends Statement { private boolean verifySchema; private List tsFiles; + private List resources; public LoadTsFileStatement(String filePath) { this.file = new File(filePath); this.autoCreateSchema = true; this.sgLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel(); this.verifySchema = true; + this.tsFiles = new ArrayList<>(); + this.resources = new ArrayList<>(); - tsFiles = new ArrayList<>(); if (file.isFile()) { tsFiles.add(file); } else { @@ -114,6 +117,14 @@ public List getTsFiles() { return tsFiles; } + public void addTsFileResource(TsFileResource resource) { + resources.add(resource); + } + + public List getResources() { + return resources; + } + @Override public List getPaths() { return Collections.emptyList(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java index 28e95a0c6b31a..799dd03c3a7b2 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/SetStorageGroupStatement.java @@ -97,4 +97,20 @@ public Integer getDataReplicationFactor() { public Long getTimePartitionInterval() { return timePartitionInterval; } + + @Override + public String toString() { + return "SetStorageGroupStatement{" + + "storageGroupPath=" + + storageGroupPath + + ", ttl=" + + ttl + + ", schemaReplicationFactor=" + + schemaReplicationFactor + + ", dataReplicationFactor=" + + dataReplicationFactor + + ", timePartitionInterval=" + + timePartitionInterval + + '}'; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 35d8500107b6b..fd3e136486686 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -68,8 +69,14 @@ public static void loadOrGenerateResource(TsFileResource tsFileResource) throws public static void updateTsFileResource( TsFileSequenceReader reader, TsFileResource tsFileResource) throws IOException { - for (Entry> entry : - reader.getAllTimeseriesMetadata(false).entrySet()) { + updateTsFileResource(reader.getAllTimeseriesMetadata(false), tsFileResource); + tsFileResource.updatePlanIndexes(reader.getMinPlanIndex()); + tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex()); + } + + public static void updateTsFileResource( + Map> device2Metadata, TsFileResource tsFileResource) { + for (Entry> entry : device2Metadata.entrySet()) { for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { tsFileResource.updateStartTime( entry.getKey(), timeseriesMetaData.getStatistics().getStartTime()); @@ -77,8 +84,6 @@ public static void updateTsFileResource( entry.getKey(), timeseriesMetaData.getStatistics().getEndTime()); } } - tsFileResource.updatePlanIndexes(reader.getMinPlanIndex()); - tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex()); } /** From 84d33d3ebd282c676d6305e0ba48d336c93dce58 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 14 Sep 2022 15:40:02 +0800 Subject: [PATCH 28/37] add resource serialize --- .../org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 2 ++ .../plan/planner/plan/node/load/LoadSingleTsFileNode.java | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 84f08773ce642..e7c6a3c2e7844 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.VerifyMetadataException; import org.apache.iotdb.db.exception.metadata.template.TemplateImcompatibeException; @@ -1507,6 +1508,7 @@ private TsFileResource analyzeTsFile( device2MaxTime.getOrDefault(device, Long.MIN_VALUE), resource.getEndTime(device))); } + resource.setStatus(TsFileResourceStatus.CLOSED); return resource; } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index bed28ef76589d..81c54f2da0506 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -111,7 +111,7 @@ public LoadSingleTsFileNode(PlanNodeId id, TsFileResource resource) throws IOExc this.resource = resource; } - public void checkIfNeedDecodeTsFile(DataPartition dataPartition) { + public void checkIfNeedDecodeTsFile(DataPartition dataPartition) throws IOException { Set allRegionReplicaSet = new HashSet<>(); needDecodeTsFile = false; for (String device : resource.getDevices()) { @@ -123,6 +123,9 @@ public void checkIfNeedDecodeTsFile(DataPartition dataPartition) { allRegionReplicaSet.addAll(dataPartition.getAllDataRegionReplicaSetForOneDevice(device)); } needDecodeTsFile = !isDispatchedToLocal(allRegionReplicaSet); + if (!needDecodeTsFile) { + resource.serialize(); + } } private boolean isDispatchedToLocal(Set replicaSets) { From 233b32d82d6e26945bfec1ba2228f70adb84ecbb Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 14 Sep 2022 15:55:21 +0800 Subject: [PATCH 29/37] add doc --- .../org/apache/iotdb/db/engine/load/LoadTsFileManager.java | 7 +++++++ .../apache/iotdb/db/engine/storagegroup/DataRegion.java | 2 +- .../db/mpp/plan/scheduler/load/LoadTsFileScheduler.java | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java index 89252280861ba..48be7a395e2b0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/load/LoadTsFileManager.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler.LoadCommand; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.exception.write.PageException; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; @@ -45,6 +46,12 @@ import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; +/** + * {@link LoadTsFileManager} is used for dealing with {@link LoadTsFilePieceNode} and {@link + * LoadCommand}. This class turn the content of a piece of loading TsFile into a new TsFile. When + * DataNode finish transfer pieces, this class will flush all TsFile and laod them into IoTDB, or + * delete all. + */ public class LoadTsFileManager { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileManager.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index a4b678cdaf70a..bf8286c9a9fa6 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -2659,7 +2659,7 @@ private void tryToDeleteLastCacheByDevice(PartialPath deviceId) { if (!IoTDBDescriptor.getInstance().getConfig().isLastCacheEnabled()) { return; } - // try { + // try { TODO: support last cache // IoTDB.schemaProcessor.deleteLastCacheByDevice(deviceId); // } catch (MetadataException e) { // // the path doesn't cache in cluster mode now, ignore diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java index 6cd9df2d17aed..cad9b8b297645 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/load/LoadTsFileScheduler.java @@ -54,6 +54,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +/** + * {@link LoadTsFileScheduler} is used for scheduling {@link LoadSingleTsFileNode} and {@link + * LoadTsFilePieceNode}. because these two nodes need two phases to finish transfer. + * + *

for more details please check: ...; + */ public class LoadTsFileScheduler implements IScheduler { private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class); From d47a0f957f200d0f8ec591a60baf6681cd94ac24 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Wed, 14 Sep 2022 18:01:36 +0800 Subject: [PATCH 30/37] finsih IT --- .../apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 107 +++++++++++++++--- .../db/mpp/plan/analyze/AnalyzeVisitor.java | 4 +- .../plan/node/load/LoadSingleTsFileNode.java | 26 +---- 3 files changed, 97 insertions(+), 40 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java index 3334155273158..d72e58eed71ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.write.TsFileWriter; @@ -72,8 +73,6 @@ public void setUp() throws Exception { originPartitionInterval = ConfigFactory.getConfig().getPartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(PARTITION_INTERVAL); EnvFactory.getEnv().initBeforeTest(); - - registerSchema(); } @After @@ -159,6 +158,8 @@ private boolean deleteDir() { @Test public void testLoad() throws Exception { + registerSchema(); + long writtenPoint1 = 0; // device 0, device 1, sg 0 try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { @@ -199,7 +200,66 @@ public void testLoad() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath())); + statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); + + try (ResultSet resultSet = + statement.executeQuery("select count(*) from root.** group by level=1,2")) { + if (resultSet.next()) { + long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + Assert.assertEquals(writtenPoint1, sg1Count); + long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); + Assert.assertEquals(writtenPoint2, sg2Count); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + @Test + public void testLoadWithAutoRegister() throws Exception { + long writtenPoint1 = 0; + // device 0, device 1, sg 0 + try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_0), + Arrays.asList( + SchemaConfig.MEASUREMENT_00, + SchemaConfig.MEASUREMENT_01, + SchemaConfig.MEASUREMENT_02, + SchemaConfig.MEASUREMENT_03)); + generator.registerAlignedTimeseries( + new Path(SchemaConfig.DEVICE_1), + Arrays.asList( + SchemaConfig.MEASUREMENT_10, + SchemaConfig.MEASUREMENT_11, + SchemaConfig.MEASUREMENT_12, + SchemaConfig.MEASUREMENT_13)); + generator.generateData(new Path(SchemaConfig.DEVICE_0), 10000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_1), 10000, true); + writtenPoint1 = generator.getTotalNumber(); + } + + long writtenPoint2 = 0; + // device 2, device 3, device4, sg 1 + try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_2), Arrays.asList(SchemaConfig.MEASUREMENT_20)); + generator.registerTimeseries( + new Path(SchemaConfig.DEVICE_3), Arrays.asList(SchemaConfig.MEASUREMENT_30)); + generator.registerAlignedTimeseries( + new Path(SchemaConfig.DEVICE_4), Arrays.asList(SchemaConfig.MEASUREMENT_40)); + generator.generateData(new Path(SchemaConfig.DEVICE_2), 10000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_3), 10000, false); + generator.generateData(new Path(SchemaConfig.DEVICE_4), 10000, true); + writtenPoint2 = generator.getTotalNumber(); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute( + String.format("load \"%s\" sglevel=2,autoregister=true", tmpDir.getAbsolutePath())); try (ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { @@ -212,6 +272,25 @@ public void testLoad() throws Exception { Assert.fail("This ResultSet is empty."); } } + + Map isAligned = new HashMap<>(); + isAligned.put(SchemaConfig.DEVICE_0, "false"); + isAligned.put(SchemaConfig.DEVICE_1, "true"); + isAligned.put(SchemaConfig.DEVICE_2, "false"); + isAligned.put(SchemaConfig.DEVICE_3, "false"); + isAligned.put(SchemaConfig.DEVICE_4, "true"); + try (ResultSet resultSet = statement.executeQuery("show devices")) { + int size = 0; + while (resultSet.next()) { + size += 1; + String device = resultSet.getString("devices"); + Assert.assertEquals(isAligned.get(device), resultSet.getString("isAligned")); + } + Assert.assertEquals(isAligned.size(), size); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail("Parse result set error."); + } } } @@ -222,39 +301,39 @@ private static class SchemaConfig { // device 0, nonaligned, sg 0 private static final String DEVICE_0 = "root.sg.test_0.d_0"; private static final MeasurementSchema MEASUREMENT_00 = - new MeasurementSchema("sensor_00", TSDataType.INT32); + new MeasurementSchema("sensor_00", TSDataType.INT32, TSEncoding.RLE); private static final MeasurementSchema MEASUREMENT_01 = - new MeasurementSchema("sensor_01", TSDataType.INT64); + new MeasurementSchema("sensor_01", TSDataType.INT64, TSEncoding.RLE); private static final MeasurementSchema MEASUREMENT_02 = - new MeasurementSchema("sensor_02", TSDataType.DOUBLE); + new MeasurementSchema("sensor_02", TSDataType.DOUBLE, TSEncoding.GORILLA); private static final MeasurementSchema MEASUREMENT_03 = - new MeasurementSchema("sensor_03", TSDataType.TEXT); + new MeasurementSchema("sensor_03", TSDataType.TEXT, TSEncoding.PLAIN); // device 1, aligned, sg 0 private static final String DEVICE_1 = "root.sg.test_0.a_1"; private static final MeasurementSchema MEASUREMENT_10 = - new MeasurementSchema("sensor_10", TSDataType.INT32); + new MeasurementSchema("sensor_10", TSDataType.INT32, TSEncoding.RLE); private static final MeasurementSchema MEASUREMENT_11 = - new MeasurementSchema("sensor_11", TSDataType.INT64); + new MeasurementSchema("sensor_11", TSDataType.INT64, TSEncoding.RLE); private static final MeasurementSchema MEASUREMENT_12 = - new MeasurementSchema("sensor_12", TSDataType.DOUBLE); + new MeasurementSchema("sensor_12", TSDataType.DOUBLE, TSEncoding.GORILLA); private static final MeasurementSchema MEASUREMENT_13 = - new MeasurementSchema("sensor_13", TSDataType.TEXT); + new MeasurementSchema("sensor_13", TSDataType.TEXT, TSEncoding.PLAIN); // device 2, non aligned, sg 1 private static final String DEVICE_2 = "root.sg.test_1.d_2"; private static final MeasurementSchema MEASUREMENT_20 = - new MeasurementSchema("sensor_20", TSDataType.INT32); + new MeasurementSchema("sensor_20", TSDataType.INT32, TSEncoding.RLE); // device 3, non aligned, sg 1 private static final String DEVICE_3 = "root.sg.test_1.d_3"; private static final MeasurementSchema MEASUREMENT_30 = - new MeasurementSchema("sensor_30", TSDataType.INT32); + new MeasurementSchema("sensor_30", TSDataType.INT32, TSEncoding.RLE); // device 4, aligned, sg 1 private static final String DEVICE_4 = "root.sg.test_1.a_4"; private static final MeasurementSchema MEASUREMENT_40 = - new MeasurementSchema("sensor_40", TSDataType.INT32); + new MeasurementSchema("sensor_40", TSDataType.INT32, TSEncoding.RLE); } public class TsFileGenerator implements AutoCloseable { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index e7c6a3c2e7844..614f775d9c3dc 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -108,6 +108,7 @@ import org.apache.iotdb.db.query.control.SessionManager; import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; @@ -1664,7 +1665,8 @@ private void verifySchema( for (int j = 0; j < measurementSize; j++) { MeasurementSchema originSchema = originSchemaList.get(j); MeasurementSchema tsFileSchema = tsFileSchemas[j]; - String measurementPath = device + originSchema.getMeasurementId(); + String measurementPath = + device + TsFileConstant.PATH_SEPARATOR + originSchema.getMeasurementId(); if (!tsFileSchema.getType().equals(originSchema.getType())) { throw new VerifyMetadataException( measurementPath, diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 81c54f2da0506..21edd2ed0b46e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -24,24 +24,15 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartition; -import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.load.AlignedChunkData; import org.apache.iotdb.db.engine.load.ChunkData; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.mpp.plan.Coordinator; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; -import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.ClusterSchemaFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.ISchemaFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.StandalonePartitionFetcher; -import org.apache.iotdb.db.mpp.plan.analyze.StandaloneSchemaFetcher; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.query.control.SessionManager; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -77,21 +68,6 @@ public class LoadSingleTsFileNode extends WritePlanNode { private static final Logger logger = LoggerFactory.getLogger(LoadSingleTsFileNode.class); - private static final Coordinator COORDINATOR = Coordinator.getInstance(); - private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); - private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); - private final IPartitionFetcher PARTITION_FETCHER; - private final ISchemaFetcher SCHEMA_FETCHER; - - { - if (CONFIG.isClusterMode()) { - PARTITION_FETCHER = ClusterPartitionFetcher.getInstance(); - SCHEMA_FETCHER = ClusterSchemaFetcher.getInstance(); - } else { - PARTITION_FETCHER = StandalonePartitionFetcher.getInstance(); - SCHEMA_FETCHER = StandaloneSchemaFetcher.getInstance(); - } - } private File tsFile; private boolean needDecodeTsFile; @@ -105,7 +81,7 @@ public LoadSingleTsFileNode(PlanNodeId id) { super(id); } - public LoadSingleTsFileNode(PlanNodeId id, TsFileResource resource) throws IOException { + public LoadSingleTsFileNode(PlanNodeId id, TsFileResource resource) { super(id); this.tsFile = resource.getTsFile(); this.resource = resource; From 7d39ee99e9561d0ae2e442b514494b6e5957bfa5 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 15 Sep 2022 20:54:40 +0800 Subject: [PATCH 31/37] fix SchemaRegionMemoryImpl --- .../db/metadata/schemaregion/SchemaRegionMemoryImpl.java | 6 ++++++ .../apache/iotdb/db/mpp/plan/execution/QueryExecution.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java index d365c728ad133..cb965be8fce84 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java @@ -1770,6 +1770,12 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) private void internalCreateTimeseries( PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index b96cc15cc98fe..a471be8780343 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -57,8 +57,8 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; -import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.read.common.block.TsBlock; From ab5af3daa8bc203d51257a0d2ea5a219cb90b6e3 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 16 Sep 2022 09:24:39 +0800 Subject: [PATCH 32/37] spotless --- .../org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index b96cc15cc98fe..a471be8780343 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -57,8 +57,8 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement; import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement; -import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.read.common.block.TsBlock; From 502c59d7717118f9e1983e0bda432bea55c3d2f8 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 16 Sep 2022 10:04:57 +0800 Subject: [PATCH 33/37] fix Collections.singletonlist --- .../schemaregion/SchemaRegionMemoryImpl.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java index cb965be8fce84..2ecae57700591 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java @@ -1787,15 +1787,6 @@ private void internalAlignedCreateTimeseries( List encodings, List compressors) throws MetadataException { - int measurementsSize = measurements.size(); - for (int i = 0; i < measurementsSize; i++) { - if (encodings.get(i) == null) { - encodings.set(i, getDefaultEncoding(dataTypes.get(i))); - } - if (compressors.get(i) == null) { - compressors.set(i, TSFileDescriptor.getInstance().getConfig().getCompressor()); - } - } createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } @@ -1830,13 +1821,17 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( if (measurementMNode == null) { if (config.isAutoCreateSchemaEnabled()) { if (aligned) { + TSDataType dataType = getDataType.apply(i); internalAlignedCreateTimeseries( devicePath, Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i)), - Collections.singletonList(encodings[i]), - Collections.singletonList(compressionTypes[i])); - + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); } else { internalCreateTimeseries( devicePath.concatNode(measurements[i]), From 5eb1e9b2c3e5a299f4a62b9e68fbc67f524ceee9 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 16 Sep 2022 15:57:15 +0800 Subject: [PATCH 34/37] spotless --- .../db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index 5a0c7ef28e09f..a63b684274164 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -68,12 +68,11 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.mpp.plan.planner.plan.node.load.LoadTsFilePieceNode; -import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.DeleteTimeSeriesNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.metedata.write.RollbackSchemaBlackListNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.mpp.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.service.DataNode; import org.apache.iotdb.db.service.RegionMigrateService; import org.apache.iotdb.db.service.metrics.MetricService; From a7c8bb0444703c98a63a78f28606a76cc41f9a04 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Fri, 16 Sep 2022 16:11:33 +0800 Subject: [PATCH 35/37] spotless --- .../plan/analyze/ClusterSchemaFetcher.java | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java index 69ef2a019abb7..a78e992016bb8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java @@ -201,16 +201,16 @@ public ISchemaTree fetchSchemaWithAutoCreate( return schemaTree; } - ClusterSchemaTree missingSchemaTree = - checkAndAutoCreateMissingMeasurements( - remoteSchemaTree, - devicePath, - indexOfMissingMeasurements, - measurements, - getDataType, - null, - null, - isAligned); + ClusterSchemaTree missingSchemaTree = + checkAndAutoCreateMissingMeasurements( + remoteSchemaTree, + devicePath, + indexOfMissingMeasurements, + measurements, + getDataType, + null, + null, + isAligned); schemaTree.mergeSchemaTree(missingSchemaTree); schemaCache.put(missingSchemaTree); @@ -242,18 +242,18 @@ public ISchemaTree fetchSchemaListWithAutoCreate( schemaCache.takeReadLock(); try { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - PathPatternTree patternTree = new PathPatternTree(); - List> indexOfMissingMeasurementsList = new ArrayList<>(devicePathList.size()); - for (int i = 0; i < devicePathList.size(); i++) { - schemaTree.mergeSchemaTree(schemaCache.get(devicePathList.get(i), measurementsList.get(i))); - List indexOfMissingMeasurements = - checkMissingMeasurements(schemaTree, devicePathList.get(i), measurementsList.get(i)); - indexOfMissingMeasurementsList.add(indexOfMissingMeasurements); - for (int index : indexOfMissingMeasurements) { - patternTree.appendFullPath(devicePathList.get(i), measurementsList.get(i)[index]); + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + PathPatternTree patternTree = new PathPatternTree(); + List> indexOfMissingMeasurementsList = new ArrayList<>(devicePathList.size()); + for (int i = 0; i < devicePathList.size(); i++) { + schemaTree.mergeSchemaTree(schemaCache.get(devicePathList.get(i), measurementsList.get(i))); + List indexOfMissingMeasurements = + checkMissingMeasurements(schemaTree, devicePathList.get(i), measurementsList.get(i)); + indexOfMissingMeasurementsList.add(indexOfMissingMeasurements); + for (int index : indexOfMissingMeasurements) { + patternTree.appendFullPath(devicePathList.get(i), measurementsList.get(i)[index]); + } } - } if (patternTree.isEmpty()) { return schemaTree; @@ -269,23 +269,23 @@ public ISchemaTree fetchSchemaListWithAutoCreate( return schemaTree; } - ClusterSchemaTree missingSchemaTree; - for (int i = 0; i < devicePathList.size(); i++) { - int finalI = i; - missingSchemaTree = - checkAndAutoCreateMissingMeasurements( - schemaTree, - devicePathList.get(i), - indexOfMissingMeasurementsList.get(i), - measurementsList.get(i), - index -> tsDataTypesList.get(finalI)[index], - encodingsList == null ? null : encodingsList.get(i), - compressionTypesList == null ? null : compressionTypesList.get(i), - isAlignedList.get(i)); - schemaTree.mergeSchemaTree(missingSchemaTree); - schemaCache.put(missingSchemaTree); - } - return schemaTree; + ClusterSchemaTree missingSchemaTree; + for (int i = 0; i < devicePathList.size(); i++) { + int finalI = i; + missingSchemaTree = + checkAndAutoCreateMissingMeasurements( + schemaTree, + devicePathList.get(i), + indexOfMissingMeasurementsList.get(i), + measurementsList.get(i), + index -> tsDataTypesList.get(finalI)[index], + encodingsList == null ? null : encodingsList.get(i), + compressionTypesList == null ? null : compressionTypesList.get(i), + isAlignedList.get(i)); + schemaTree.mergeSchemaTree(missingSchemaTree); + schemaCache.put(missingSchemaTree); + } + return schemaTree; } finally { schemaCache.releaseReadLock(); } From d69fc73e630fe93483efc3b89bf952588ba1f59d Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 19 Sep 2022 12:55:25 +0800 Subject: [PATCH 36/37] fix merge --- .../apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 6 +----- .../apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 1 - .../iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 4c492c5fdc085..080a229537c41 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; @@ -34,9 +35,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.db.engine.StorageEngineV2; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.template.TemplateImcompatibeException; import org.apache.iotdb.db.exception.sql.MeasurementNotExistException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -49,7 +47,6 @@ import org.apache.iotdb.db.mpp.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; -import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; import org.apache.iotdb.db.mpp.plan.Coordinator; import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult; import org.apache.iotdb.db.mpp.plan.expression.Expression; @@ -115,7 +112,6 @@ import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java index 610209fa725fc..a688949cbb2ba 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ISchemaFetcher.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.metadata.template.Template; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; -import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java index d49432cfcdc6c..f4060e7c4ed05 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/StandaloneSchemaFetcher.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.mpp.common.schematree.DeviceGroupSchemaTree; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; -import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; From 855366f575b7a370e23bb96e2a5078e98b9427e0 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Mon, 19 Sep 2022 15:15:47 +0800 Subject: [PATCH 37/37] fix init class --- .../db/mpp/plan/analyze/AnalyzeVisitor.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 080a229537c41..c0df8bc3cbcb1 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -28,7 +28,6 @@ import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngineV2; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -153,9 +152,6 @@ public class AnalyzeVisitor extends StatementVisitor private static final Logger logger = LoggerFactory.getLogger(Analyzer.class); - private final Coordinator coordinator = Coordinator.getInstance(); - private final SessionManager sessionManager = SessionManager.getInstance(); - private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private final IPartitionFetcher partitionFetcher; private final ISchemaFetcher schemaFetcher; private final MPPQueryContext context; @@ -1634,16 +1630,17 @@ private void autoCreateSg(int sgLevel, Map> } private void executeSetStorageGroupStatement(Statement statement) throws LoadFileException { - long queryId = sessionManager.requestQueryId(false); + long queryId = SessionManager.getInstance().requestQueryId(false); ExecutionResult result = - coordinator.execute( - statement, - queryId, - null, - "", - partitionFetcher, - schemaFetcher, - config.getQueryTimeoutThreshold()); + Coordinator.getInstance() + .execute( + statement, + queryId, + null, + "", + partitionFetcher, + schemaFetcher, + IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold()); if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() && result.status.code != TSStatusCode.STORAGE_GROUP_ALREADY_EXISTS.getStatusCode()) { logger.error(String.format("Set Storage group error, statement: %s.", statement));