From 2ba68f064ebcdeb0902e4744cf84f84a1e3ec7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=AD=90=E5=9D=A4?= <55695098+DanielWang2035@users.noreply.github.com> Date: Tue, 15 Oct 2024 19:45:51 +0800 Subject: [PATCH] Load: fix some minor issues --- .../plan/analyze/load/LoadTsFileAnalyzer.java | 6 ++- .../load/LoadTsFileToTableModelAnalyzer.java | 10 ++++- .../load/LoadTsFileToTreeModelAnalyzer.java | 2 + .../relational/sql/parser/AstBuilder.java | 44 +++++++------------ .../relational/sql/util/SqlFormatter.java | 32 +++++++------- .../relational/grammar/sql/RelationalSql.g4 | 6 +-- 6 files changed, 49 insertions(+), 51 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 6e965c80b01db..9a65349ab34d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze.load; import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.db.exception.VerifyMetadataException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; @@ -143,7 +144,7 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { } catch (Exception e) { final String exceptionMessage = String.format( - "The file %s is not a valid tsfile. Please check the input file. Detail: %s", + "Loading file %s failed. Detail: %s", tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage()); LOGGER.warn(exceptionMessage, e); analysis.setFinishQueryAfterAnalyze(true); @@ -154,7 +155,8 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { return true; } - protected abstract void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException; + protected abstract void analyzeSingleTsFile(final File tsFile) + throws IOException, AuthException, VerifyMetadataException; protected TsFileResource constructTsFileResource( final TsFileSequenceReader reader, final File tsFile) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java index 7316e7a3de136..27c7efa1b2379 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java @@ -115,7 +115,8 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } @Override - protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException { + protected void analyzeSingleTsFile(final File tsFile) + throws IOException, AuthException, VerifyMetadataException { try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { // can be reused when constructing tsfile resource final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator = @@ -127,6 +128,8 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx } // check whether the tsfile is table-model or not + // TODO: currently, loading a file with both tree-model and table-model data is not supported. + // May need to support this and remove this check in the future. if (Objects.isNull(reader.readFileMetadata().getTableSchemaMap()) || reader.readFileMetadata().getTableSchemaMap().size() == 0) { throw new SemanticException("Attempted to load a tree-model TsFile into table-model."); @@ -147,7 +150,10 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx true) .orElse(null); if (Objects.isNull(realSchema)) { - LOGGER.warn("Failed to validata schema for table {}", name2Schema.getValue()); + throw new VerifyMetadataException( + String.format( + "Failed to validate schema for table {%s, %s}", + name2Schema.getKey(), name2Schema.getValue())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java index 5552109999d28..6e2382dc30d92 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java @@ -119,6 +119,8 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx } // check whether the tsfile is tree-model or not + // TODO: currently, loading a file with both tree-model and table-model data is not supported. + // May need to support this and remove this check in the future. if (Objects.nonNull(reader.readFileMetadata().getTableSchemaMap()) && reader.readFileMetadata().getTableSchemaMap().size() != 0) { throw new SemanticException("Attempted to load a table-model TsFile into tree-model."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index ce4ae9f7a52ea..1ccb712b84c50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -561,40 +561,26 @@ public Node visitShowFunctionsStatement(RelationalSqlParser.ShowFunctionsStateme @Override public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementContext ctx) { - final Map loadTsFileAttributes = new HashMap<>(); - if (ctx.loadFileWithAttributeClauses() != null) { - for (RelationalSqlParser.LoadFileWithAttributeClauseContext attributeContext : - ctx.loadFileWithAttributeClauses().loadFileWithAttributeClause()) { - final String key = - parseStringLiteral(attributeContext.loadFileWithKey.getText()).trim().toLowerCase(); - final String value = - parseStringLiteral(attributeContext.loadFileWithValue.getText()).trim().toLowerCase(); - - LoadTsFileConfigurator.validateParameters(key, value); - loadTsFileAttributes.put(key, value); - } - } + final Map withAttributes = + ctx.loadFileWithAttributesClause() != null + ? parseLoadFileWithAttributeClauses( + ctx.loadFileWithAttributesClause().loadFileWithAttributeClause()) + : new HashMap<>(); + withAttributes.forEach(LoadTsFileConfigurator::validateParameters); return new LoadTsFile( - getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(), loadTsFileAttributes); + getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(), withAttributes); } - public static String parseStringLiteral(String src) { - if (2 <= src.length()) { - // do not unescape string - String unWrappedString = src.substring(1, src.length() - 1); - if (src.charAt(0) == '\"' && src.charAt(src.length() - 1) == '\"') { - // replace "" with " - String replaced = unWrappedString.replace("\"\"", "\""); - return replaced.length() == 0 ? "" : replaced; - } - if ((src.charAt(0) == '\'' && src.charAt(src.length() - 1) == '\'')) { - // replace '' with ' - String replaced = unWrappedString.replace("''", "'"); - return replaced.length() == 0 ? "" : replaced; - } + private Map parseLoadFileWithAttributeClauses( + List contexts) { + final Map withAttributesMap = new HashMap<>(); + for (RelationalSqlParser.LoadFileWithAttributeClauseContext context : contexts) { + withAttributesMap.put( + ((StringLiteral) visit(context.loadFileWithKey)).getValue(), + ((StringLiteral) visit(context.loadFileWithValue)).getValue()); } - return src; + return withAttributesMap; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 355f56cdea179..e5b0fb3e969f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -804,21 +804,23 @@ protected Void visitLoadTsFile(final LoadTsFile node, final Integer indent) { builder.append("\"" + node.getFilePath() + "\""); builder.append(" \n"); - builder - .append("WITH (") - .append("\n") - .append( - node.getLoadAttributes().entrySet().stream() - .map( - entry -> - indentString(1) - + "\"" - + entry.getKey() - + "\" = \"" - + entry.getValue() - + "\"") - .collect(joining(", " + "\n"))) - .append(")\n"); + if (!node.getLoadAttributes().isEmpty()) { + builder + .append("WITH (") + .append("\n") + .append( + node.getLoadAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")\n"); + } return null; } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index f1d5f14c4150b..4515c46e86632 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -240,10 +240,10 @@ showFunctionsStatement // -------------------------------------------- Load Statement --------------------------------------------------------- loadTsFileStatement - : LOAD fileName=string (loadFileWithAttributeClauses)? + : LOAD fileName=string (loadFileWithAttributesClause)? ; -loadFileWithAttributeClauses +loadFileWithAttributesClause : WITH '(' (loadFileWithAttributeClause ',')* loadFileWithAttributeClause? @@ -251,7 +251,7 @@ loadFileWithAttributeClauses ; loadFileWithAttributeClause - : loadFileWithKey=STRING EQ loadFileWithValue=STRING + : loadFileWithKey=string EQ loadFileWithValue=string ;