Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.analyze.load;

import org.apache.iotdb.commons.auth.AuthException;
import org.apache.iotdb.db.exception.VerifyMetadataException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
Expand Down Expand Up @@ -143,7 +144,7 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) {
} catch (Exception e) {
final String exceptionMessage =
String.format(
"The file %s is not a valid tsfile. Please check the input file. Detail: %s",
"Loading file %s failed. Detail: %s",
tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage());
LOGGER.warn(exceptionMessage, e);
analysis.setFinishQueryAfterAnalyze(true);
Expand All @@ -154,7 +155,8 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) {
return true;
}

protected abstract void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException;
protected abstract void analyzeSingleTsFile(final File tsFile)
throws IOException, AuthException, VerifyMetadataException;

protected TsFileResource constructTsFileResource(
final TsFileSequenceReader reader, final File tsFile) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) {
}

@Override
protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException {
protected void analyzeSingleTsFile(final File tsFile)
throws IOException, AuthException, VerifyMetadataException {
try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
// can be reused when constructing tsfile resource
final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator =
Expand All @@ -127,6 +128,8 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx
}

// check whether the tsfile is table-model or not
// TODO: currently, loading a file with both tree-model and table-model data is not supported.
// May need to support this and remove this check in the future.
if (Objects.isNull(reader.readFileMetadata().getTableSchemaMap())
|| reader.readFileMetadata().getTableSchemaMap().size() == 0) {
throw new SemanticException("Attempted to load a tree-model TsFile into table-model.");
Expand All @@ -147,7 +150,10 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx
true)
.orElse(null);
if (Objects.isNull(realSchema)) {
LOGGER.warn("Failed to validata schema for table {}", name2Schema.getValue());
throw new VerifyMetadataException(
String.format(
"Failed to validate schema for table {%s, %s}",
name2Schema.getKey(), name2Schema.getValue()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthEx
}

// check whether the tsfile is tree-model or not
// TODO: currently, loading a file with both tree-model and table-model data is not supported.
// May need to support this and remove this check in the future.
if (Objects.nonNull(reader.readFileMetadata().getTableSchemaMap())
&& reader.readFileMetadata().getTableSchemaMap().size() != 0) {
throw new SemanticException("Attempted to load a table-model TsFile into tree-model.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -561,40 +561,26 @@ public Node visitShowFunctionsStatement(RelationalSqlParser.ShowFunctionsStateme

@Override
public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementContext ctx) {
final Map<String, String> loadTsFileAttributes = new HashMap<>();
if (ctx.loadFileWithAttributeClauses() != null) {
for (RelationalSqlParser.LoadFileWithAttributeClauseContext attributeContext :
ctx.loadFileWithAttributeClauses().loadFileWithAttributeClause()) {
final String key =
parseStringLiteral(attributeContext.loadFileWithKey.getText()).trim().toLowerCase();
final String value =
parseStringLiteral(attributeContext.loadFileWithValue.getText()).trim().toLowerCase();

LoadTsFileConfigurator.validateParameters(key, value);
loadTsFileAttributes.put(key, value);
}
}
final Map<String, String> withAttributes =
ctx.loadFileWithAttributesClause() != null
? parseLoadFileWithAttributeClauses(
ctx.loadFileWithAttributesClause().loadFileWithAttributeClause())
: new HashMap<>();

withAttributes.forEach(LoadTsFileConfigurator::validateParameters);
return new LoadTsFile(
getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(), loadTsFileAttributes);
getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(), withAttributes);
}

public static String parseStringLiteral(String src) {
if (2 <= src.length()) {
// do not unescape string
String unWrappedString = src.substring(1, src.length() - 1);
if (src.charAt(0) == '\"' && src.charAt(src.length() - 1) == '\"') {
// replace "" with "
String replaced = unWrappedString.replace("\"\"", "\"");
return replaced.length() == 0 ? "" : replaced;
}
if ((src.charAt(0) == '\'' && src.charAt(src.length() - 1) == '\'')) {
// replace '' with '
String replaced = unWrappedString.replace("''", "'");
return replaced.length() == 0 ? "" : replaced;
}
private Map<String, String> parseLoadFileWithAttributeClauses(
List<RelationalSqlParser.LoadFileWithAttributeClauseContext> contexts) {
final Map<String, String> withAttributesMap = new HashMap<>();
for (RelationalSqlParser.LoadFileWithAttributeClauseContext context : contexts) {
withAttributesMap.put(
((StringLiteral) visit(context.loadFileWithKey)).getValue(),
((StringLiteral) visit(context.loadFileWithValue)).getValue());
}
return src;
return withAttributesMap;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -804,21 +804,23 @@ protected Void visitLoadTsFile(final LoadTsFile node, final Integer indent) {
builder.append("\"" + node.getFilePath() + "\"");
builder.append(" \n");

builder
.append("WITH (")
.append("\n")
.append(
node.getLoadAttributes().entrySet().stream()
.map(
entry ->
indentString(1)
+ "\""
+ entry.getKey()
+ "\" = \""
+ entry.getValue()
+ "\"")
.collect(joining(", " + "\n")))
.append(")\n");
if (!node.getLoadAttributes().isEmpty()) {
builder
.append("WITH (")
.append("\n")
.append(
node.getLoadAttributes().entrySet().stream()
.map(
entry ->
indentString(1)
+ "\""
+ entry.getKey()
+ "\" = \""
+ entry.getValue()
+ "\"")
.collect(joining(", " + "\n")))
.append(")\n");
}
return null;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -240,18 +240,18 @@ showFunctionsStatement

// -------------------------------------------- Load Statement ---------------------------------------------------------
loadTsFileStatement
: LOAD fileName=string (loadFileWithAttributeClauses)?
: LOAD fileName=string (loadFileWithAttributesClause)?
;

loadFileWithAttributeClauses
loadFileWithAttributesClause
: WITH
'('
(loadFileWithAttributeClause ',')* loadFileWithAttributeClause?
')'
;

loadFileWithAttributeClause
: loadFileWithKey=STRING EQ loadFileWithValue=STRING
: loadFileWithKey=string EQ loadFileWithValue=string
;


Expand Down