Skip to content

Commit

Permalink
Use a batched style to get series type in PhysicalGenerator, TSServic…
Browse files Browse the repository at this point in the history 
…eImpl and ConcatPathOptimizer
  • Loading branch information
@jt2594838
jt2594838 committed Mar 11, 2020
1 parent 66eb044 commit 4d62652
Show file tree
Hide file tree
Showing 10 changed files with 195 additions and 86 deletions.
5 changes: 5 additions & 0 deletions server/src/main/java/org/apache/iotdb/db/exception/query/LogicalOperatorException.java
View file
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.exception.query;

import org.apache.iotdb.db.exception.IoTDBException;
import org.apache.iotdb.rpc.TSStatusCode;

/**
Expand All @@ -44,4 +45,8 @@ public LogicalOperatorException(String type, String message) {
super(String.format("Unsupported type: [%s]. %s", type, message),
TSStatusCode.LOGICAL_OPERATOR_ERROR.getStatusCode());
}

public LogicalOperatorException(IoTDBException e) {
super(e);
}
}
5 changes: 5 additions & 0 deletions server/src/main/java/org/apache/iotdb/db/exception/query/LogicalOptimizeException.java
View file
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.exception.query;

import org.apache.iotdb.db.exception.IoTDBException;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.rpc.TSStatusCode;

Expand All @@ -36,4 +37,8 @@ public LogicalOptimizeException(String filterOperator, int tokenInt) {
super(String.format("Unknown token in [%s]: [%s], [%s].", filterOperator, tokenInt,
SQLConstant.tokenNames.get(tokenInt)), TSStatusCode.LOGICAL_OPTIMIZE_ERROR.getStatusCode());
}

public LogicalOptimizeException(IoTDBException e) {
super(e);
}
}
4 changes: 4 additions & 0 deletions server/src/main/java/org/apache/iotdb/db/qp/Planner.java
View file
Expand Up @@ -19,6 +19,7 @@
package org.apache.iotdb.db.qp;

import java.time.ZoneId;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
Expand All @@ -35,6 +36,7 @@
import org.apache.iotdb.db.qp.strategy.optimizer.MergeSingleFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.RemoveNotOptimizer;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.read.common.Path;

/**
* provide a integration method for other user.
Expand Down Expand Up @@ -117,13 +119,15 @@ private SFWOperator optimizeSFWOperator(SFWOperator root)
if (filter == null) {
return root;
}
Set<Path> pathSet = filter.getPathSet();
RemoveNotOptimizer removeNot = new RemoveNotOptimizer();
filter = removeNot.optimize(filter);
DnfFilterOptimizer dnf = new DnfFilterOptimizer();
filter = dnf.optimize(filter);
MergeSingleFilterOptimizer merge = new MergeSingleFilterOptimizer();
filter = merge.optimize(filter);
root.setFilterOperator(filter);
filter.setPathSet(pathSet);
return root;
}

Expand Down
8 changes: 5 additions & 3 deletions server/src/main/java/org/apache/iotdb/db/qp/logical/crud/BasicFunctionOperator.java
View file
Expand Up @@ -18,11 +18,11 @@
*/
package org.apache.iotdb.db.qp.logical.crud;

import java.util.Map;
import java.util.Objects;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
import org.apache.iotdb.db.exception.runtime.SQLParserException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
Expand Down Expand Up @@ -74,9 +74,10 @@ public void reverseFunc() {
}

@Override
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter()
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter(
Map<Path, TSDataType> pathTSDataTypeHashMap)
throws LogicalOperatorException, MetadataException {
TSDataType type = MManager.getInstance().getSeriesType(singlePath.toString());
TSDataType type = pathTSDataTypeHashMap.get(singlePath);
if (type == null) {
throw new MetadataException(
"given seriesPath:{" + singlePath.getFullPath() + "} don't exist in metadata");
Expand Down Expand Up @@ -134,6 +135,7 @@ public BasicFunctionOperator copy() {
ret.tokenSymbol = tokenSymbol;
ret.isLeaf = isLeaf;
ret.isSingle = isSingle;
ret.pathSet = pathSet;
return ret;
}

Expand Down
31 changes: 24 additions & 7 deletions server/src/main/java/org/apache/iotdb/db/qp/logical/crud/FilterOperator.java
View file
Expand Up @@ -23,11 +23,14 @@

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.IUnaryExpression;
Expand All @@ -54,6 +57,8 @@ public class FilterOperator extends Operator implements Comparable<FilterOperato
boolean isSingle = false;
// if isSingle = false, singlePath must be null
Path singlePath = null;
// all paths involved in this filter
Set<Path> pathSet;

public FilterOperator(int tokenType) {
super(tokenType);
Expand Down Expand Up @@ -105,17 +110,27 @@ public boolean addChildOperator(FilterOperator op) {
return true;
}

public void setPathSet(Set<Path> pathSet) {
this.pathSet = pathSet;
}

public Set<Path> getPathSet() {
return pathSet;
}

/**
* For a filter operator, if isSingle, call transformToSingleQueryFilter.<br> FilterOperator
* cannot be leaf.
*
* @return QueryFilter in TsFile
* @param pathTSDataTypeHashMap
*/
public IExpression transformToExpression() throws QueryProcessException {
public IExpression transformToExpression(
Map<Path, TSDataType> pathTSDataTypeHashMap) throws QueryProcessException {
if (isSingle) {
Pair<IUnaryExpression, String> ret;
try {
ret = transformToSingleQueryFilter();
ret = transformToSingleQueryFilter(pathTSDataTypeHashMap);
} catch (MetadataException e) {
throw new QueryProcessException(e);
}
Expand All @@ -125,10 +140,10 @@ public IExpression transformToExpression() throws QueryProcessException {
throw new LogicalOperatorException(String.valueOf(tokenIntType),
"this filter is not leaf, but it's empty");
}
IExpression retFilter = childOperators.get(0).transformToExpression();
IExpression retFilter = childOperators.get(0).transformToExpression(pathTSDataTypeHashMap);
IExpression currentFilter;
for (int i = 1; i < childOperators.size(); i++) {
currentFilter = childOperators.get(i).transformToExpression();
currentFilter = childOperators.get(i).transformToExpression(pathTSDataTypeHashMap);
switch (tokenIntType) {
case KW_AND:
retFilter = BinaryExpression.and(retFilter, currentFilter);
Expand All @@ -151,21 +166,23 @@ public IExpression transformToExpression() throws QueryProcessException {
* @return - pair.left: UnaryQueryFilter constructed by its one child; pair.right: Path
* represented by this child.
* @throws MetadataException exception in filter transforming
* @param pathTSDataTypeHashMap
*/
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter()
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter(
Map<Path, TSDataType> pathTSDataTypeHashMap)
throws LogicalOperatorException, MetadataException {
if (childOperators.isEmpty()) {
throw new LogicalOperatorException(String.valueOf(tokenIntType),
"TransformToSingleFilter: this filter is not a leaf, but it's empty.");
}
Pair<IUnaryExpression, String> currentPair = childOperators.get(0)
.transformToSingleQueryFilter();
.transformToSingleQueryFilter(pathTSDataTypeHashMap);

IUnaryExpression retFilter = currentPair.left;
String path = currentPair.right;

for (int i = 1; i < childOperators.size(); i++) {
currentPair = childOperators.get(i).transformToSingleQueryFilter();
currentPair = childOperators.get(i).transformToSingleQueryFilter(pathTSDataTypeHashMap);
if (!path.equals(currentPair.right)) {
throw new LogicalOperatorException(
"TransformToSingleFilter: paths among children are not inconsistent: one is: "
Expand Down
8 changes: 5 additions & 3 deletions server/src/main/java/org/apache/iotdb/db/qp/logical/crud/InOperator.java
View file
Expand Up @@ -22,11 +22,11 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOperatorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
Expand Down Expand Up @@ -75,9 +75,10 @@ public void reverseFunc() {
}

@Override
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter()
protected Pair<IUnaryExpression, String> transformToSingleQueryFilter(
Map<Path, TSDataType> pathTSDataTypeHashMap)
throws LogicalOperatorException, MetadataException {
TSDataType type = MManager.getInstance().getSeriesType(singlePath.toString());
TSDataType type = pathTSDataTypeHashMap.get(singlePath);
if (type == null) {
throw new MetadataException(
"given seriesPath:{" + singlePath.getFullPath() + "} don't exist in metadata");
Expand Down Expand Up @@ -154,6 +155,7 @@ public InOperator copy() {
ret.tokenSymbol = tokenSymbol;
ret.isLeaf = isLeaf;
ret.isSingle = isSingle;
ret.pathSet = pathSet;
return ret;
}

Expand Down
69 changes: 50 additions & 19 deletions server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
View file
Expand Up @@ -212,8 +212,13 @@ public PhysicalPlan transformToPhysicalPlan(Operator operator)
}
}

protected TSDataType getSeriesType(String path) throws MetadataException {
return SchemaUtils.getSeriesType(path);
protected List<TSDataType> getSeriesTypes(List<String> paths,
String aggregation) throws MetadataException {
return SchemaUtils.getSeriesTypesByString(paths, aggregation);
}

protected List<TSDataType> getSeriesTypes(List<Path> paths) throws MetadataException {
return SchemaUtils.getSeriesTypesByPath(paths);
}

private PhysicalPlan transformQuery(QueryOperator queryOperator)
Expand Down Expand Up @@ -304,22 +309,23 @@ private PhysicalPlan transformQuery(QueryOperator queryOperator)
}
}

for (String pathStr : actualPaths) {
String aggregation = originAggregations != null && !originAggregations.isEmpty() ?
originAggregations.get(i) : null;
List<TSDataType> dataTypes = getSeriesTypes(actualPaths, aggregation);
for (int pathIdx = 0; pathIdx < actualPaths.size(); pathIdx++) {
String pathStr = actualPaths.get(pathIdx);
Path path = new Path(pathStr);

// check datatype consistency
// a example of inconsistency: select s0 from root.sg1.d1, root.sg2.d3 align by device,
// while root.sg1.d1.s0 is INT32 and root.sg2.d3.s0 is FLOAT.
String pathForDataType;
String measurementChecked;
if (originAggregations != null && !originAggregations.isEmpty()) {
pathForDataType = originAggregations.get(i) + "(" + path.getFullPath() + ")";
measurementChecked = originAggregations.get(i) + "(" + path.getMeasurement() + ")";
} else {
pathForDataType = path.getFullPath();
measurementChecked = path.getMeasurement();
}
TSDataType dataType = getSeriesType(pathForDataType);
TSDataType dataType = dataTypes.get(pathIdx);
if (measurementDataTypeMap.containsKey(measurementChecked)) {
if (!dataType.equals(measurementDataTypeMap.get(measurementChecked))) {
throw new QueryProcessException(
Expand Down Expand Up @@ -392,8 +398,18 @@ private PhysicalPlan transformQuery(QueryOperator queryOperator)
FilterOperator filterOperator = queryOperator.getFilterOperator();

if (filterOperator != null) {
IExpression expression = filterOperator.transformToExpression();
((RawDataQueryPlan) queryPlan).setExpression(expression);
List<Path> filterPaths = new ArrayList<>(filterOperator.getPathSet());
try {
List<TSDataType> seriesTypes = getSeriesTypes(filterPaths);
HashMap<Path, TSDataType> pathTSDataTypeHashMap = new HashMap<>();
for (int i = 0; i < filterPaths.size(); i++) {
pathTSDataTypeHashMap.put(filterPaths.get(i), seriesTypes.get(i));
}
IExpression expression = filterOperator.transformToExpression(pathTSDataTypeHashMap);
((RawDataQueryPlan) queryPlan).setExpression(expression);
} catch (MetadataException e) {
throw new LogicalOptimizeException(e);
}
}
}
try {
Expand All @@ -416,11 +432,23 @@ private Map<String, IExpression> concatFilterByDevice(List<String> devices,
FilterOperator operator)
throws QueryProcessException {
Map<String, IExpression> deviceToFilterMap = new HashMap<>();
Set<Path> filterPaths = new HashSet<>();
for (String device : devices) {
FilterOperator newOperator = operator.copy();
concatFilterPath(device, newOperator);

deviceToFilterMap.put(device, newOperator.transformToExpression());
concatFilterPath(device, newOperator, filterPaths);
// transform to a list so it can be indexed
List<Path> filterPathList = new ArrayList<>(filterPaths);
try {
List<TSDataType> seriesTypes = getSeriesTypes(filterPathList);
Map<Path, TSDataType> pathTSDataTypeHashMap = new HashMap<>();
for (int i = 0; i < filterPathList.size(); i++) {
pathTSDataTypeHashMap.put(filterPathList.get(i), seriesTypes.get(i));
}
deviceToFilterMap.put(device, newOperator.transformToExpression(pathTSDataTypeHashMap));
filterPaths.clear();
} catch (MetadataException e) {
throw new QueryProcessException(e);
}
}

return deviceToFilterMap;
Expand All @@ -442,10 +470,11 @@ private List<String> removeStarsInDeviceWithUnique(List<Path> paths)
return retDevices;
}

private void concatFilterPath(String prefix, FilterOperator operator) {
private void concatFilterPath(String prefix, FilterOperator operator,
Set<Path> filterPaths) {
if (!operator.isLeaf()) {
for (FilterOperator child : operator.getChildren()) {
concatFilterPath(prefix, child);
concatFilterPath(prefix, child, filterPaths);
}
return;
}
Expand All @@ -454,20 +483,22 @@ private void concatFilterPath(String prefix, FilterOperator operator) {

// do nothing in the cases of "where time > 5" or "where root.d1.s1 > 5"
if (SQLConstant.isReservedPath(filterPath) || filterPath.startWith(SQLConstant.ROOT)) {
filterPaths.add(filterPath);
return;
}

Path concatPath = Path.addPrefixPath(filterPath, prefix);
filterPaths.add(concatPath);
basicOperator.setSinglePath(concatPath);
}

private void generateDataTypes(QueryPlan queryPlan) throws MetadataException {
List<Path> paths = queryPlan.getPaths();
List<TSDataType> dataTypes = new ArrayList<>(paths.size());
for (Path path : paths) {
TSDataType seriesType = getSeriesType(path.toString());
dataTypes.add(seriesType);
queryPlan.addTypeMapping(path, seriesType);
List<TSDataType> dataTypes = getSeriesTypes(paths);
for (int i = 0; i < paths.size(); i++) {
Path path = paths.get(i);
TSDataType dataType = dataTypes.get(i);
queryPlan.addTypeMapping(path, dataType);
}
queryPlan.setDataTypes(dataTypes);
}
Expand Down

0 comments on commit 4d62652

Please sign in to comment.