diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 415d7c7f557a..51ead6ee79d6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -104,9 +104,11 @@ import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; import org.apache.hadoop.hive.metastore.model.MWMResourcePlan; import org.apache.hadoop.hive.metastore.parser.ExpressionTree; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Condition; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.MultiAndLeafNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; @@ -1368,6 +1370,9 @@ private static String generateSqlFilter(String catName, String dbName, String ta PartitionFilterGenerator visitor = new PartitionFilterGenerator( catName, dbName, tableName, partitionKeys, params, joins, dbHasJoinCastBug, defaultPartName, dbType, schema); + TreeNode flattened = PartFilterExprUtil.flattenAndExpressions(tree.getRoot()); + tree.setRoot(flattened); + tree.accept(visitor); if (visitor.filterBuffer.hasError()) { LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage()); @@ -1422,14 +1427,6 @@ private static enum FilterType { this.clazz = clazz; } - public Set getType() { - return colTypes; - } - - public Class getClazz() { - return clazz; - } - public static FilterType fromType(String colTypeStr) { for (FilterType filterType : FilterType.values()) { if (filterType.colTypes.contains(colTypeStr)) { @@ -1451,24 +1448,69 @@ public static FilterType fromClass(Object value){ @Override public void visit(LeafNode node) throws MetaException { - int partColCount = partitionKeys.size(); - int partColIndex = LeafNode.getPartColIndexForFilter(node.keyName, partitionKeys, filterBuffer); + String filter = visitCondition(node.getCondition(), true); if (filterBuffer.hasError()) { return; } + filterBuffer.append("(" + filter + ")"); + } + + @Override + public void visit(MultiAndLeafNode node) throws MetaException { + StringBuilder filterBuilder = new StringBuilder(); + List partValues = new ArrayList<>(Collections.nCopies(partitionKeys.size(), null)); + boolean hasEqualCondition = false; + for (Condition condition : node.getConditions()) { + boolean isEqual = Operator.isEqualOperator(condition.getOperator()); + if (isEqual) { + hasEqualCondition = true; + int partColIndex = getPartColIndexForFilter(condition.getKeyName(), partitionKeys, filterBuffer); + if (filterBuffer.hasError()) { + return; + } + String partValue = partValues.get(partColIndex); + String nodeValueStr = condition.getValue().toString(); + if (partValue != null && !partValue.equals(nodeValueStr)) { + // Conflicting equal conditions for the same partition key - the filter is unsatisfiable. + filterBuffer.append("(1 = 0)"); + return; + } + partValues.set(partColIndex, nodeValueStr); + } + if (!filterBuilder.isEmpty()) { + filterBuilder.append(" and "); + } + filterBuilder.append(visitCondition(condition, !isEqual)); + } + // Concatenate equality conditions to match a longer index prefix. + if (hasEqualCondition) { + String partName = Warehouse.makePartName(partitionKeys, partValues, "%"); + filterBuilder.append(" and " + PARTITIONS + ".\"PART_NAME\" like ?"); + params.add(partName); + } + + filterBuffer.append("(" + filterBuilder.toString() + ")"); + } + + private String visitCondition(Condition condition, boolean addPartNameFilter) throws MetaException { + int partColIndex = getPartColIndexForFilter(condition.getKeyName(), partitionKeys, filterBuffer); + if (filterBuffer.hasError()) { + return null; + } + FieldSchema partCol = partitionKeys.get(partColIndex); String colTypeStr = ColumnType.getTypeName(partCol.getType()); FilterType colType = FilterType.fromType(colTypeStr); if (colType == FilterType.Invalid) { filterBuffer.setError("Filter pushdown not supported for type " + colTypeStr); - return; + return null; } - FilterType valType = FilterType.fromClass(node.value); - Object nodeValue = node.value; + Object nodeValue = condition.getValue(); + FilterType valType = FilterType.fromClass(nodeValue); if (valType == FilterType.Invalid) { - filterBuffer.setError("Filter pushdown not supported for value " + node.value.getClass()); - return; + filterBuffer.setError("Filter pushdown not supported for value " + nodeValue.getClass()); + return null; } String nodeValue0 = "?"; @@ -1487,7 +1529,7 @@ public void visit(LeafNode node) throws MetaException { } else if (colType == FilterType.Timestamp) { if (dbType.isDERBY() || dbType.isMYSQL()) { filterBuffer.setError("Filter pushdown on timestamp not supported for " + dbType.dbType); - return; + return null; } try { MetaStoreUtils.convertStringToTimestamp((String) nodeValue); @@ -1506,7 +1548,7 @@ public void visit(LeafNode node) throws MetaException { // to be coerced?). Let the expression evaluation sort this one out, not metastore. filterBuffer.setError("Cannot push down filter for " + colTypeStr + " column and value " + nodeValue.getClass()); - return; + return null; } if (joins.isEmpty()) { @@ -1514,7 +1556,7 @@ public void visit(LeafNode node) throws MetaException { // joining multiple times for one column (if there are several filters on it), we will // keep numCols elements in the list, one for each column; we will fill it with nulls, // put each join at a corresponding index when necessary, and remove nulls in the end. - for (int i = 0; i < partColCount; ++i) { + for (int i = 0; i < partitionKeys.size(); ++i) { joins.add(null); } } @@ -1527,7 +1569,8 @@ public void visit(LeafNode node) throws MetaException { // Build the filter and add parameters linearly; we are traversing leaf nodes LTR. String tableValue = "\"FILTER" + partColIndex + "\".\"PART_KEY_VAL\""; - if (node.isReverseOrder && nodeValue != null) { + boolean isReverseOrder = condition.isReverseOrder(); + if (isReverseOrder && nodeValue != null) { params.add(nodeValue); } String tableColumn = tableValue; @@ -1559,22 +1602,23 @@ public void visit(LeafNode node) throws MetaException { tableValue += " then " + tableValue0 + " else null end)"; } - if (!node.isReverseOrder && nodeValue != null) { + if (!isReverseOrder && nodeValue != null) { params.add(nodeValue); } // The following syntax is required for using LIKE clause wildcards '_' and '%' as literals. - if (node.operator == Operator.LIKE) { + Operator operator = condition.getOperator(); + if (operator == Operator.LIKE) { nodeValue0 = nodeValue0 + " ESCAPE '\\' "; } - String filter = node.isReverseOrder - ? nodeValue0 + " " + node.operator.getSqlOp() + " " + tableValue - : tableValue + " " + node.operator.getSqlOp() + " " + nodeValue0; + String filter = isReverseOrder + ? nodeValue0 + " " + operator.getSqlOp() + " " + tableValue + : tableValue + " " + operator.getSqlOp() + " " + nodeValue0; // For equals and not-equals filter, we can add partition name filter to improve performance. - boolean isOpEquals = Operator.isEqualOperator(node.operator); - boolean isOpNotEqual = Operator.isNotEqualOperator(node.operator); - String nodeValueStr = node.value.toString(); - if (StringUtils.isNotEmpty(nodeValueStr) && (isOpEquals || isOpNotEqual)) { + boolean isOpEquals = Operator.isEqualOperator(operator); + boolean isOpNotEqual = Operator.isNotEqualOperator(operator); + String nodeValueStr = condition.getValue().toString(); + if (addPartNameFilter && StringUtils.isNotEmpty(nodeValueStr) && (isOpEquals || isOpNotEqual)) { Map partKeyToVal = new HashMap<>(); partKeyToVal.put(partCol.getName(), nodeValueStr); String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false); @@ -1583,6 +1627,7 @@ public void visit(LeafNode node) throws MetaException { // match PART_NAME by like clause. escapedNameFragment += "%"; } + int partColCount = partitionKeys.size(); if (colType != FilterType.Date && partColCount == 1) { // Case where partition column type is not date and there is no other partition columns params.add(escapedNameFragment); @@ -1604,8 +1649,7 @@ public void visit(LeafNode node) throws MetaException { filter += " and " + PARTITIONS + ".\"PART_NAME\"" + (isOpEquals ? " like ? " : " not like ? "); } } - - filterBuffer.append("(" + filter + ")"); + return filter; } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartFilterExprUtil.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartFilterExprUtil.java index 6cd48d53f4bf..8d3cbde007b3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartFilterExprUtil.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartFilterExprUtil.java @@ -18,15 +18,24 @@ package org.apache.hadoop.hive.metastore; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.BaseLeafNode; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Condition; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.MultiAndLeafNode; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.PartFilterParser; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree; /** * Utility functions for working with partition filter expressions @@ -116,4 +125,59 @@ private static ExpressionTree makeExpressionTree(String filter) throws MetaExcep public static ExpressionTree parseFilterTree(String filter) throws MetaException { return PartFilterParser.parseFilter(filter); } + + public static TreeNode buildTreeFromNodes(List nodes, LogicalOperator operator) { + // The 'nodes' list is expected to have at least one element. + // If the list if empty, the lexer parse would have failed. + assert !nodes.isEmpty() ; + if (nodes.size() == 1) { + return nodes.get(0); + } + TreeNode root = new TreeNode(nodes.get(0), operator, nodes.get(1)); + for (int i = 2; i < nodes.size(); ++i) { + TreeNode tmp = new TreeNode(root, operator, nodes.get(i)); + root = tmp; + } + return root; + } + + /** + * Flatten all AND-connected leaf nodes in the given expression tree + * into MultiAndLeafNodes for more efficient evaluation. + */ + public static TreeNode flattenAndExpressions(TreeNode node) { + if (node == null || node instanceof BaseLeafNode) { + return node; + } + TreeNode left = flattenAndExpressions(node.getLhs()); + TreeNode right = flattenAndExpressions(node.getRhs()); + if (node.getAndOr() == LogicalOperator.AND) { + List flatConditions = new ArrayList<>(); + List orNodes = new ArrayList<>(); + flattenConditions(left, flatConditions, orNodes); + flattenConditions(right, flatConditions, orNodes); + if (!flatConditions.isEmpty()) { + TreeNode andNode = flatConditions.size() == 1 ? + new LeafNode(flatConditions.get(0)) : + new MultiAndLeafNode(flatConditions); + orNodes.add(andNode); + } + return buildTreeFromNodes(orNodes, LogicalOperator.AND); + } + return new TreeNode(left, node.getAndOr(), right); + } + + private static void flattenConditions(TreeNode node, List flatConditions, List orNodes) { + if (node == null) { + return; + } + if (node instanceof BaseLeafNode leaf) { + flatConditions.addAll(leaf.getConditions()); + } else if (node.getAndOr() == LogicalOperator.AND) { + flattenConditions(node.getLhs(), flatConditions, orNodes); + flattenConditions(node.getRhs(), flatConditions, orNodes); + } else { + orNodes.add(node); + } + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java index b6baa3333c1a..7629a11a1085 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java @@ -19,11 +19,11 @@ import java.sql.Timestamp; import java.sql.Date; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Stack; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.ColumnType; @@ -137,9 +137,28 @@ protected void beginTreeNode(TreeNode node) throws MetaException {} protected void midTreeNode(TreeNode node) throws MetaException {} protected void endTreeNode(TreeNode node) throws MetaException {} protected void visit(LeafNode node) throws MetaException {} + protected void visit(MultiAndLeafNode node) throws MetaException {} protected boolean shouldStop() { return false; } + + /** + * Get partition column index in the table partition column list that + * corresponds to the key that is being filtered on by this tree node. + * @param partitionKeys list of partition keys. + * @param filterBuilder filter builder used to report error, if any. + * @return The index. + */ + protected int getPartColIndexForFilter(String partitionKeyName, + List partitionKeys, FilterBuilder filterBuilder) throws MetaException { + int partitionColumnIndex = Iterables.indexOf(partitionKeys, key -> partitionKeyName.equalsIgnoreCase(key.getName())); + if( partitionColumnIndex < 0) { + filterBuilder.setError("Specified key <" + partitionKeyName + + "> is not a partitioning key for the table"); + return -1; + } + return partitionColumnIndex; + } } /** @@ -239,49 +258,113 @@ public String toString() { } } - /** - * The Class representing the leaf level nodes in the ExpressionTree. - */ - public static class LeafNode extends TreeNode { - public String keyName; - public Operator operator; + public static class Condition { + private final String keyName; + private final Operator operator; /** * Constant expression side of the operator. Can currently be a String or a Long. */ - public Object value; - public boolean isReverseOrder = false; + private final Object value; + private final boolean isReverseOrder; - @Override - protected void accept(TreeVisitor visitor) throws MetaException { - visitor.visit(this); + public Condition(String keyName, Operator operator, Object value) { + this(keyName, operator, value, false); + } + + public Condition(String keyName, Operator operator, Object value, boolean isReverseOrder) { + this.keyName = keyName; + this.operator = operator; + this.value = value; + this.isReverseOrder = isReverseOrder; + } + + public String getKeyName() { + return keyName; + } + + public Operator getOperator() { + return operator; + } + + public Object getValue() { + return value; + } + + public boolean isReverseOrder() { + return isReverseOrder; } @Override public String toString() { - return "LeafNode{" + + return "{" + "keyName='" + keyName + '\'' + ", operator='" + operator + '\'' + ", value=" + value + (isReverseOrder ? ", isReverseOrder=true" : "") + '}'; } + } - /** - * Get partition column index in the table partition column list that - * corresponds to the key that is being filtered on by this tree node. - * @param partitionKeys list of partition keys. - * @param filterBuilder filter builder used to report error, if any. - * @return The index. - */ - public static int getPartColIndexForFilter(String partitionKeyName, - List partitionKeys, FilterBuilder filterBuilder) throws MetaException { - int partitionColumnIndex = Iterables.indexOf(partitionKeys, key -> partitionKeyName.equalsIgnoreCase(key.getName())); - if( partitionColumnIndex < 0) { - filterBuilder.setError("Specified key <" + partitionKeyName + - "> is not a partitioning key for the table"); - return -1; - } - return partitionColumnIndex; + /** + * The Class representing the leaf level nodes in the ExpressionTree. + */ + public static abstract class BaseLeafNode extends TreeNode { + public abstract List getConditions(); + } + + /** + * Leaf node with a single condition. + */ + public static class LeafNode extends BaseLeafNode { + private final Condition condition; + + public LeafNode(Condition condition) { + this.condition = condition; + } + + @Override + public List getConditions() { + return Collections.singletonList(condition); + } + + public Condition getCondition() { + return condition; + } + + @Override + protected void accept(TreeVisitor visitor) throws MetaException { + visitor.visit(this); + } + + @Override + public String toString() { + return "LeafNode" + condition.toString(); + } + } + + /** + * Leaf node with multiple AND-connected conditions. + */ + public static class MultiAndLeafNode extends BaseLeafNode { + private final List conditions; + + public MultiAndLeafNode(List conditions) { + this.conditions = conditions; + } + + @Override + public List getConditions() { + return conditions; + } + + @Override + protected void accept(TreeVisitor visitor) throws MetaException { + visitor.visit(this); + } + + @Override + public String toString() { + return "MultiAndLeafNode" + conditions.toString(); } } @@ -339,12 +422,12 @@ protected void endTreeNode(TreeNode node) throws MetaException { @Override protected void visit(LeafNode node) throws MetaException { beforeParsing(); - keyName = node.keyName; - operator = node.operator; - value = node.value; - isReverseOrder = node.isReverseOrder; - if (node.keyName.startsWith(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS) - && DatabaseProduct.isDerbyOracle() && node.operator == Operator.EQUALS) { + keyName = node.getCondition().getKeyName(); + operator = node.getCondition().getOperator(); + value = node.getCondition().getValue(); + isReverseOrder = node.getCondition().isReverseOrder(); + if (keyName.startsWith(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS) + && DatabaseProduct.isDerbyOracle() && operator == Operator.EQUALS) { // Rewrite the EQUALS operator to LIKE operator = Operator.LIKE; } @@ -437,7 +520,7 @@ private void generateJDOFilterGeneral(Map params, private void generateJDOFilterOverPartitions(Configuration conf, Map params, FilterBuilder filterBuilder, List partitionKeys) throws MetaException { int partitionColumnCount = partitionKeys.size(); - int partitionColumnIndex = LeafNode.getPartColIndexForFilter(keyName, partitionKeys, filterBuilder); + int partitionColumnIndex = getPartColIndexForFilter(keyName, partitionKeys, filterBuilder); if (filterBuilder.hasError()) return; boolean canPushDownIntegral = @@ -619,11 +702,6 @@ private static void makeFilterForEquals(String keyName, String value, String par */ private TreeNode root = null; - /** - * The node stack used to keep track of the tree nodes during parsing. - */ - private final Stack nodeStack = new Stack<>(); - public TreeNode getRoot() { return this.root; } @@ -631,32 +709,4 @@ public TreeNode getRoot() { public void setRoot(TreeNode tn) { this.root = tn; } - - - /** - * Adds a intermediate node of either type(AND/OR). Pops last two nodes from - * the stack and sets them as children of the new node and pushes itself - * onto the stack. - * @param andOr the operator type - */ - public void addIntermediateNode(LogicalOperator andOr) { - - TreeNode rhs = nodeStack.pop(); - TreeNode lhs = nodeStack.pop(); - TreeNode newNode = new TreeNode(lhs, andOr, rhs); - nodeStack.push(newNode); - root = newNode; - } - - /** - * Adds a leaf node, pushes the new node onto the stack. - * @param newNode the new node - */ - public void addLeafNode(LeafNode newNode) { - if( root == null ) { - root = newNode; - } - nodeStack.push(newNode); - } - } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/PartFilterVisitor.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/PartFilterVisitor.java index 28c61ba1c2a9..673db04fffcf 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/PartFilterVisitor.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/PartFilterVisitor.java @@ -26,12 +26,14 @@ import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.TerminalNode; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Condition; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import static org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; -import static org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; -import static org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; -import static org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; +import static org.apache.hadoop.hive.metastore.PartFilterExprUtil.buildTreeFromNodes; public class PartFilterVisitor extends PartitionFilterBaseVisitor { @@ -72,20 +74,6 @@ public TreeNode visitAndExpression(PartitionFilterParser.AndExpressionContext ct return buildTreeFromNodes(nodes, LogicalOperator.AND); } - private TreeNode buildTreeFromNodes(List nodes, LogicalOperator operator) { - // The 'nodes' list is expected to have at least one element. - // If the list if empty, the lexer parse would have failed. - if (nodes.size() == 1) { - return nodes.get(0); - } - TreeNode root = new TreeNode(nodes.get(0), operator, nodes.get(1)); - for (int i = 2; i < nodes.size(); ++i) { - TreeNode tmp = new TreeNode(root, operator, nodes.get(i)); - root = tmp; - } - return root; - } - @Override public TreeNode visitExpression(PartitionFilterParser.ExpressionContext ctx) { if (ctx.orExpression() != null) { @@ -96,38 +84,34 @@ public TreeNode visitExpression(PartitionFilterParser.ExpressionContext ctx) { @Override public TreeNode visitComparison(PartitionFilterParser.ComparisonContext ctx) { - LeafNode leafNode = new LeafNode(); - leafNode.keyName = (String) visit(ctx.key); - leafNode.value = visit(ctx.value); - leafNode.operator = visitComparisonOperator(ctx.comparisonOperator()); - return leafNode; + String keyName = (String) visit(ctx.key); + Object value = visit(ctx.value); + Operator operator = visitComparisonOperator(ctx.comparisonOperator()); + return new LeafNode(new Condition(keyName, operator, value)); } @Override public Object visitReverseComparison(PartitionFilterParser.ReverseComparisonContext ctx) { - LeafNode leafNode = new LeafNode(); - leafNode.keyName = (String) visit(ctx.key); - leafNode.value = visit(ctx.value); - leafNode.operator = visitComparisonOperator(ctx.comparisonOperator()); - leafNode.isReverseOrder = true; - return leafNode; + String keyName = (String) visit(ctx.key); + Object value = visit(ctx.value); + Operator operator = visitComparisonOperator(ctx.comparisonOperator()); + return new LeafNode(new Condition(keyName, operator, value, true)); } @Override public TreeNode visitBetweenCondition(PartitionFilterParser.BetweenConditionContext ctx) { - LeafNode left = new LeafNode(); - LeafNode right = new LeafNode(); - left.keyName = right.keyName = (String) visit(ctx.key); - left.value = visit(ctx.lower); - right.value = visit(ctx.upper); + String keyName = (String) visit(ctx.key); + Object leftValue = visit(ctx.lower); + Object rightValue = visit(ctx.upper); boolean isPositive = ctx.NOT() == null; - left.operator = isPositive ? Operator.GREATERTHANOREQUALTO : Operator.LESSTHAN; - right.operator = isPositive ? Operator.LESSTHANOREQUALTO : Operator.GREATERTHAN; + Operator leftOperator = isPositive ? Operator.GREATERTHANOREQUALTO : Operator.LESSTHAN; + Operator rightOperator = isPositive ? Operator.LESSTHANOREQUALTO : Operator.GREATERTHAN; LogicalOperator rootOperator = isPositive ? LogicalOperator.AND : LogicalOperator.OR; - TreeNode treeNode = new TreeNode(left, rootOperator, right); - return treeNode; + LeafNode left = new LeafNode(new Condition(keyName, leftOperator, leftValue)); + LeafNode right = new LeafNode(new Condition(keyName, rightOperator, rightValue)); + return new TreeNode(left, rootOperator, right); } @Override @@ -141,11 +125,9 @@ public TreeNode visitInCondition(PartitionFilterParser.InConditionContext ctx) { private TreeNode buildInCondition(String keyName, List values, boolean isPositive) { List nodes = values.stream() .map(value -> { - LeafNode leafNode = new LeafNode(); - leafNode.keyName = keyName; - leafNode.value = value; - leafNode.operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; - return leafNode; }) + Operator operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; + Condition condition = new Condition(keyName, operator, value); + return new LeafNode(condition); }) .collect(Collectors.toList()); return buildTreeFromNodes(nodes, isPositive ? LogicalOperator.OR : LogicalOperator.AND); } @@ -164,10 +146,10 @@ public TreeNode visitMultiColInExpression(PartitionFilterParser.MultiColInExpres } List nodes = new ArrayList<>(struct.size()); for (int j = 0; j < struct.size(); ++j) { - LeafNode leafNode = new LeafNode(); - leafNode.keyName = keyNames.get(j); - leafNode.value = struct.get(j); - leafNode.operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; + String keyName = keyNames.get(j); + Object value = struct.get(j); + Operator operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; + LeafNode leafNode = new LeafNode(new Condition(keyName, operator, value)); nodes.add(leafNode); } treeNodes.add(buildTreeFromNodes(nodes, isPositive ? LogicalOperator.AND : LogicalOperator.OR)); @@ -327,13 +309,10 @@ private TreeNode negateTree(TreeNode node) { } private LeafNode negateLeafNode(LeafNode leaf) { - LeafNode negatedLeaf = new LeafNode(); - negatedLeaf.keyName = leaf.keyName; - - // Invert the operator for the leaf node - negatedLeaf.operator = invertOperator(leaf.operator); - negatedLeaf.value = leaf.value; - return negatedLeaf; + Condition condition = leaf.getCondition(); + Operator invertedOperator = invertOperator(condition.getOperator()); + Condition negatedCondition = new Condition(condition.getKeyName(), invertedOperator, condition.getValue()); + return new LeafNode(negatedCondition); } @Override diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestPartFilterExprUtil.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestPartFilterExprUtil.java index f4a7cfd58dac..56ab4168422b 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestPartFilterExprUtil.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestPartFilterExprUtil.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.parser.ExpressionTree; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -34,21 +35,29 @@ public class TestPartFilterExprUtil { @Test public void testAndOrPrecedence() throws MetaException { checkFilter("dt=10 or dt=20 and dt=30 and dt=40 or dt=50 or dt=60 and dt=70", - "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=40}}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=40}}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=20}, {keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=40}]}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=60}, {keyName='dt', operator='=', value=70}]}"); checkFilter("dt=10 or dt=20 and (dt=30 and dt=40 or dt=50) or dt=60 and dt=70", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=40}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=40}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=MultiAndLeafNode[{keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=40}], andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=20}}}, andOr='OR', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=60}, {keyName='dt', operator='=', value=70}]}"); checkFilter("dt=10 or dt=20 and dt=30 and (dt=40 or dt=50 or dt=60) and dt=70", - "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=20}, {keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=70}]}}"); checkFilter("(dt=10 or dt=20) and dt=30 and (dt=40 or dt=50) or dt=60 and dt=70", - "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=60}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='OR', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=60}, {keyName='dt', operator='=', value=70}]}"); checkFilter("(dt=10 or dt=20) and dt=30 and ((dt=40 or dt=50) or dt=60) and dt=70", - "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}"); + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}", + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=70}]}"); checkFilter("(dt=10 or dt=20) and (dt=30 and ((dt=40 or dt=50) or dt=60) and dt=70)", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=70}]}"); checkFilter("dt=10 or dt=20 and (dt=30 and ((dt=40 or dt=50) or dt=60) and dt=70)", - "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=30}, andOr='AND', rhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}}", + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=20}, {keyName='dt', operator='=', value=30}, {keyName='dt', operator='=', value=70}]}}"); checkFilter("dt=10 or (dt=20 and dt=30 and (dt=40 or dt=50) or dt=60) and dt=70", - "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=20}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=30}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}", + "TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=10}, andOr='OR', rhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='dt', operator='=', value=40}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=50}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='dt', operator='=', value=20}, {keyName='dt', operator='=', value=30}]}, andOr='OR', rhs=LeafNode{keyName='dt', operator='=', value=60}}, andOr='AND', rhs=LeafNode{keyName='dt', operator='=', value=70}}}"); } @Test @@ -56,7 +65,8 @@ public void testExpressionCombination() throws MetaException { checkFilter("(a) in (10, 20) and a != 30", "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='a', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='=', value=20}}, andOr='AND', rhs=LeafNode{keyName='a', operator='!=', value=30}}"); checkFilter("(a) in (10, 20) and a between 10 and 15", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='a', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='a', operator='>=', value=10}, andOr='AND', rhs=LeafNode{keyName='a', operator='<=', value=15}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='a', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='a', operator='>=', value=10}, andOr='AND', rhs=LeafNode{keyName='a', operator='<=', value=15}}}", + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='a', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='=', value=20}}, andOr='AND', rhs=MultiAndLeafNode[{keyName='a', operator='>=', value=10}, {keyName='a', operator='<=', value=15}]}"); checkFilter("(a) in (10, 20) and a not between 10 and 15", "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='a', operator='=', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='=', value=20}}, andOr='AND', rhs=TreeNode{lhs=LeafNode{keyName='a', operator='<', value=10}, andOr='OR', rhs=LeafNode{keyName='a', operator='>', value=15}}}"); checkFilter("(a) in (10, 20) and a not between 10 and 15 and b < 10", @@ -92,13 +102,15 @@ public void testSingleColInExpressionWhenDateLiteralTypeIsSpecified() throws Met @Test public void testMultiColInExpressionWhenDateLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct(2000-05-08, 2001-04-08), struct(2000-05-09, 2001-04-09))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08}, {keyName='ds2', operator='=', value=2001-04-08}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09}, {keyName='ds2', operator='=', value=2001-04-09}]}"); } @Test public void testMultiColInExpressionWhenDateLiteralTypeIsSpecified() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct(DATE'2000-05-08',DATE'2001-04-08'), struct(DATE'2000-05-09',DATE'2001-04-09'))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08}, {keyName='ds2', operator='=', value=2001-04-08}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09}, {keyName='ds2', operator='=', value=2001-04-09}]}"); } @Test @@ -116,61 +128,71 @@ public void testSingleColInExpressionWhenTimestampLiteralTypeIsSpecified() throw @Test public void testMultiColInExpressionWhenTimestampLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct(2000-05-08 01:00:00, 2001-04-08 01:00:00), struct(2000-05-09 01:00:00, 2001-04-09 01:00:00))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, {keyName='ds2', operator='=', value=2001-04-08 01:00:00}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, {keyName='ds2', operator='=', value=2001-04-09 01:00:00}]}"); } @Test public void testMultiColInExpressionWhenTimestampLiteralTypeIsSpecified() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct(TIMESTAMP'2000-05-08 01:00:00',TIMESTAMP'2001-04-08 01:00:00'), struct(TIMESTAMP'2000-05-09 01:00:00',TIMESTAMP'2001-04-09 01:00:00'))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, {keyName='ds2', operator='=', value=2001-04-08 01:00:00}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, {keyName='ds2', operator='=', value=2001-04-09 01:00:00}]}"); } @Test public void testBetweenExpressionWhenDateLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("(j BETWEEN 1990-11-10 AND 1990-11-11)", - "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}", + "MultiAndLeafNode[{keyName='j', operator='>=', value=1990-11-10}, {keyName='j', operator='<=', value=1990-11-11}]"); } @Test public void testBetweenExpressionWhenDateLiteralTypeIsSpecified() throws MetaException { checkFilter("(j BETWEEN DATE'1990-11-10' AND DATE'1990-11-11')", - "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}", + "MultiAndLeafNode[{keyName='j', operator='>=', value=1990-11-10}, {keyName='j', operator='<=', value=1990-11-11}]"); } @Test public void testBetweenExpressionWhenTimestampLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("dt BETWEEN 2000-01-01 01:00:00 AND 2000-01-01 01:42:00)", - "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}", + "MultiAndLeafNode[{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, {keyName='dt', operator='<=', value=2000-01-01 01:42:00}]"); } @Test public void testBetweenExpressionWhenTimestampLiteralTypeIsSpecified() throws MetaException { checkFilter("dt BETWEEN TIMESTAMP'2000-01-01 01:00:00' AND TIMESTAMP'2000-01-01 01:42:00')", - "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}", + "MultiAndLeafNode[{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, {keyName='dt', operator='<=', value=2000-01-01 01:42:00}]"); } @Test public void testBinaryExpressionWhenDateLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("(j = 1990-11-10 or j = 1990-11-11 and j = 1990-11-12)", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11}, {keyName='j', operator='=', value=1990-11-12}]}"); } @Test public void testBinaryExpressionWhenDateLiteralTypeIsSpecified() throws MetaException { checkFilter("(j = DATE'1990-11-10' or j = DATE'1990-11-11' and j = DATE'1990-11-12')", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11}, {keyName='j', operator='=', value=1990-11-12}]}"); } @Test public void testBinaryExpressionWhenTimeStampLiteralTypeIsNotSpecifiedNorQuoted() throws MetaException { checkFilter("(j = 1990-11-10 01:00:00 or j = 1990-11-11 01:00:24 and j = 1990-11-12 01:42:00)", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11 01:00:24}, {keyName='j', operator='=', value=1990-11-12 01:42:00}]}"); } @Test public void testBinaryExpressionWhenTimeStampLiteralTypeIsSpecified() throws MetaException { checkFilter("(j = TIMESTAMP'1990-11-10 01:00:00' or j = TIMESTAMP'1990-11-11 01:00:24' and j = TIMESTAMP'1990-11-12 01:42:00')", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11 01:00:24}, {keyName='j', operator='=', value=1990-11-12 01:42:00}]}"); } @@ -183,7 +205,8 @@ public void testSingleColInExpressionWithIntLiteral() throws MetaException { @Test public void testBetweenExpressionWithIntLiteral() throws MetaException { checkFilter("dt between 10 and 20", - "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=10}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=20}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=10}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=20}}", + "MultiAndLeafNode[{keyName='dt', operator='>=', value=10}, {keyName='dt', operator='<=', value=20}]"); } @Test @@ -213,19 +236,22 @@ public void testSingleColInExpressionWithStringLikeDate() throws MetaException { @Test public void testMultiColInExpressionWithDateLikeString() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct('2000-05-08','2001-04-08'), struct('2000-05-09','2001-04-09'))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08}, {keyName='ds2', operator='=', value=2001-04-08}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09}, {keyName='ds2', operator='=', value=2001-04-09}]}"); } @Test public void testBetweenExpressionWithStringLikeDate() throws MetaException { checkFilter("(j BETWEEN '1990-11-10' AND '1990-11-11')", - "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='>=', value=1990-11-10}, andOr='AND', rhs=LeafNode{keyName='j', operator='<=', value=1990-11-11}}", + "MultiAndLeafNode[{keyName='j', operator='>=', value=1990-11-10}, {keyName='j', operator='<=', value=1990-11-11}]"); } @Test public void testBinaryExpressionWithStringLikeDate() throws MetaException { checkFilter("(j = '1990-11-10' or j = '1990-11-11' and j = '1990-11-12')", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11}, {keyName='j', operator='=', value=1990-11-12}]}"); } @Test @@ -237,24 +263,34 @@ public void testSingleColInExpressionWithStringLikeTimestamp() throws MetaExcept @Test public void testMultiColInExpressionWithTimestampLikeString() throws MetaException { checkFilter("(struct(ds1,ds2)) IN (struct('2000-05-08 01:00:00','2001-04-08 01:00:00'), struct('2000-05-09 01:00:00','2001-04-09 01:00:00'))", - "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}"); + "TreeNode{lhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-08 01:00:00}}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, andOr='AND', rhs=LeafNode{keyName='ds2', operator='=', value=2001-04-09 01:00:00}}}", + "TreeNode{lhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-08 01:00:00}, {keyName='ds2', operator='=', value=2001-04-08 01:00:00}], andOr='OR', rhs=MultiAndLeafNode[{keyName='ds1', operator='=', value=2000-05-09 01:00:00}, {keyName='ds2', operator='=', value=2001-04-09 01:00:00}]}"); } @Test public void testBetweenExpressionWithStringLikeTimestamp() throws MetaException { checkFilter("dt BETWEEN '2000-01-01 01:00:00' AND '2000-01-01 01:42:00')", - "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}"); + "TreeNode{lhs=LeafNode{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, andOr='AND', rhs=LeafNode{keyName='dt', operator='<=', value=2000-01-01 01:42:00}}", + "MultiAndLeafNode[{keyName='dt', operator='>=', value=2000-01-01 01:00:00}, {keyName='dt', operator='<=', value=2000-01-01 01:42:00}]"); } @Test public void testBinaryExpressionWithStringLikeTimeStamp() throws MetaException { checkFilter("(j = '1990-11-10 01:00:00' or j = '1990-11-11 01:00:24' and j = '1990-11-12 01:42:00')", - "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}"); + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-11 01:00:24}, andOr='AND', rhs=LeafNode{keyName='j', operator='=', value=1990-11-12 01:42:00}}}", + "TreeNode{lhs=LeafNode{keyName='j', operator='=', value=1990-11-10 01:00:00}, andOr='OR', rhs=MultiAndLeafNode[{keyName='j', operator='=', value=1990-11-11 01:00:24}, {keyName='j', operator='=', value=1990-11-12 01:42:00}]}"); } private void checkFilter(String filter, String expectTreeString) throws MetaException { + checkFilter(filter, expectTreeString, expectTreeString); + } + + private void checkFilter(String filter, String expectTreeString, String expectedFlattenedTreeString) + throws MetaException { ExpressionTree expressionTree = PartFilterExprUtil.parseFilterTree(filter); assertThat(expressionTree.getRoot().toString(), is(expectTreeString)); + TreeNode flattened = PartFilterExprUtil.flattenAndExpressions(expressionTree.getRoot()); + assertThat(flattened.toString(), is(expectedFlattenedTreeString)); } @Test