Skip to content

Commit

Permalink
HIVE-24835: Replace HiveSubQueryFinder with RexUtil.SubQueryFinder (S…
Browse files Browse the repository at this point in the history 
…tamatis Zampetakis, reviewed by (Krisztian Kasa)

Change HiveSubQueryRemoveRule to avoid unnecessary matches in
non-relevant nodes.

Replace deprecated CalcitePlanner#hepPlan with purpose specific and more
readable CalcitePlanner#removeSubqueries.
  • Loading branch information
@zabetak
zabetak committed Mar 2, 2021
1 parent 4ef69f6 commit 6f2b888
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 101 deletions.
103 changes: 20 additions & 83 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,16 @@

import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rex.LogicVisitor;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexSubQuery;
Expand All @@ -45,15 +43,13 @@
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;

import com.google.common.collect.ImmutableList;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;

import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubqueryRuntimeException;
Expand Down Expand Up @@ -84,11 +80,22 @@
*/
public class HiveSubQueryRemoveRule extends RelOptRule {

public static RelOptRule forProject(HiveConf conf) {
return new HiveSubQueryRemoveRule(
RelOptRule.operandJ(HiveProject.class, null, RexUtil.SubQueryFinder::containsSubQuery, any()),
"SubQueryRemoveRule:Project", conf);
}

public static RelOptRule forFilter(HiveConf conf) {
return new HiveSubQueryRemoveRule(
RelOptRule.operandJ(HiveFilter.class, null, RexUtil.SubQueryFinder::containsSubQuery, any()),
"SubQueryRemoveRule:Filter", conf);
}

private final HiveConf conf;

public HiveSubQueryRemoveRule(HiveConf conf) {
super(operandJ(RelNode.class, null, HiveSubQueryFinder.RELNODE_PREDICATE, any()),
HiveRelFactories.HIVE_BUILDER, "SubQueryRemoveRule:Filter");
private HiveSubQueryRemoveRule(RelOptRuleOperand operand, String description, HiveConf conf) {
super(operand, HiveRelFactories.HIVE_BUILDER, description);
this.conf = conf;
}

Expand All @@ -98,8 +105,8 @@ public HiveSubQueryRemoveRule(HiveConf conf) {
new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null);

// if subquery is in FILTER
if (relNode instanceof Filter) {
final Filter filter = call.rel(0);
if (relNode instanceof HiveFilter) {
final HiveFilter filter = call.rel(0);
final RexSubQuery e = RexUtil.SubQueryFinder.find(filter.getCondition());
assert e != null;

Expand All @@ -108,9 +115,7 @@ public HiveSubQueryRemoveRule(HiveConf conf) {
builder.push(filter.getInput());
final int fieldCount = builder.peek().getRowType().getFieldCount();

assert (filter instanceof HiveFilter);
SubqueryConf subqueryConfig = filter.getCluster().getPlanner().
getContext().unwrap(SubqueryConf.class);
SubqueryConf subqueryConfig = filter.getCluster().getPlanner().getContext().unwrap(SubqueryConf.class);
boolean isCorrScalarQuery = subqueryConfig.getCorrScalarRexSQWithAgg().contains(e.rel);

final RexNode target =
Expand All @@ -121,9 +126,9 @@ public HiveSubQueryRemoveRule(HiveConf conf) {
builder.project(fields(builder, filter.getRowType().getFieldCount()));
RelNode newRel = builder.build();
call.transformTo(newRel);
} else if (relNode instanceof Project) {
} else if (relNode instanceof HiveProject) {
// if subquery is in PROJECT
final Project project = call.rel(0);
final HiveProject project = call.rel(0);
final RexSubQuery e = RexUtil.SubQueryFinder.find(project.getProjects());
assert e != null;

Expand Down Expand Up @@ -619,74 +624,6 @@ private static class ReplaceSubQueryShuttle extends RexShuttle {
}
}

// TODO:
// Following HiveSubQueryFinder has been copied from RexUtil::SubQueryFinder
// since there is BUG in there (CALCITE-1726).
// Once CALCITE-1726 is fixed we should get rid of the following code

/**
* Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if
* applied to an expression that contains a {@link RexSubQuery}.
*/
public static final class HiveSubQueryFinder extends RexVisitorImpl<Void> {
public static final HiveSubQueryFinder INSTANCE = new HiveSubQueryFinder();

/**
* Returns whether a {@link Project} contains a sub-query.
*/
public static final Predicate<RelNode> RELNODE_PREDICATE = new Predicate<RelNode>() {
@Override public boolean test(RelNode relNode) {
if (relNode instanceof Project) {
Project project = (Project) relNode;
for (RexNode node : project.getProjects()) {
try {
node.accept(INSTANCE);
} catch (Util.FoundOne e) {
return true;
}
}
return false;
} else if (relNode instanceof Filter) {
try {
((Filter) relNode).getCondition().accept(INSTANCE);
return false;
} catch (Util.FoundOne e) {
return true;
}
}
return false;
}
};

private HiveSubQueryFinder() {
super(true);
}

@Override public Void visitSubQuery(RexSubQuery subQuery) {
throw new Util.FoundOne(subQuery);
}

public static RexSubQuery find(Iterable<RexNode> nodes) {
for (RexNode node : nodes) {
try {
node.accept(INSTANCE);
} catch (Util.FoundOne e) {
return (RexSubQuery) e.getNode();
}
}
return null;
}

public static RexSubQuery find(RexNode node) {
try {
node.accept(INSTANCE);
return null;
} catch (Util.FoundOne e) {
return (RexSubQuery) e.getNode();
}
}
}

public static void subqueryRestriction(RelNode relNode) {
if (relNode instanceof HiveAggregate) {
HiveAggregate aggregate = (HiveAggregate) relNode;
Expand Down
27 changes: 9 additions & 18 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1870,8 +1870,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu
if (LOG.isDebugEnabled()) {
LOG.debug("Plan before removing subquery:\n" + RelOptUtil.toString(calciteGenPlan));
}
calciteGenPlan = hepPlan(calciteGenPlan, false, mdProvider.getMetadataProvider(), null,
HepMatchOrder.DEPTH_FIRST, new HiveSubQueryRemoveRule(conf));
calciteGenPlan = removeSubqueries(calciteGenPlan, mdProvider.getMetadataProvider());
if (LOG.isDebugEnabled()) {
LOG.debug("Plan just after removing subquery:\n" + RelOptUtil.toString(calciteGenPlan));
}
Expand Down Expand Up @@ -2611,23 +2610,15 @@ public void visit(RelNode node, int ordinal, RelNode parent) {
}

/**
* Run the HEP Planner with the given rule set.
*
* @param basePlan
* @param followPlanChanges
* @param mdProvider
* @param executorProvider
* @param order
* @param rules
* @return optimized RelNode
* Removes sub-queries (if present) from the specified query plan.
* @return a new query plan without subquery expressions.
*/
@Deprecated
private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges,
RelMetadataProvider mdProvider, RexExecutor executorProvider, HepMatchOrder order,
RelOptRule... rules) {
final HepProgramBuilder programBuilder = new HepProgramBuilder();
generatePartialProgram(programBuilder, followPlanChanges, order, rules);
return executeProgram(basePlan, programBuilder.build(), mdProvider, executorProvider);
private RelNode removeSubqueries(RelNode basePlan, RelMetadataProvider mdProvider) {
final HepProgramBuilder builder = new HepProgramBuilder();
builder.addMatchOrder(HepMatchOrder.DEPTH_FIRST);
builder.addRuleCollection(
ImmutableList.of(HiveSubQueryRemoveRule.forFilter(conf), HiveSubQueryRemoveRule.forProject(conf)));
return executeProgram(basePlan, builder.build(), mdProvider, null);
}

/**
Expand Down

0 comments on commit 6f2b888

Please sign in to comment.