diff --git a/core/src/test/resources/sql/hep.iq b/core/src/test/resources/sql/hep.iq new file mode 100644 index 000000000000..8c5b36fe9005 --- /dev/null +++ b/core/src/test/resources/sql/hep.iq @@ -0,0 +1,117 @@ +# hep.iq - hep tests can customizable optimization rules for hep planner +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This documents the required format for .iq files containing hep planner rule configurations +# +# SYNTAX: +# ------- +# !set hep-rules " +# +CoreRules.RULE_NAME1, +# -CoreRules.RULE_NAME2, +# +EnumerableRules.ENUMERABLE_RULE_NAME" +# +# RULES FORMAT SPECIFICATION: +# --------------------------- +# 1. Must begin with: !set hep-rules " (double quote) +# 2. Each rule must be on its own line. All lines except last must end with comma. +# The last rule line must close with " (double quote) on final line +# 3. Rules must start with - (remove) or + (add) +# +# RULE PROCESSING LOGIC: +# ---------------------- +# 1. Rule prefix interpretation: +# - CoreRules.RULE_NAME -> added/removed from hep program +# - EnumerableRules.RULE_NAME -> added/removed from volcano program +# - RULE_NAME (no prefix) -> treated as CoreRules, added/removed from hep program +# 2. Duplicate operations on the same rule: last operation wins +# Example: "+Rule1,+Rule1,-Rule1" results in rule being removed +# 3. Optimizer execution pipeline: +# The optimizer uses a multi-program architecture with the following execution order: +# - HEP program — executed by HepPlanner; initial rules set by CoreRules.* +# - Volcano program — executed by VolcanoPlanner; initial rules set by EnumerableRules.ENUMERABLE_RULES +# and modified by configurable EnumerableRules.* rule sets. +# - Calc program — executed by HepPlanner; initial rules set by RelOptRules.CALC_RULES (non-configurable) + +!use scott +!set outputformat mysql + +# The hep-rules test is functioning correctly. +!set hep-rules " ++CoreRules.PROJECT_FILTER_TRANSPOSE, ++CoreRules.UNION_FILTER_TO_FILTER" + +SELECT mgr, comm FROM emp WHERE mgr = 12 +UNION +SELECT mgr, comm FROM emp WHERE comm > 5; ++------+---------+ +| MGR | COMM | ++------+---------+ +| 7698 | 1400.00 | +| 7698 | 300.00 | +| 7698 | 500.00 | ++------+---------+ +(3 rows) + +!ok +EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[=($t8, $t9)], expr#11=[CAST($t6):DECIMAL(12, 2)], expr#12=[5.00:DECIMAL(12, 2)], expr#13=[>($t11, $t12)], expr#14=[OR($t10, $t13)], MGR=[$t3], COMM=[$t6], $condition=[$t14]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!set hep-rules original + +# Testing with the planner-rules shows that due to cost-based selection issues, +# the planner fails to choose a plan that includes the Aggregate operator. +!set planner-rules " ++CoreRules.PROJECT_FILTER_TRANSPOSE, ++CoreRules.UNION_FILTER_TO_FILTER" + +SELECT mgr, comm FROM emp WHERE mgr = 12 +UNION +SELECT mgr, comm FROM emp WHERE comm > 5; ++------+---------+ +| MGR | COMM | ++------+---------+ +| 7698 | 1400.00 | +| 7698 | 300.00 | +| 7698 | 500.00 | ++------+---------+ +(3 rows) + +!ok +EnumerableUnion(all=[false]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[=($t8, $t9)], MGR=[$t3], COMM=[$t6], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], MGR=[$t3], COMM=[$t6], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan +!set planner-rules original + +# This hep-rules test validates support for modifying EnumerableRules. +!set hep-rules " ++CoreRules.PROJECT_FILTER_TRANSPOSE, ++CoreRules.UNION_FILTER_TO_FILTER, +-EnumerableRules.ENUMERABLE_AGGREGATE_RULE" + +SELECT mgr, comm FROM emp WHERE mgr = 12 +UNION +SELECT mgr, comm FROM emp WHERE comm > 5; +Missing conversion is LogicalAggregate[convention: NONE -> ENUMERABLE] +!error +!set hep-rules original + +# End hep.iq diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java index 83419d6fb75c..f55b2c060fe0 100644 --- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java @@ -24,6 +24,7 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.prepare.Prepare; +import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -41,8 +42,12 @@ import org.apache.calcite.test.schemata.catchall.CatchallSchema; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.Bug; import org.apache.calcite.util.Closer; +import org.apache.calcite.util.Holder; import org.apache.calcite.util.Sources; import org.apache.calcite.util.Util; @@ -261,6 +266,30 @@ protected void checkRun(String path) throws Exception { })); } } + + if (propertyName.equals("hep-rules")) { + closer.add( + Hook.PROGRAM.addThread((Consumer>) + holder -> { + List hepRules = new ArrayList<>(); + List volcanoRules = + new ArrayList<>(EnumerableRules.ENUMERABLE_RULES); + + applyRulesInOrder((String) value, hepRules, volcanoRules); + + Program hepProgram = + Programs.hep(RuleSets.ofList(hepRules), false, + DefaultRelMetadataProvider.INSTANCE); + Program calcProgram = + Programs.calc(DefaultRelMetadataProvider.INSTANCE); + Program volcanoProgram = + Programs.of(RuleSets.ofList(volcanoRules)); + + Program combinedProgram = + Programs.sequence(hepProgram, volcanoProgram, calcProgram); + holder.set(combinedProgram); + })); + } }) .withEnv(QuidemTest::getEnv) .build(); @@ -317,6 +346,49 @@ private static void parseRules(String value, List rulesAdd, } } + /** + * Parse the rules string and apply operations in textual order to the + * provided hepRules and volcanoRules lists. + */ + private static void applyRulesInOrder(String value, + List hepRules, List volcanoRules) { + Pattern pattern = Pattern.compile("([+-])((CoreRules|EnumerableRules)\\.)?(\\w+)"); + Matcher matcher = pattern.matcher(value); + + while (matcher.find()) { + char operation = matcher.group(1).charAt(0); + String ruleSource = matcher.group(3); + String ruleName = matcher.group(4); + + try { + RelOptRule rule; + boolean targetVolcano = false; + if (ruleSource == null || ruleSource.equals("CoreRules")) { + rule = getCoreRule(ruleName); + } else if (ruleSource.equals("EnumerableRules")) { + Object ruleObj = EnumerableRules.class.getField(ruleName).get(null); + rule = (RelOptRule) ruleObj; + targetVolcano = true; + } else { + throw new RuntimeException("Unknown rule: " + ruleName); + } + + List target = targetVolcano ? volcanoRules : hepRules; + if (operation == '+') { + if (!target.contains(rule)) { + target.add(rule); + } + } else if (operation == '-') { + target.remove(rule); + } else { + throw new RuntimeException("unknown operation '" + operation + "'"); + } + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException("set rules failed: " + e.getMessage(), e); + } + } + } + public static RelOptRule getCoreRule(String ruleName) { try { Field ruleField = CoreRules.class.getField(ruleName); @@ -334,9 +406,17 @@ public static RelOptRule getCoreRule(String ruleName) { private static void setRules(char operation, RelOptRule rule, List rulesAdd, List rulesRemove) { if (operation == '+') { - rulesAdd.add(rule); + // Remove from remove list if present, then add to add list + rulesRemove.remove(rule); + if (!rulesAdd.contains(rule)) { + rulesAdd.add(rule); + } } else if (operation == '-') { - rulesRemove.add(rule); + // Remove from add list if present, then add to remove list + rulesAdd.remove(rule); + if (!rulesRemove.contains(rule)) { + rulesRemove.add(rule); + } } else { throw new RuntimeException("unknown operation '" + operation + "'"); }