[CALCITE-7235] Support Flexible HEP and Volcano Planner Rule Configuration in Quidem Tests

core/src/test/resources/sql/hep.iq

@@ -0,0 +1,117 @@
+# hep.iq - hep tests can customizable optimization rules for hep planner
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This documents the required format for .iq files containing hep planner rule configurations
+#
+# SYNTAX:
+# -------
+# !set hep-rules "
+# +CoreRules.RULE_NAME1,
+# -CoreRules.RULE_NAME2,
+# +EnumerableRules.ENUMERABLE_RULE_NAME"
+#
+# RULES FORMAT SPECIFICATION:
+# ---------------------------
+# 1. Must begin with: !set hep-rules " (double quote)
+# 2. Each rule must be on its own line. All lines except last must end with comma.
+#    The last rule line must close with " (double quote) on final line
+# 3. Rules must start with - (remove) or + (add)
+#
+# RULE PROCESSING LOGIC:
+# ----------------------
+# 1. Rule prefix interpretation:
+#    - CoreRules.RULE_NAME -> added/removed from hep program
+#    - EnumerableRules.RULE_NAME -> added/removed from volcano program
+#    - RULE_NAME (no prefix) -> treated as CoreRules, added/removed from hep program
+# 2. Duplicate operations on the same rule: last operation wins
+#    Example: "+Rule1,+Rule1,-Rule1" results in rule being removed
+# 3. Optimizer execution pipeline:
+#    The optimizer uses a multi-program architecture with the following execution order:
+#    - HEP program — executed by HepPlanner; initial rules set by CoreRules.*
+#    - Volcano program — executed by VolcanoPlanner; initial rules set by EnumerableRules.ENUMERABLE_RULES
+#      and modified by configurable EnumerableRules.* rule sets.
+#    - Calc program — executed by HepPlanner; initial rules set by RelOptRules.CALC_RULES (non-configurable)
+
+!use scott
+!set outputformat mysql
+
+# The hep-rules test is functioning correctly.
+!set hep-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
++------+---------+
+| MGR  | COMM    |
++------+---------+
+| 7698 | 1400.00 |
+| 7698 |  300.00 |
+| 7698 |  500.00 |
++------+---------+
+(3 rows)
+
+!ok
+EnumerableAggregate(group=[{0, 1}])
+  EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[=($t8, $t9)], expr#11=[CAST($t6):DECIMAL(12, 2)], expr#12=[5.00:DECIMAL(12, 2)], expr#13=[>($t11, $t12)], expr#14=[OR($t10, $t13)], MGR=[$t3], COMM=[$t6], $condition=[$t14])
+    EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!set hep-rules original
+
+# Testing with the planner-rules shows that due to cost-based selection issues,
+# the planner fails to choose a plan that includes the Aggregate operator.
+!set planner-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
++------+---------+
+| MGR  | COMM    |
++------+---------+
+| 7698 | 1400.00 |
+| 7698 |  300.00 |
+| 7698 |  500.00 |
++------+---------+
+(3 rows)
+
+!ok
+EnumerableUnion(all=[false])
+  EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER], expr#9=[12], expr#10=[=($t8, $t9)], MGR=[$t3], COMM=[$t6], $condition=[$t10])
+    EnumerableTableScan(table=[[scott, EMP]])
+  EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)], expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], MGR=[$t3], COMM=[$t6], $condition=[$t10])
+    EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!set planner-rules original
+
+# This hep-rules test validates support for modifying EnumerableRules.
+!set hep-rules "
++CoreRules.PROJECT_FILTER_TRANSPOSE,
++CoreRules.UNION_FILTER_TO_FILTER,
+-EnumerableRules.ENUMERABLE_AGGREGATE_RULE"
+
+SELECT mgr, comm FROM emp WHERE mgr = 12
+UNION
+SELECT mgr, comm FROM emp WHERE comm > 5;
+Missing conversion is LogicalAggregate[convention: NONE -> ENUMERABLE]
+!error
+!set hep-rules original
+
+# End hep.iq

testkit/src/main/java/org/apache/calcite/test/QuidemTest.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.plan.RelOptPlanner;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
 import org.apache.calcite.rel.rules.CoreRules;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -41,8 +42,12 @@
 import org.apache.calcite.test.schemata.catchall.CatchallSchema;
 import org.apache.calcite.tools.Frameworks;
 import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.Program;
+import org.apache.calcite.tools.Programs;
+import org.apache.calcite.tools.RuleSets;
 import org.apache.calcite.util.Bug;
 import org.apache.calcite.util.Closer;
+import org.apache.calcite.util.Holder;
 import org.apache.calcite.util.Sources;
 import org.apache.calcite.util.Util;
 
@@ -261,6 +266,30 @@ protected void checkRun(String path) throws Exception {
                         }));
               }
             }
+
+            if (propertyName.equals("hep-rules")) {
+              closer.add(
+                  Hook.PROGRAM.addThread((Consumer<Holder<Program>>)
+                      holder -> {
+                        List<RelOptRule> hepRules = new ArrayList<>();
+                        List<RelOptRule> volcanoRules =
+                            new ArrayList<>(EnumerableRules.ENUMERABLE_RULES);
+
+                        applyRulesInOrder((String) value, hepRules, volcanoRules);
+
+                        Program hepProgram =
+                            Programs.hep(RuleSets.ofList(hepRules), false,
+                                DefaultRelMetadataProvider.INSTANCE);
+                        Program calcProgram =
+                            Programs.calc(DefaultRelMetadataProvider.INSTANCE);
+                        Program volcanoProgram =
+                            Programs.of(RuleSets.ofList(volcanoRules));
+
+                        Program combinedProgram =
+                            Programs.sequence(hepProgram, volcanoProgram, calcProgram);
+                        holder.set(combinedProgram);
+                      }));
+            }
           })
           .withEnv(QuidemTest::getEnv)
           .build();
@@ -317,6 +346,49 @@ private static void parseRules(String value, List<RelOptRule> rulesAdd,
     }
   }
 
+  /**
+   * Parse the rules string and apply operations in textual order to the
+   * provided hepRules and volcanoRules lists.
+   */
+  private static void applyRulesInOrder(String value,
+      List<RelOptRule> hepRules, List<RelOptRule> volcanoRules) {
+    Pattern pattern = Pattern.compile("([+-])((CoreRules|EnumerableRules)\\.)?(\\w+)");
+    Matcher matcher = pattern.matcher(value);
+
+    while (matcher.find()) {
+      char operation = matcher.group(1).charAt(0);
+      String ruleSource = matcher.group(3);
+      String ruleName = matcher.group(4);
+
+      try {
+        RelOptRule rule;
+        boolean targetVolcano = false;
+        if (ruleSource == null || ruleSource.equals("CoreRules")) {
+          rule = getCoreRule(ruleName);
+        } else if (ruleSource.equals("EnumerableRules")) {
+          Object ruleObj = EnumerableRules.class.getField(ruleName).get(null);
+          rule = (RelOptRule) ruleObj;
+          targetVolcano = true;
+        } else {
+          throw new RuntimeException("Unknown rule: " + ruleName);
+        }
+
+        List<RelOptRule> target = targetVolcano ? volcanoRules : hepRules;
+        if (operation == '+') {
+          if (!target.contains(rule)) {
+            target.add(rule);
+          }
+        } else if (operation == '-') {
+          target.remove(rule);
+        } else {
+          throw new RuntimeException("unknown operation '" + operation + "'");
+        }
+      } catch (NoSuchFieldException | IllegalAccessException e) {
+        throw new RuntimeException("set rules failed: " + e.getMessage(), e);
+      }
+    }
+  }
+
   public static RelOptRule getCoreRule(String ruleName) {
     try {
       Field ruleField = CoreRules.class.getField(ruleName);
@@ -334,9 +406,17 @@ public static RelOptRule getCoreRule(String ruleName) {
   private static void setRules(char operation, RelOptRule rule,
       List<RelOptRule> rulesAdd, List<RelOptRule> rulesRemove) {
     if (operation == '+') {
-      rulesAdd.add(rule);
+      // Remove from remove list if present, then add to add list
+      rulesRemove.remove(rule);
+      if (!rulesAdd.contains(rule)) {
+        rulesAdd.add(rule);
+      }
     } else if (operation == '-') {
-      rulesRemove.add(rule);
+      // Remove from add list if present, then add to remove list
+      rulesAdd.remove(rule);
+      if (!rulesRemove.contains(rule)) {
+        rulesRemove.add(rule);
+      }
     } else {
       throw new RuntimeException("unknown operation '" + operation + "'");
     }