[CALCITE-3409] Add an interface in RelOptMaterializations to allow re…

…gistering UnifyRule (XuZhaohui JinXing)

core/src/main/java/org/apache/calcite/plan/AbstractRelOptPlanner.java

@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.plan;
 
+import org.apache.calcite.plan.SubstitutionVisitor.UnifyRule;
 import org.apache.calcite.plan.volcano.RelSubset;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -197,6 +198,10 @@ public boolean isRuleExcluded(RelOptRule rule) {
     return this;
   }
 
+  @Override public void addExtraMaterializationRules(List<UnifyRule> rules) {
+    // ignore - this planner does not support materializations
+  }
+
   @Override public void addMaterialization(RelOptMaterialization materialization) {
     // ignore - this planner does not support materializations
   }

core/src/main/java/org/apache/calcite/plan/RelOptMaterializations.java

@@ -50,6 +50,12 @@
  */
 public abstract class RelOptMaterializations {
 
+  @Deprecated // to be removed before 2.0
+  public static List<Pair<RelNode, List<RelOptMaterialization>>> useMaterializedViews(
+      final RelNode rel, List<RelOptMaterialization> materializations) {
+    return useMaterializedViews(rel, materializations, ImmutableList.of());
+  }
+
   /**
    * Returns a list of RelNode transformed from all possible combination of
    * materialized view uses. Big queries will likely have more than one
@@ -60,7 +66,8 @@ public abstract class RelOptMaterializations {
    *         materialized views used in the transformation.
    */
   public static List<Pair<RelNode, List<RelOptMaterialization>>> useMaterializedViews(
-      final RelNode rel, List<RelOptMaterialization> materializations) {
+      final RelNode rel, List<RelOptMaterialization> materializations,
+      List<SubstitutionVisitor.UnifyRule> materializationRules) {
     final List<RelOptMaterialization> applicableMaterializations =
         getApplicableMaterializations(rel, materializations);
     final List<Pair<RelNode, List<RelOptMaterialization>>> applied =
@@ -70,7 +77,7 @@ public static List<Pair<RelNode, List<RelOptMaterialization>>> useMaterializedVi
       int count = applied.size();
       for (int i = 0; i < count; i++) {
         Pair<RelNode, List<RelOptMaterialization>> current = applied.get(i);
-        List<RelNode> sub = substitute(current.left, m);
+        List<RelNode> sub = substitute(current.left, m, materializationRules);
         if (!sub.isEmpty()) {
           ImmutableList.Builder<RelOptMaterialization> builder =
               ImmutableList.builder();
@@ -170,7 +177,8 @@ && usesTable(materialization.qualifiedTableName, queryTablesUsed, frozenGraph))
   }
 
   private static List<RelNode> substitute(
-      RelNode root, RelOptMaterialization materialization) {
+      RelNode root, RelOptMaterialization materialization,
+      List<SubstitutionVisitor.UnifyRule> materializationRules) {
     // First, if the materialization is in terms of a star table, rewrite
     // the query in terms of the star table.
     if (materialization.starTable != null) {
@@ -214,7 +222,11 @@ private static List<RelNode> substitute(
     hepPlanner.setRoot(root);
     root = hepPlanner.findBestExp();
 
-    return new SubstitutionVisitor(target, root).go(materialization.tableRel);
+    return new SubstitutionVisitor(target, root, ImmutableList.
+        <SubstitutionVisitor.UnifyRule>builder()
+        .addAll(SubstitutionVisitor.DEFAULT_RULES)
+        .addAll(materializationRules)
+        .build()).go(materialization.tableRel);
   }
 
   /**

core/src/main/java/org/apache/calcite/plan/RelOptPlanner.java

@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.plan;
 
+import org.apache.calcite.plan.SubstitutionVisitor.UnifyRule;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
 import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -157,6 +158,16 @@ public interface RelOptPlanner {
    */
   RelOptPlanner chooseDelegate();
 
+  /**
+   * In addition to the internal defined materialization matching rules in
+   * {@link org.apache.calcite.plan.MaterializedViewSubstitutionVisitor}
+   * and {@link org.apache.calcite.plan.SubstitutionVisitor}, this method
+   * allows user to SET self defined {@link SubstitutionVisitor.UnifyRule}s,
+   * thus to extend the ability of substitution based materialization
+   * matching in different scenarios.
+   */
+  void addExtraMaterializationRules(List<UnifyRule> rules);
+
   /**
    * Defines a pair of relational expressions that are equivalent.
    *

core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java

@@ -853,7 +853,7 @@ protected static class MatchFailed extends ControlFlowException {
    * <p>The rule declares the query and target types; this allows the
    * engine to fire only a few rules in a given context.</p>
    */
-  protected abstract static class UnifyRule {
+  public abstract static class UnifyRule {
     protected final int slotCount;
     protected final Operand queryOperand;
     protected final Operand targetOperand;
@@ -920,7 +920,7 @@ protected <E> ImmutableList<E> copy(E[] slots, int slotCount) {
   /**
    * Arguments to an application of a {@link UnifyRule}.
    */
-  protected class UnifyRuleCall {
+  public class UnifyRuleCall {
     protected final UnifyRule rule;
     public final MutableRel query;
     public final MutableRel target;
@@ -975,7 +975,7 @@ public RexSimplify getSimplify() {
    * contains {@code target}. {@code stopTrying} indicates whether there's
    * no need to do matching for the same query node again.
    */
-  protected static class UnifyResult {
+  public static class UnifyResult {
     private final UnifyRuleCall call;
     private final MutableRel result;
     private final boolean stopTrying;
@@ -990,7 +990,7 @@ assert equalType("query", call.query, "result", result,
   }
 
   /** Abstract base class for implementing {@link UnifyRule}. */
-  protected abstract static class AbstractUnifyRule extends UnifyRule {
+  public abstract static class AbstractUnifyRule extends UnifyRule {
     protected AbstractUnifyRule(Operand queryOperand, Operand targetOperand,
         int slotCount) {
       super(slotCount, queryOperand, targetOperand);
@@ -1748,7 +1748,7 @@ private static int fieldCnt(MutableRel rel) {
   }
 
   /** Explain filtering condition and projections from MutableCalc. */
-  private static Pair<RexNode, List<RexNode>> explainCalc(MutableCalc calc) {
+  public static Pair<RexNode, List<RexNode>> explainCalc(MutableCalc calc) {
     final RexShuttle shuttle = getExpandShuttle(calc.program);
     final RexNode condition = shuttle.apply(calc.program.getCondition());
     final List<RexNode> projects = new ArrayList<>();
@@ -2056,7 +2056,7 @@ public static boolean equalType(String desc0, MutableRel rel0, String desc1,
   }
 
   /** Operand to a {@link UnifyRule}. */
-  protected abstract static class Operand {
+  public abstract static class Operand {
     protected final Class<? extends MutableRel> clazz;
 
     protected Operand(Class<? extends MutableRel> clazz) {

core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java

@@ -38,6 +38,7 @@
 import org.apache.calcite.plan.RelTrait;
 import org.apache.calcite.plan.RelTraitDef;
 import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.SubstitutionVisitor.UnifyRule;
 import org.apache.calcite.rel.PhysicalNode;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.convert.Converter;
@@ -162,6 +163,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
    */
   private boolean noneConventionHasInfiniteCost = true;
 
+  private ImmutableList<UnifyRule> materializationRules = ImmutableList.of();
+
   private final List<RelOptMaterialization> materializations =
       new ArrayList<>();
 
@@ -280,6 +283,10 @@ public void setTopDownOpt(boolean value) {
     return ImmutableList.copyOf(materializations);
   }
 
+  @Override public void addExtraMaterializationRules(List<UnifyRule> rules) {
+    materializationRules = ImmutableList.copyOf(rules);
+  }
+
   @Override public void addMaterialization(
       RelOptMaterialization materialization) {
     materializations.add(materialization);
@@ -303,7 +310,8 @@ protected void registerMaterializations() {
 
     // Register rels using materialized views.
     final List<Pair<RelNode, List<RelOptMaterialization>>> materializationUses =
-        RelOptMaterializations.useMaterializedViews(originalRoot, materializations);
+        RelOptMaterializations.useMaterializedViews(originalRoot, materializations,
+            materializationRules);
     for (Pair<RelNode, List<RelOptMaterialization>> use : materializationUses) {
       RelNode rel = use.left;
       Hook.SUB.run(rel);

core/src/test/java/org/apache/calcite/materialize/NormalizationTrimFieldTest.java

@@ -97,7 +97,7 @@ public static Frameworks.ConfigBuilder config() {
             target, null, Lists.newArrayList("mv0"));
     final List<Pair<RelNode, List<RelOptMaterialization>>> relOptimized =
         RelOptMaterializations.useMaterializedViews(query,
-            ImmutableList.of(relOptMaterialization));
+            ImmutableList.of(relOptMaterialization), ImmutableList.of());
 
     final String optimized = ""
         + "LogicalProject(deptno=[CAST($0):TINYINT], count_sal=[$1])\n"

core/src/test/java/org/apache/calcite/test/MaterializationTest.java

@@ -18,22 +18,37 @@
 
 import org.apache.calcite.adapter.java.ReflectiveSchema;
 import org.apache.calcite.materialize.MaterializationService;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRules;
 import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.SubstitutionVisitor;
 import org.apache.calcite.prepare.Prepare;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.RelReferentialConstraint;
 import org.apache.calcite.rel.RelReferentialConstraintImpl;
 import org.apache.calcite.rel.RelVisitor;
 import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.mutable.MutableCalc;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.runtime.Hook;
 import org.apache.calcite.schema.QueryableTable;
 import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.test.JdbcTest.Department;
 import org.apache.calcite.test.JdbcTest.Dependent;
 import org.apache.calcite.test.JdbcTest.Employee;
 import org.apache.calcite.test.JdbcTest.Event;
 import org.apache.calcite.test.JdbcTest.Location;
+import org.apache.calcite.tools.RuleSet;
+import org.apache.calcite.tools.RuleSets;
 import org.apache.calcite.util.JsonBuilder;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Smalls;
 import org.apache.calcite.util.TryThreadLocal;
 import org.apache.calcite.util.mapping.IntPair;
@@ -342,6 +357,112 @@ public class MaterializationTest {
     }
   }
 
+  @Test public void checkRegisterAdditionalMaterializationRules() {
+    String mv = "select * from \"emps\" where \"name\" like 'ABC%'";
+    String query = "select * from \"emps\" where \"name\" like 'ABCD%'";
+    checkMaterialize(mv, query, HR_FKUK_MODEL, CONTAINS_M0,
+        RuleSets.ofList(ImmutableList.of()),
+        ImmutableList.of(CustomizedMaterializationRule.INSTANCE));
+  }
+
+  /**
+   * A customized materialization rule, which match expression of 'LIKE'
+   * and match by compensation.
+   */
+  private static class CustomizedMaterializationRule
+      extends SubstitutionVisitor.AbstractUnifyRule {
+
+    public static final CustomizedMaterializationRule INSTANCE =
+        new CustomizedMaterializationRule();
+
+    private CustomizedMaterializationRule() {
+      super(operand(MutableCalc.class, query(0)),
+          operand(MutableCalc.class, target(0)), 1);
+    }
+
+    @Override protected SubstitutionVisitor.UnifyResult apply(
+        SubstitutionVisitor.UnifyRuleCall call) {
+      final MutableCalc query = (MutableCalc) call.query;
+      final Pair<RexNode, List<RexNode>> queryExplained = SubstitutionVisitor.explainCalc(query);
+      final RexNode queryCond = queryExplained.left;
+      final List<RexNode> queryProjs = queryExplained.right;
+
+      final MutableCalc target = (MutableCalc) call.target;
+      final Pair<RexNode, List<RexNode>> targetExplained = SubstitutionVisitor.explainCalc(target);
+      final RexNode targetCond = targetExplained.left;
+      final List<RexNode> targetProjs = targetExplained.right;
+      final List parsedQ = parseLikeCondition(queryCond);
+      final List parsedT = parseLikeCondition(targetCond);
+      if (RexUtil.isIdentity(queryProjs, query.getInput().rowType)
+          && RexUtil.isIdentity(targetProjs, target.getInput().rowType)
+          && parsedQ != null && parsedT != null) {
+        if (parsedQ.get(0).equals(parsedT.get(0))) {
+          String literalQ = ((NlsString) parsedQ.get(1)).getValue();
+          String literalT = ((NlsString) parsedT.get(1)).getValue();
+          if (literalQ.endsWith("%") && literalT.endsWith("%")
+              && !literalQ.equals(literalT)
+              && literalQ.startsWith(literalT.substring(0, literalT.length() - 1))) {
+            return call.result(MutableCalc.of(target, query.program));
+          }
+        }
+      }
+      return null;
+    }
+
+    private List parseLikeCondition(RexNode rexNode) {
+      if (rexNode instanceof RexCall) {
+        RexCall rexCall = (RexCall) rexNode;
+        if (rexCall.getKind() == SqlKind.LIKE
+            && rexCall.operands.get(0) instanceof RexInputRef
+            && rexCall.operands.get(1) instanceof RexLiteral) {
+          return ImmutableList.of(rexCall.operands.get(0),
+              ((RexLiteral) (rexCall.operands.get(1))).getValue());
+        }
+      }
+      return null;
+    }
+  }
+
+  private void checkMaterialize(String materialize, String query, String model,
+      Consumer<ResultSet> explainChecker, final RuleSet rules,
+      final List<SubstitutionVisitor.UnifyRule> materializationRules) {
+    checkThatMaterialize(materialize, query, "m0", false, model, explainChecker,
+        rules, false, materializationRules).sameResultWithMaterializationsDisabled();
+  }
+
+  /** Checks that a given query can use a materialized view with a given
+   * definition. */
+  private CalciteAssert.AssertQuery checkThatMaterialize(String materialize,
+      String query, String name, boolean existing, String model,
+      Consumer<ResultSet> explainChecker, final RuleSet rules,
+      boolean onlyBySubstitution,
+      final List<SubstitutionVisitor.UnifyRule> materializationRules) {
+    try (TryThreadLocal.Memo ignored = Prepare.THREAD_TRIM.push(true)) {
+      MaterializationService.setThreadLocal();
+      CalciteAssert.AssertQuery that = CalciteAssert.that()
+          .withMaterializations(model, existing, name, materialize)
+          .query(query)
+          .enableMaterializations(true);
+
+      // Add any additional rules required for the test
+      if (rules.iterator().hasNext() || !materializationRules.isEmpty()) {
+        that.withHook(Hook.PLANNER, (Consumer<RelOptPlanner>) planner -> {
+          for (RelOptRule rule : rules) {
+            planner.addRule(rule);
+          }
+          if (onlyBySubstitution) {
+            RelOptRules.MATERIALIZATION_RULES.forEach(rule -> {
+              planner.removeRule(rule);
+            });
+          }
+          planner.addExtraMaterializationRules(materializationRules);
+        });
+      }
+
+      return that.explainMatches("", explainChecker);
+    }
+  }
+
   private static <E> List<List<List<E>>> list3(E[][][] as) {
     final ImmutableList.Builder<List<List<E>>> builder =
         ImmutableList.builder();