HIVE-23456: Upgrade Calcite to 1.25.0 (Soumyakanti Das, Stamatis Zamp…

…etakis, reviewed by Jesus Camacho Rodriguez)

Closes #2203

itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out

@@ -84,6 +84,8 @@ POSTHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 PARTITION (key) VALUE
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
+OPTIMIZED SQL: SELECT *
+FROM TABLE(INLINE(ARRAY[ (1, '101'),  (2, '202'),  (3, '303'),  (4, '404'),  (5, '505')]))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out

@@ -48,6 +48,8 @@ POSTHOOK: query: EXPLAIN EXTENDED INSERT INTO TABLE table1 VALUES (1)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
+OPTIMIZED SQL: SELECT *
+FROM TABLE(INLINE(ARRAY[ (1)]))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out

@@ -102,6 +102,8 @@ POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 PARTITION (key)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
+OPTIMIZED SQL: SELECT *
+FROM TABLE(INLINE(ARRAY[ (1, '101'),  (2, '202'),  (3, '303'),  (4, '404'),  (5, '505')]))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out

@@ -56,6 +56,8 @@ POSTHOOK: query: EXPLAIN EXTENDED INSERT OVERWRITE TABLE table1 VALUES (1)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
+OPTIMIZED SQL: SELECT *
+FROM TABLE(INLINE(ARRAY[ (1)]))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

pom.xml

@@ -115,7 +115,7 @@
     <avatica.version>1.12.0</avatica.version>
     <avro.version>1.8.2</avro.version>
     <bcprov-jdk15on.version>1.64</bcprov-jdk15on.version>
-    <calcite.version>1.21.0</calcite.version>
+    <calcite.version>1.25.0</calcite.version>
     <datanucleus-api-jdo.version>5.2.2</datanucleus-api-jdo.version>
     <datanucleus-core.version>5.2.2</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-m13</datanucleus-jdo.version>

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -533,6 +533,8 @@ public static void closeCurrent() {
    */
   private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException {
     conf = c;
+    // turn off calcite rexnode normalization
+    System.setProperty("calcite.enable.rexnode.digest.normalize", "false");
     if (doRegisterAllFns) {
       registerAllFunctionsOnce();
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java

@@ -53,6 +53,18 @@ public final class Bug {
    */
   public static final boolean CALCITE_4499_FIXED = false;
 
+  /**
+   * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4200">issue
+   * CALCITE-4200</a> is fixed.
+   */
+  public static final boolean CALCITE_4200_FIXED = false;
+
+  /**
+   * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4513">issue
+   * CALCITE-4513</a> is fixed.
+   */
+  public static final boolean CALCITE_4513_FIXED=false;
+
   /**
    * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4574">CALCITE-4574</a> is fixed.
    */

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -18,12 +18,13 @@
 package org.apache.hadoop.hive.ql.optimizer.calcite;
 
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
 
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptUtil;
@@ -34,6 +35,7 @@
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
 import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
 import org.apache.calcite.rel.core.Sort;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
@@ -61,8 +63,6 @@
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.type.SqlTypeFamily;
-import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Pair;
@@ -247,9 +247,9 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel
     // added project if need to produce new keys than the original input
     // fields
     if (newKeyCount > 0) {
-      leftRel = factory.createProject(leftRel, newLeftFields,
+      leftRel = factory.createProject(leftRel, Collections.emptyList(), newLeftFields,
           SqlValidatorUtil.uniquify(newLeftFieldNames));
-      rightRel = factory.createProject(rightRel, newRightFields,
+      rightRel = factory.createProject(rightRel, Collections.emptyList(), newRightFields,
           SqlValidatorUtil.uniquify(newRightFieldNames));
     }
 
@@ -908,7 +908,7 @@ public static List<ExprNodeDesc> getExprNodes(List<Integer> inputRefs, RelNode i
       String inputTabAlias) {
     List<ExprNodeDesc> exprNodes = new ArrayList<ExprNodeDesc>();
     List<RexNode> rexInputRefs = getInputRef(inputRefs, inputRel);
-    List<RexNode> exprs = inputRel.getChildExps();
+    List<RexNode> exprs = inputRel instanceof Project ? ((Project) inputRel).getProjects() : null;
     // TODO: Change ExprNodeConverter to be independent of Partition Expr
     ExprNodeConverter exprConv = new ExprNodeConverter(inputTabAlias, inputRel.getRowType(),
         new HashSet<Integer>(), inputRel.getCluster().getTypeFactory());

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java

@@ -98,7 +98,8 @@ public static RelBuilderFactory proto(final Context context) {
     return new RelBuilderFactory() {
       @Override
       public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) {
-        return new HiveRelBuilder(context, cluster, schema);
+        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED));
+        return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema);
       }
     };
   }
@@ -257,14 +258,6 @@ public RexNode visitInputRef(RexInputRef inputRef) {
     }
   }
 
-  @Override
-  protected boolean shouldMergeProject() {
-    /* CALCITE-2470 added ability to merge Project-s together.
-     * The problem with it is that it may merge 2 windowing expressions.
-     */
-    return false;
-  }
-
   /** Make the method visible */
   @Override
   public AggCall aggregateCall(SqlAggFunction aggFunction, boolean distinct, boolean approximate, boolean ignoreNulls,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java

@@ -38,6 +38,7 @@
 import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory;
 import org.apache.calcite.rel.core.RelFactories.SetOpFactory;
 import org.apache.calcite.rel.core.RelFactories.SortFactory;
+import org.apache.calcite.rel.hint.RelHint;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
@@ -106,7 +107,7 @@ private HiveRelFactories() {
    */
   private static class HiveProjectFactoryImpl implements ProjectFactory {
     @Override
-    public RelNode createProject(RelNode child,
+    public RelNode createProject(RelNode child, List<RelHint> hints,
         List<? extends RexNode> childExprs, List<String> fieldNames) {
       RelOptCluster cluster = child.getCluster();
       RelDataType rowType = RexUtil.createStructType(
@@ -143,30 +144,18 @@ private static class HiveJoinFactoryImpl implements JoinFactory {
      *          Right input
      * @param condition
      *          Join condition
-     * @param joinType
-     *          Join type
      * @param variablesStopped
      *          Set of names of variables which are set by the LHS and used by
      *          the RHS and are not available to nodes above this JoinRel in the
      *          tree
+     *@param joinType
+     *             Join type
      * @param semiJoinDone
      *          Whether this join has been translated to a semi-join
      */
     @Override
-    public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRelType joinType,
-        Set<String> variablesStopped, boolean semiJoinDone) {
-      if (joinType == JoinRelType.SEMI) {
-        final RelOptCluster cluster = left.getCluster();
-        return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition);
-      }
-      return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType);
-    }
-
-    @Override
-    public RelNode createJoin(RelNode left, RelNode right, RexNode condition,
-        Set<CorrelationId> variablesSet, JoinRelType joinType, boolean semiJoinDone) {
-      // According to calcite, it is going to be removed before Calcite-2.0
-      // TODO: to handle CorrelationId
+    public RelNode createJoin(RelNode left, RelNode right, List<RelHint> hints, RexNode condition,
+      Set<CorrelationId> variablesStopped, JoinRelType joinType, boolean semiJoinDone) {
       if (joinType == JoinRelType.SEMI) {
         final RelOptCluster cluster = left.getCluster();
         return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition);
@@ -216,7 +205,7 @@ public RelNode createSortExchange(RelNode input, RelDistribution distribution, R
 
   private static class HiveAggregateFactoryImpl implements AggregateFactory {
     @Override
-    public RelNode createAggregate(RelNode child,
+    public RelNode createAggregate(RelNode child, List<RelHint> hints,
             ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets,
             List<AggregateCall> aggCalls) {
       return new HiveAggregate(child.getCluster(), child.getTraitSet(), child,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java

@@ -21,18 +21,22 @@
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
+
+
+import com.google.common.collect.ImmutableList;
+
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
-
-import com.google.common.collect.ImmutableList;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelOptUtil;
@@ -491,7 +495,7 @@ public static RelNode createSingleValueAggRel(
               ImmutableList.of(i), -1, 0, rel, null, null));
     }
 
-    return aggregateFactory.createAggregate(rel, false, ImmutableBitSet.of(), null, aggCalls);
+    return aggregateFactory.createAggregate(rel, Collections.emptyList(), ImmutableBitSet.of(), null, aggCalls);
   }
 
   /**
@@ -1102,8 +1106,8 @@ public static List<RelFieldCollation> getNewRelFieldCollations(
       }
     }
     Map<Integer, Integer> m = new HashMap<>();
-    for (int projPos = 0; projPos < project.getChildExps().size(); projPos++) {
-      RexNode expr = project.getChildExps().get(projPos);
+    for (int projPos = 0; projPos < project.getProjects().size(); projPos++) {
+      RexNode expr = project.getProjects().get(projPos);
       if (expr instanceof RexInputRef) {
         Set<Integer> positions = HiveCalciteUtil.getInputRefs(expr);
         if (positions.size() <= 1) {
@@ -1136,8 +1140,8 @@ public static List<Integer> getNewRelDistributionKeys(
           HiveProject project, RelDistribution distribution) {
     Set<Integer> needed = new HashSet<>(distribution.getKeys());
     Map<Integer, Integer> m = new HashMap<>();
-    for (int projPos = 0; projPos < project.getChildExps().size(); projPos++) {
-      RexNode expr = project.getChildExps().get(projPos);
+    for (int projPos = 0; projPos < project.getProjects().size(); projPos++) {
+      RexNode expr = project.getProjects().get(projPos);
       if (expr instanceof RexInputRef) {
         Set<Integer> positions = HiveCalciteUtil.getInputRefs(expr);
         if (positions.size() <= 1) {

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.rel.core.TableFunctionScan;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalCalc;
 import org.apache.calcite.rel.logical.LogicalCorrelate;
 import org.apache.calcite.rel.logical.LogicalExchange;
 import org.apache.calcite.rel.logical.LogicalFilter;
@@ -33,6 +34,7 @@
 import org.apache.calcite.rel.logical.LogicalMinus;
 import org.apache.calcite.rel.logical.LogicalProject;
 import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalTableModify;
 import org.apache.calcite.rel.logical.LogicalUnion;
 import org.apache.calcite.rel.logical.LogicalValues;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
@@ -102,6 +104,11 @@ public RelNode visit(LogicalFilter filter) {
         return visitChild(filter, 0, filter.getInput());
     }
 
+    @Override
+    public RelNode visit(LogicalCalc calc) {
+        return visitChild(calc, 0, calc.getInput());
+    }
+
     public RelNode visit(HiveProject project) {
         return visitChild(project, 0, project.getInput());
     }
@@ -142,6 +149,11 @@ public RelNode visit(LogicalExchange exchange) {
         return visitChildren(exchange);
     }
 
+    @Override
+    public RelNode visit(LogicalTableModify modify) {
+        return visitChildren(modify);
+    }
+
     public RelNode visit(RelNode other) {
         return visitChildren(other);
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java

@@ -248,6 +248,11 @@ public boolean isKey(ImmutableBitSet columns) {
     return false;
   }
 
+  @Override
+  public List<ImmutableBitSet> getKeys() {
+    return keys;
+  }
+
   @Override
   public List<RelReferentialConstraint> getReferentialConstraints() {
     return referentialConstraints;

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java

@@ -85,7 +85,6 @@ public void registerClass(RelNode node) {
       addRule(HiveDruidRules.FILTER_PROJECT_TRANSPOSE);
       addRule(HiveDruidRules.SORT_PROJECT_TRANSPOSE);
       addRule(HiveDruidRules.SORT);
-      addRule(HiveDruidRules.PROJECT_SORT_TRANSPOSE);
       return;
     }
     super.registerClass(node);

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java

@@ -159,10 +159,6 @@ public List<RelNode> getInputs() {
     return inputs;
   }
 
-  @Override public List<RexNode> getChildExps() {
-    return ImmutableList.of(condition);
-  }
-
   @Override
   public RelNode accept(RexShuttle shuttle) {
     RexNode joinFilter = shuttle.apply(this.condition);

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java

@@ -25,11 +25,12 @@
 import org.apache.calcite.rel.RelCollation;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.RelWriter;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
 import org.apache.calcite.util.Util;
 import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -132,8 +133,9 @@ public boolean isSynthetic() {
     return shuttle.visit(this);
   }
 
-  public boolean containsOver() {
-    return RexOver.containsOver(this.getChildExps(), null);
+  @Override
+  public RelWriter explainTerms(RelWriter pw) {
+    return super.explainTerms(pw)
+        .itemIf("synthetic", this.isSysnthetic, pw.getDetailLevel() == SqlExplainLevel.DIGEST_ATTRIBUTES);
   }
-
 }