HIVE-26524: Use Calcite to remove sections of a query plan known never produces rows #3588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

kasakrisz merged 47 commits into apache:master from kasakrisz:HIVE-26524-master-empty-op

Oct 7, 2022

itests/hive-blobstore/src/test/results/clientpositive/map_join_on_filter.q.out

-Original file line number
+Diff line change
@@ Expand Up @@
     POSTHOOK: type: LOAD
     #### A masked pattern was here ####
     POSTHOOK: Output: default@src_a_data
-    Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
     PREHOOK: query: SELECT /*+ MAPJOIN(src1, src2) */ *
     FROM src_a_data src1
     RIGHT OUTER JOIN src_a_data src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10)
@@ Expand Down @@

itests/hive-blobstore/src/test/results/clientpositive/zero_rows_blobstore.q.out

-Original file line number
+Diff line change
@@ Expand Up / @@ -75,13 +75,15 @@ POSTHOOK: Input: default@blobstore_target @@
     PREHOOK: query: INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobstore_source WHERE FALSE
     PREHOOK: type: QUERY
+    PREHOOK: Input: _dummy_database@_dummy_table
     PREHOOK: Input: default@blobstore_source
     PREHOOK: Output: default@blobstore_target
     POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobstore_source WHERE FALSE
     POSTHOOK: type: QUERY
+    POSTHOOK: Input: _dummy_database@_dummy_table
     POSTHOOK: Input: default@blobstore_source
     POSTHOOK: Output: default@blobstore_target
-    POSTHOOK: Lineage: blobstore_target.key SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:key, type:int, comment:null), ]
+    POSTHOOK: Lineage: blobstore_target.key SIMPLE []
     PREHOOK: query: SELECT COUNT(*) FROM blobstore_target
     PREHOOK: type: QUERY
     PREHOOK: Input: default@blobstore_target
@@ Expand Down @@

itests/hive-blobstore/src/test/results/clientpositive/zero_rows_hdfs.q.out

-Original file line number
+Diff line change
@@ Expand Up / @@ -71,13 +71,15 @@ POSTHOOK: Input: default@hdfs_target @@
     PREHOOK: query: INSERT OVERWRITE TABLE hdfs_target SELECT key FROM blobstore_source WHERE FALSE
     PREHOOK: type: QUERY
+    PREHOOK: Input: _dummy_database@_dummy_table
     PREHOOK: Input: default@blobstore_source
     PREHOOK: Output: default@hdfs_target
     POSTHOOK: query: INSERT OVERWRITE TABLE hdfs_target SELECT key FROM blobstore_source WHERE FALSE
     POSTHOOK: type: QUERY
+    POSTHOOK: Input: _dummy_database@_dummy_table
     POSTHOOK: Input: default@blobstore_source
     POSTHOOK: Output: default@hdfs_target
-    POSTHOOK: Lineage: hdfs_target.key SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:key, type:int, comment:null), ]
+    POSTHOOK: Lineage: hdfs_target.key SIMPLE []
     PREHOOK: query: SELECT COUNT(*) FROM hdfs_target
     PREHOOK: type: QUERY
     PREHOOK: Input: default@hdfs_target
@@ Expand Down @@

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -74,4 +74,14 @@ public final class Bug { @@
        * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-4704">CALCITE-4704</a> is fixed.
        */
       public static final boolean CALCITE_4704_FIXED = false;
+      /**
+       * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5293">CALCITE-5293</a> is fixed.
+       */
+      public static final boolean CALCITE_5293_FIXED = false;
+      /**
+       * Whether <a href="https://issues.apache.org/jira/browse/CALCITE-5294">CALCITE-5294</a> is fixed.
+       */
+      public static final boolean CALCITE_5294_FIXED = false;
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -75,6 +75,7 @@ @@
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSqlFunction;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
+    import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveValues;
     import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
     import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
     import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
@@ Expand Down Expand Up @@
         RelNode originalProjRel = null;
         while (tmpRel != null) {
-          if (tmpRel instanceof HiveProject || tmpRel instanceof HiveTableFunctionScan) {
+          if (tmpRel instanceof HiveProject || tmpRel instanceof HiveTableFunctionScan || tmpRel instanceof HiveValues) {
             originalProjRel = tmpRel;
             break;
           }
@@ Expand Down @@

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -24,7 +24,6 @@ @@
     import org.apache.calcite.plan.RelOptCluster;
     import org.apache.calcite.plan.RelOptSchema;
     import org.apache.calcite.plan.RelOptUtil;
-    import org.apache.calcite.rel.RelCollations;
     import org.apache.calcite.rel.RelNode;
     import org.apache.calcite.rel.core.CorrelationId;
     import org.apache.calcite.rel.core.Join;
@@ Expand Down Expand Up @@
         return this;
       }
-      /**
-       * Empty relationship can be expressed in many different ways, e.g.,
-       * filter(cond=false), empty LogicalValues(), etc. Calcite default implementation
-       * uses empty LogicalValues(); however, currently there is not an equivalent to
-       * this expression in Hive. Thus, we use limit 0, since Hive already includes
-       * optimizations that will do early pruning of the result tree when it is found,
-       * e.g., GlobalLimitOptimizer.
-       */
-      @Override
-      public RelBuilder empty() {
-        final RelNode input = build();
-        final RelNode sort = HiveRelFactories.HIVE_SORT_FACTORY.createSort(
-                input, RelCollations.of(), null, literal(0));
-        return this.push(sort);
-      }
       public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) {
         switch (flag) {
           case YEAR:
@@ Expand Down @@

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -40,6 +40,7 @@ @@
     import org.apache.calcite.rel.core.RelFactories.SortFactory;
     import org.apache.calcite.rel.hint.RelHint;
     import org.apache.calcite.rel.type.RelDataType;
+    import org.apache.calcite.rex.RexLiteral;
     import org.apache.calcite.rex.RexNode;
     import org.apache.calcite.rex.RexUtil;
     import org.apache.calcite.sql.SqlKind;
@@ Expand All / @@ -48,11 +49,13 @@ @@
     import org.apache.calcite.util.ImmutableBitSet;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAntiJoin;
+    import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveValues;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+    import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
     import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
@@ Expand All / @@ -79,6 +82,9 @@ public class HiveRelFactories { @@
       public static final RelFactories.SortExchangeFactory HIVE_SORT_EXCHANGE_FACTORY =
               new HiveSortExchangeFactoryImpl();
+      public static final RelFactories.ValuesFactory HIVE_VALUES_FACTORY =
+              new HiveValuesFactoryImpl();
       public static final AggregateFactory HIVE_AGGREGATE_FACTORY =
               new HiveAggregateFactoryImpl();
@@ Expand All / @@ -94,6 +100,7 @@ public class HiveRelFactories { @@
                   HIVE_SEMI_JOIN_FACTORY,
                   HIVE_SORT_FACTORY,
                   HIVE_SORT_EXCHANGE_FACTORY,
+                  HIVE_VALUES_FACTORY,
                   HIVE_AGGREGATE_FACTORY,
                   HIVE_SET_OP_FACTORY));
@@ Expand Down Expand Up @@
         }
       }
+      private static class HiveValuesFactoryImpl implements RelFactories.ValuesFactory {
+        @Override
+        public RelNode createValues(RelOptCluster cluster, RelDataType rowType, List<ImmutableList<RexLiteral>> tuples) {
+          return new HiveValues(
+                  cluster, rowType, ImmutableList.copyOf(tuples), cluster.traitSetOf(HiveRelNode.CONVENTION));
+        }
+      }
       private static class HiveAggregateFactoryImpl implements AggregateFactory {
         @Override
         public RelNode createAggregate(RelNode child, List<RelHint> hints,
@@ Expand Down @@

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveValues.java

-Original file line number
+Diff line change
@@ -0,0 +1,46 @@
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one
+     * or more contributor license agreements.  See the NOTICE file
+     * distributed with this work for additional information
+     * regarding copyright ownership.  The ASF licenses this file
+     * to you under the Apache License, Version 2.0 (the
+     * "License"); you may not use this file except in compliance
+     * with the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
+    import com.google.common.collect.ImmutableList;
+    import org.apache.calcite.plan.RelOptCluster;
+    import org.apache.calcite.plan.RelTraitSet;
+    import org.apache.calcite.rel.RelNode;
+    import org.apache.calcite.rel.core.Values;
+    import org.apache.calcite.rel.type.RelDataType;
+    import org.apache.calcite.rex.RexLiteral;
+    import java.util.List;
+    /**
+     * Subclass of {@link org.apache.calcite.rel.core.Values}.
+     * Specialized to Hive engine.
+     */
+    public class HiveValues extends Values {
+      public HiveValues(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples,
+                        RelTraitSet traits) {
+        super(cluster, rowType, tuples, traits);
+      }
+      @Override
+      public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+        return new HiveValues(getCluster(), getRowType(), tuples, getTraitSet());
+      }
+    }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -32,6 +32,7 @@ @@
     import org.apache.calcite.plan.RelOptTable;
     import org.apache.calcite.plan.RelOptUtil;
     import org.apache.calcite.rel.RelCollation;
+    import org.apache.calcite.rel.RelCollations;
     import org.apache.calcite.rel.RelDistribution;
     import org.apache.calcite.rel.RelFieldCollation;
     import org.apache.calcite.rel.RelNode;
@@ Expand Down Expand Up @@
           if (aggCall.filterArg >= 0) {
             aggCallFieldsUsedBuilder.set(aggCall.filterArg);
           }
+          aggCallFieldsUsedBuilder.addAll(RelCollations.ordinals(aggCall.collation));
         }
         // transform if group by contain constant keys
@@ Expand Down Expand Up @@
             final RexNode filterArg = aggCall.filterArg < 0 ? null
                 : relBuilder.field(Mappings.apply(inputMapping, aggCall.filterArg));
             RelBuilder.AggCall newAggCall =
-                relBuilder.aggregateCall(aggCall.getAggregation(),
-                    aggCall.isDistinct(), aggCall.isApproximate(),
-                    filterArg, aggCall.name, args);
+                    relBuilder.aggregateCall(aggCall.getAggregation(), args)
+                            .distinct(aggCall.isDistinct())
+                            .filter(filterArg)
+                            .approximate(aggCall.isApproximate())
+                            .sort(relBuilder.fields(aggCall.collation))
+                            .as(aggCall.name);
             mapping.set(j, updatedGroupCount +  newAggCallList.size());
             newAggCallList.add(newAggCall);
           }
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

HIVE-26524: Use Calcite to remove sections of a query plan known never produces rows #3588

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!