From 703e01a99d3c200dc9a0b4fb65b742206d2c4cad Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Mon, 7 Mar 2022 13:46:43 +0800 Subject: [PATCH 01/21] [CALCITE-5031] Release Calcite 1.30.0 --- site/_docs/history.md | 149 +++++++++++++++++++++++ site/_posts/2022-03-04-release-1.30.0.md | 38 ++++++ 2 files changed, 187 insertions(+) create mode 100644 site/_posts/2022-03-04-release-1.30.0.md diff --git a/site/_docs/history.md b/site/_docs/history.md index eaa1a2efdcf8..711901ecec5a 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -210,6 +210,155 @@ mans2singh. * Site: Add syntax highlighting to SQL statements * Site: Improve HTML tables display & update CSV tutorial +## 1.30.0 / 2022-03-04 +{: #v1-30-0} + +This release comes two months after [1.29.0](#v1-29-0), +contains contributions from 29 authors, +and resolves 37 issues. + +Compatibility: This release is tested on Linux, macOS, Microsoft Windows; +using JDK/OpenJDK versions 8 to 17; +Guava versions 19.0 to 31.0.1-jre; +other software versions as specified in gradle.properties. + +Contributors to this release: + +Alessandro Solimando, +Bill Neil, +Chen Kai, +Eugen Stan, +Feng Zhu, +Jacques Nadeau, +Jake Xie, +Jay Narale, +Jiatao Tao, +Jing Zhang, +Julian Hyde, +LM Kang, +Liya Fan (release manager), +Marco Jorge, +Marieke Gueye, +NobiGo, +Roman Puchkovskiy, +Ruben Quesada Lopez, +Scott Reynolds, +Soumyakanti Das, +Stamatis Zampetakis, +Vova Vysotskyi, +Will Noble, +Xiong Duan, +Yanjing Wang, +Yiqun Zhang, +Xurenhe, +Zhe Hu, +mans2singh. + +#### New features +{: #new-features-1-30-0} + +* [CALCITE-4822] + Add `ARRAY_CONCAT`, `ARRAY_REVERSE`, `ARRAY_LENGTH` functions for BigQuery dialect +* [CALCITE-4980] + Make Babel parser support MySQL NULL-safe equal operator '<=>' +* [CALCITE-4967] + Support SQL hints for temporal table join +* [CALCITE-3673] + Support `ListTransientTable` without tables in the schema + +### Improvements +{: #fixes-1-30-0} + +* [CALCITE-5019] + Avoid multiple scans when the table is `ProjectableFilterableTable` +* [CALCITE-5008] + Ignore synthetic and static methods in `MetadataDef` during instrumentation +* [CALCITE-4996] + In `RelJson`, add a `readExpression` method that converts JSON to a RexNode expression +* [CALCITE-4994] + Improve the performance of SQL-to-RelNode conversion when the table contains hundreds of fields +* [CALCITE-4991] + Make RuleEventLogger print input rels in `FULL_PLAN` mode +* [CALCITE-4986] + Make `HepProgram` thread-safe +* [CALCITE-4963] + Improve the extensibility of `SqlDialectFactory` by making the default behavior in + `SqlDialectFactoryImpl` reusable +* [CALCITE-4960] + Enable unit tests in Elasticsearch Adapter +* [CALCITE-4953] + Deprecate `TableAccessMap` class +* [CALCITE-4952] + Enable use of RelMetadataQuery through a simplistic & slow proxy path +* [CALCITE-4885] + Fluent test fixtures so that dependent projects can write parser, validator and rules tests +* [CALCITE-1794] + Simplify expressions with numeric comparisons when CAST is present + +#### Bug-fixes +{: #fixes-1-30-0} + +* [CALCITE-5011] + Fix the initialization error for `CassandraAdapterDataTypesTest` +* [CALCITE-5006] + Make Gradle tasks for launching JDBC integration tests working +* [CALCITE-4997] + Keep `APPROX_COUNT_DISTINCT` in some `SqlDialect`s +* [CALCITE-4995] + Fix `AssertionError` caused by `RelFieldTrimmer` on SEMI/ANTI join +* [CALCITE-4988] + Expression `((A IS NOT NULL OR B) AND A IS NOT NULL)` can't be simplify to `(A IS NOT NULL)` when `A` + is deterministic +* [CALCITE-4968] + Fix the invalid MS SQL queries generated by Calcite for the query with `LIMIT` statement +* [CALCITE-4965] + Fix the failure of IS NOT NULL in Elasticsearch Adapter +* [CALCITE-4912] + Fix the confusing Javadoc of `RexSimplify#simplify` +* [CALCITE-4901] + Fix the problem that JDBC adapter incorrectly adds ORDER BY columns to the SELECT + list of generated SQL query +* [CALCITE-4872] + Fix the problem that UNKNOWN SqlTypeName is erroneously treated as NULL +* [CALCITE-4702] + Fix the error when executing query with GROUP BY constant via JDBC adapter +* [CALCITE-4683] + Fix the type mismatch exception when IN-list is converted to JOIN +* [CALCITE-4323] + If a view definition has an ORDER BY clause, retain the sort if the view is used in a + query at top level +* [CALCITE-4292] + Fix wrong results in ElasticSearch when query contains 'NOT EQUAL' +* [CALCITE-4054] + Fix NPE caused by RepeatUnion containing a Correlate with a transientScan on its RHS +* [CALCITE-3627] + Fix the incorrect null semantic for ROW function + +#### Dependency version upgrade +{: #dependency-1-30-0} + +* [CALCITE-5030] + Upgrade jsonpath version from 2.4.0 to 2.7.0 +* [CALCITE-5025] + Upgrade commons-io version from 2.4 to 2.11.0 +* [CALCITE-5007] + Upgrade H2 database version to 2.1.210 +* [CALCITE-4973] + Upgrade log4j2 version to 2.17.1 + +#### Web site and documentation +{: #site-1-30-0} + +* Site: Update PMC Chair +* Site: Add external resources section in the community page +* Site: Add "calcite-clj - Use Calcite with Clojure" in talks section +* Site: Add Alessandro Solimando as committer +* Site: Fix typo in howto.md +* Site: Change the javadoc title to Apache Calcite API +* Site: For tables that display results, center the content horizontally +* Site: Add syntax highlighting to SQL statements +* Site: Improve HTML tables display & update CSV tutorial + ## 1.29.0 / 2021-12-26 {: #v1-29-0} diff --git a/site/_posts/2022-03-04-release-1.30.0.md b/site/_posts/2022-03-04-release-1.30.0.md new file mode 100644 index 000000000000..8a91ec8659b9 --- /dev/null +++ b/site/_posts/2022-03-04-release-1.30.0.md @@ -0,0 +1,38 @@ +--- +layout: news_item +date: "2022-03-04 00:00:00 +0800" +author: liyafan82 +version: 1.30.0 +categories: [release] +tag: v1-30-0 +sha: +--- + + +The [Apache Calcite PMC]({{ site.baseurl }}) +is pleased to announce +[Apache Calcite release 1.30.0]({{ site.baseurl }}/docs/history.html#v1-30-0). + +This release comes two months after [1.29.0](#v1-29-0), +contains contributions from 29 authors, +and resolves 37 issues. + +This release fixes vulnerability issues +such as CVE-2021-27568. From 8fc0f2bc687142141ab04d63a67952b9022f2cf0 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Wed, 1 Dec 2021 20:01:08 +0300 Subject: [PATCH 02/21] CALCITE-4913 Correlated variables in a select list are not deduplicated --- .../calcite/rel/logical/LogicalProject.java | 69 +++++++++++++++---- .../rel/rules/ProjectWindowTransposeRule.java | 3 +- .../calcite/rel/rules/SubQueryRemoveRule.java | 26 ++++++- .../calcite/sql2rel/SqlToRelConverter.java | 26 ++++--- .../calcite/test/SqlToRelConverterTest.java | 13 ++++ .../apache/calcite/test/RelOptRulesTest.xml | 12 ++-- .../calcite/test/SqlToRelConverterTest.xml | 36 +++++++++- 7 files changed, 148 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index cb64b372700c..7ef4e6ca8d20 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -24,6 +24,8 @@ import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMdCollation; @@ -35,16 +37,21 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Objects; +import java.util.Set; /** * Sub-class of {@link org.apache.calcite.rel.core.Project} not * targeted at any particular engine or calling convention. */ public final class LogicalProject extends Project { + private final ImmutableSet variablesSet; + //~ Constructors ----------------------------------------------------------- /** @@ -52,12 +59,14 @@ public final class LogicalProject extends Project { * *

Use {@link #create} unless you know what you're doing. * - * @param cluster Cluster this relational expression belongs to - * @param traitSet Traits of this relational expression - * @param hints Hints of this relational expression - * @param input Input relational expression - * @param projects List of expressions for the input columns - * @param rowType Output row type + * @param cluster Cluster this relational expression belongs to + * @param traitSet Traits of this relational expression + * @param hints Hints of this relational expression + * @param input Input relational expression + * @param projects List of expressions for the input columns + * @param rowType Output row type + * @param variablesSet Correlation variables set by this relational expression + * to be used by nested expressions */ public LogicalProject( RelOptCluster cluster, @@ -65,22 +74,26 @@ public LogicalProject( List hints, RelNode input, List projects, - RelDataType rowType) { + RelDataType rowType, + ImmutableSet variablesSet) { super(cluster, traitSet, hints, input, projects, rowType); assert traitSet.containsIfApplicable(Convention.NONE); + this.variablesSet = Objects.requireNonNull(variablesSet, "variablesSet"); } @Deprecated // to be removed before 2.0 public LogicalProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, + ImmutableSet.of()); } @Deprecated // to be removed before 2.0 public LogicalProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, int flags) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, + ImmutableSet.of()); Util.discard(flags); } @@ -90,7 +103,7 @@ public LogicalProject(RelOptCluster cluster, RelNode input, this(cluster, cluster.traitSetOf(RelCollations.EMPTY), ImmutableList.of(), input, projects, RexUtil.createStructType(cluster.getTypeFactory(), projects, - fieldNames, null)); + fieldNames, null), ImmutableSet.of()); Util.discard(flags); } @@ -99,6 +112,7 @@ public LogicalProject(RelOptCluster cluster, RelNode input, */ public LogicalProject(RelInput input) { super(input); + this.variablesSet = ImmutableSet.of(); } //~ Methods ---------------------------------------------------------------- @@ -123,12 +137,31 @@ public static LogicalProject create(final RelNode input, List hints, cluster.traitSet().replace(Convention.NONE) .replaceIfs(RelCollationTraitDef.INSTANCE, () -> RelMdCollation.project(mq, input, projects)); - return new LogicalProject(cluster, traitSet, hints, input, projects, rowType); + return new LogicalProject(cluster, traitSet, hints, input, projects, rowType, + ImmutableSet.of()); + } + + /** Creates a LogicalProject, specifying row type rather than field names. */ + public static LogicalProject create(final RelNode input, List hints, + final List projects, RelDataType rowType, + ImmutableSet variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = + cluster.traitSet().replace(Convention.NONE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.project(mq, input, projects)); + return new LogicalProject(cluster, traitSet, hints, input, projects, rowType, variablesSet); + } + + @Override public Set getVariablesSet() { + return variablesSet; } @Override public LogicalProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType); + return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType, + variablesSet); } @Override public RelNode accept(RelShuttle shuttle) { @@ -137,14 +170,20 @@ public static LogicalProject create(final RelNode input, List hints, @Override public RelNode withHints(List hintList) { return new LogicalProject(getCluster(), traitSet, hintList, - input, getProjects(), getRowType()); + input, getProjects(), getRowType(), variablesSet); + } + + @Override public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw) + .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); } @Override public boolean deepEquals(@Nullable Object obj) { - return deepEquals0(obj); + return deepEquals0(obj) + && variablesSet.equals(((LogicalProject) obj).variablesSet); } @Override public int deepHashCode() { - return deepHashCode0(); + return Objects.hash(deepHashCode0(), variablesSet); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index 761dd1df7055..5512459076f9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -37,6 +37,7 @@ import org.apache.calcite.util.ImmutableBitSet; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; @@ -99,7 +100,7 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { final LogicalProject projectBelowWindow = new LogicalProject(cluster, window.getTraitSet(), ImmutableList.of(), - window.getInput(), exps, builder.build()); + window.getInput(), exps, builder.build(), ImmutableSet.of()); // Create a new LogicalWindow with necessary inputs only final List groups = new ArrayList<>(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index b3877241539b..c57f2853036b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel.rules; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelRule; @@ -41,12 +42,14 @@ import org.apache.calcite.sql.fun.SqlQuantifyOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql2rel.DeduplicateCorrelateVariables; import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; @@ -129,8 +132,25 @@ protected RexNode apply(RexSubQuery e, Set variablesSet, */ private static RexNode rewriteScalarQuery(RexSubQuery e, Set variablesSet, RelBuilder builder, int inputCount, int offset) { - builder.push(e.rel); - final RelMetadataQuery mq = e.rel.getCluster().getMetadataQuery(); + // in case the relation has several scalar queries + // referred to the same correlated variable, every + // such a query will be re-written to a particular + // correlate with the common set of variables, + // resulting a different relations will be trying to + // set the same correlated variable. To overcome this + // let's rewrite an every correlated variable with a + // new one created specially for current scalar query + RelNode r = e.rel; + RelOptCluster cluster = r.getCluster(); + ImmutableSet.Builder newVariablesSet = ImmutableSet.builder(); + for (CorrelationId corrId : variablesSet) { + CorrelationId newId = cluster.createCorrel(); + newVariablesSet.add(newId); + r = DeduplicateCorrelateVariables.go(cluster.getRexBuilder(), newId, + ImmutableSet.of(corrId), r); + } + builder.push(r); + final RelMetadataQuery mq = cluster.getMetadataQuery(); final Boolean unique = mq.areColumnsUnique(builder.peek(), ImmutableBitSet.of()); if (unique == null || !unique) { @@ -138,7 +158,7 @@ private static RexNode rewriteScalarQuery(RexSubQuery e, Set vari builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, builder.field(0))); } - builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet); + builder.join(JoinRelType.LEFT, builder.literal(true), newVariablesSet.build()); return field(builder, inputCount, offset); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 91d223c62bf6..638f757109c8 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2696,16 +2696,6 @@ protected void convertCollectionTable( validator().getValidatedNodeType(call), columnMappings); - final SqlValidatorScope selectScope = - ((DelegatingScope) bb.scope()).getParent(); - final Blackboard seekBb = createBlackboard(selectScope, null, false); - - final CorrelationUse p = getCorrelationUse(seekBb, callRel); - if (p != null) { - assert p.r instanceof LogicalTableFunctionScan; - callRel = (LogicalTableFunctionScan) p.r; - } - bb.setRoot(callRel, true); afterTableFunction(bb, call, callRel); } @@ -4400,7 +4390,21 @@ private void convertSelectList( relBuilder.push(bb.root()) .projectNamed(exprs, fieldNames, true); - bb.setRoot(relBuilder.build(), false); + + RelNode project = relBuilder.build(); + + final RelNode r; + final CorrelationUse p = getCorrelationUse(bb, project); + if (p != null) { + assert p.r instanceof Project; + Project proj = (Project) p.r; + r = LogicalProject.create(proj.getInput(), proj.getHints(), + proj.getProjects(), proj.getRowType(), ImmutableSet.of(p.id)); + } else { + r = project; + } + + bb.setRoot(r, false); assert bb.columnMonotonicities.isEmpty(); bb.columnMonotonicities.addAll(columnMonotonicityList); diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 41b38ec74697..3b183a114a78 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1288,6 +1288,19 @@ class SqlToRelConverterTest extends SqlToRelTestBase { + "from emp e"); } + @Test void testCorrelatedScalarSubQueryInSelectList() { + Consumer fn = sql -> { + sql(sql).expand(true).decorrelate(false) + .convertsTo("${planExpanded}"); + sql(sql).expand(false).decorrelate(false) + .convertsTo("${planNotExpanded}"); + }; + fn.accept("select e.deptno,\n" + + " (select count(*) from emp where e.deptno > 0),\n" + + " (select count(*) from emp where e.deptno > 0 and e.deptno < 10)\n" + + "from emp e"); + } + @Test void testCorrelationLateralSubQuery() { String sql = "SELECT deptno, ename\n" + "FROM\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index fa18df0f9a64..e0eea6816939 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1704,11 +1704,11 @@ LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject("K0"=[$0], "C1"=[$1], "F1"."A0"=[$2], "F2"."A0"=[$3], "F0"."C0"=[$4], "F1"."C0"=[$5], "F0"."C1"=[$6], "F1"."C2"=[$7], "F2"."C3"=[$8]) LogicalProject("K0"=[$0], "C1"=[$1], "F1"."A0"=[$2], "F2"."A0"=[$3], "F0"."C0"=[$4], "F1"."C0"=[$5], "F0"."C1"=[$6], "F1"."C2"=[$7], "F2"."C3"=[$8]) LogicalFilter(condition=[=($4, $9)]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}]) LogicalTableScan(table=[[CATALOG, STRUCT, T]]) LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject("F1"."C0"=[$5]) - LogicalFilter(condition=[=($cor0."K0", $0)]) + LogicalFilter(condition=[=($cor1."K0", $0)]) LogicalTableScan(table=[[CATALOG, STRUCT, T]]) ]]> @@ -3763,11 +3763,11 @@ LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject(EMPNO=[$0]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[=($5, $9)]) - LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject(SAL=[$5]) - LogicalFilter(condition=[=($0, $cor0.EMPNO)]) + LogicalFilter(condition=[=($0, $cor1.EMPNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -11760,7 +11760,7 @@ LogicalProject(EXPR$0=[> SOME($0, { LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($cor0.JOB, $1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) -})]) +})], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -11802,7 +11802,7 @@ LogicalProject(SAL=[$5], EXPR$1=[NOT(IN($0, { LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($cor0.JOB, $1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) -}))]) +}))], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 7d883fbaaac1..0301832f4bc6 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -587,6 +587,40 @@ LogicalProject(DEPTNO=[$0], ENAME=[$1]) LogicalProject(ENAME=[$1], SAL=[$5]) LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + ($cor0.DEPTNO, 0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})], EXPR$2=[$SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(>($cor0.DEPTNO, 0), <($cor0.DEPTNO, 10))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})], variablesSet=[[$cor0]]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($cor0.DEPTNO, 0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(>($cor1.DEPTNO, 0), <($cor1.DEPTNO, 10))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -750,7 +784,7 @@ LogicalProject(DEPTNO=[$7], EXPR$1=[$9]) LogicalProject(DEPTNO=[$7], EXPR$1=[$SCALAR_QUERY({ LogicalProject(NAME=[$0]) LogicalTableFunctionScan(invocation=[DEDUP($cor0.DEPTNO, $cor0.DEPTNO)], rowType=[RecordType(VARCHAR(1024) NAME)]) -})]) +})], variablesSet=[[$cor0]]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> From 6a98c552a4356eecdb19ba65e799d96e7dc8c286 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 3 Dec 2021 16:39:49 +0300 Subject: [PATCH 03/21] move variableSet to project and explain it before expressions --- .../org/apache/calcite/rel/core/Project.java | 47 ++++++++++++++----- .../calcite/rel/logical/LogicalProject.java | 26 ++-------- .../apache/calcite/test/RelOptRulesTest.xml | 8 ++-- .../calcite/test/SqlToRelConverterTest.xml | 8 ++-- 4 files changed, 47 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index f4a2e3ac3ddc..41d2ecc888a2 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -45,6 +45,7 @@ import org.apache.calcite.util.mapping.Mappings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; @@ -70,17 +71,21 @@ public abstract class Project extends SingleRel implements Hintable { protected final ImmutableList hints; + protected final ImmutableSet variablesSet; + //~ Constructors ----------------------------------------------------------- /** * Creates a Project. * - * @param cluster Cluster that this relational expression belongs to - * @param traits Traits of this relational expression - * @param hints Hints of this relation expression - * @param input Input relational expression - * @param projects List of expressions for the input columns - * @param rowType Output row type + * @param cluster Cluster that this relational expression belongs to + * @param traits Traits of this relational expression + * @param hints Hints of this relation expression + * @param input Input relational expression + * @param projects List of expressions for the input columns + * @param rowType Output row type + * @param variablesSet Correlation variables set by this relational expression to be used by + * nested expressions */ @SuppressWarnings("method.invocation.invalid") protected Project( @@ -89,25 +94,33 @@ protected Project( List hints, RelNode input, List projects, - RelDataType rowType) { + RelDataType rowType, + Set variablesSet) { super(cluster, traits, input); assert rowType != null; this.exps = ImmutableList.copyOf(projects); this.hints = ImmutableList.copyOf(hints); this.rowType = rowType; + this.variablesSet = Objects.requireNonNull( + ImmutableSet.copyOf(variablesSet), "variablesSet"); assert isValid(Litmus.THROW, null); } + protected Project(RelOptCluster cluster, RelTraitSet traits, List hints, + RelNode input, List projects, RelDataType rowType) { + this(cluster, traits, hints, input, projects, rowType, ImmutableSet.of()); + } + @Deprecated // to be removed before 2.0 protected Project(RelOptCluster cluster, RelTraitSet traits, RelNode input, List projects, RelDataType rowType) { - this(cluster, traits, ImmutableList.of(), input, projects, rowType); + this(cluster, traits, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); } @Deprecated // to be removed before 2.0 protected Project(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, int flags) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); Util.discard(flags); } @@ -120,7 +133,8 @@ protected Project(RelInput input) { ImmutableList.of(), input.getInput(), requireNonNull(input.getExpressionList("exprs"), "exprs"), - input.getRowType("exprs", "fields")); + input.getRowType("exprs", "fields"), + ImmutableSet.of()); } //~ Methods ---------------------------------------------------------------- @@ -265,7 +279,9 @@ private static int countTrivial(List refs) { } @Override public RelWriter explainTerms(RelWriter pw) { - super.explainTerms(pw); + super.explainTerms(pw) + .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); + // Skip writing field names so the optimizer can reuse the projects that differ in // field names only if (pw.getDetailLevel() == SqlExplainLevel.DIGEST_ATTRIBUTES) { @@ -297,6 +313,10 @@ private static int countTrivial(List refs) { return pw; } + @Override public Set getVariablesSet() { + return variablesSet; + } + @API(since = "1.24", status = API.Status.INTERNAL) @EnsuresNonNullIf(expression = "#1", result = true) protected boolean deepEquals0(@Nullable Object obj) { @@ -311,12 +331,13 @@ protected boolean deepEquals0(@Nullable Object obj) { && input.deepEquals(o.input) && exps.equals(o.exps) && hints.equals(o.hints) - && getRowType().equalsSansFieldNames(o.getRowType()); + && getRowType().equalsSansFieldNames(o.getRowType()) + && variablesSet.equals(o.variablesSet); } @API(since = "1.24", status = API.Status.INTERNAL) protected int deepHashCode0() { - return Objects.hash(traitSet, input.deepHashCode(), exps, hints); + return Objects.hash(traitSet, input.deepHashCode(), exps, hints, variablesSet); } /** diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index 7ef4e6ca8d20..35bba8c54196 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -24,7 +24,6 @@ import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; -import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.hint.RelHint; @@ -42,7 +41,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; -import java.util.Objects; import java.util.Set; /** @@ -50,8 +48,6 @@ * targeted at any particular engine or calling convention. */ public final class LogicalProject extends Project { - private final ImmutableSet variablesSet; - //~ Constructors ----------------------------------------------------------- /** @@ -75,10 +71,9 @@ public LogicalProject( RelNode input, List projects, RelDataType rowType, - ImmutableSet variablesSet) { - super(cluster, traitSet, hints, input, projects, rowType); + Set variablesSet) { + super(cluster, traitSet, hints, input, projects, rowType, variablesSet); assert traitSet.containsIfApplicable(Convention.NONE); - this.variablesSet = Objects.requireNonNull(variablesSet, "variablesSet"); } @Deprecated // to be removed before 2.0 @@ -112,7 +107,6 @@ public LogicalProject(RelOptCluster cluster, RelNode input, */ public LogicalProject(RelInput input) { super(input); - this.variablesSet = ImmutableSet.of(); } //~ Methods ---------------------------------------------------------------- @@ -144,7 +138,7 @@ public static LogicalProject create(final RelNode input, List hints, /** Creates a LogicalProject, specifying row type rather than field names. */ public static LogicalProject create(final RelNode input, List hints, final List projects, RelDataType rowType, - ImmutableSet variablesSet) { + Set variablesSet) { final RelOptCluster cluster = input.getCluster(); final RelMetadataQuery mq = cluster.getMetadataQuery(); final RelTraitSet traitSet = @@ -154,10 +148,6 @@ public static LogicalProject create(final RelNode input, List hints, return new LogicalProject(cluster, traitSet, hints, input, projects, rowType, variablesSet); } - @Override public Set getVariablesSet() { - return variablesSet; - } - @Override public LogicalProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType, @@ -173,17 +163,11 @@ public static LogicalProject create(final RelNode input, List hints, input, getProjects(), getRowType(), variablesSet); } - @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw) - .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); - } - @Override public boolean deepEquals(@Nullable Object obj) { - return deepEquals0(obj) - && variablesSet.equals(((LogicalProject) obj).variablesSet); + return deepEquals0(obj); } @Override public int deepHashCode() { - return Objects.hash(deepHashCode0(), variablesSet); + return deepHashCode0(); } } diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index e0eea6816939..f2599d5922b8 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11756,11 +11756,11 @@ from emp SOME($0, { +LogicalProject(variablesSet=[[$cor0]], EXPR$0=[> SOME($0, { LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($cor0.JOB, $1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) -})], variablesSet=[[$cor0]]) +})]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -11798,11 +11798,11 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS TRUE(>($0, $9)), IS NOT TRUE(OR(IS NULL($1 diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 0301832f4bc6..9fc019c22210 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -593,7 +593,7 @@ LogicalProject(DEPTNO=[$0], ENAME=[$1]) ($cor0.DEPTNO, 0)]) @@ -603,7 +603,7 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) LogicalProject($f0=[0]) LogicalFilter(condition=[AND(>($cor0.DEPTNO, 0), <($cor0.DEPTNO, 10))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -})], variablesSet=[[$cor0]]) +})]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -781,10 +781,10 @@ LogicalProject(DEPTNO=[$7], EXPR$1=[$9]) From ca982a4ecfe64656f525c3eeb0a5f8d9ef072b68 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 3 Dec 2021 17:35:36 +0300 Subject: [PATCH 04/21] rename term to correlate and fix an issue with null checking --- core/src/main/java/org/apache/calcite/rel/core/Project.java | 6 +++--- .../resources/org/apache/calcite/test/RelOptRulesTest.xml | 4 ++-- .../org/apache/calcite/test/SqlToRelConverterTest.xml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 41d2ecc888a2..65bc32fcb82c 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -101,8 +101,8 @@ protected Project( this.exps = ImmutableList.copyOf(projects); this.hints = ImmutableList.copyOf(hints); this.rowType = rowType; - this.variablesSet = Objects.requireNonNull( - ImmutableSet.copyOf(variablesSet), "variablesSet"); + this.variablesSet = ImmutableSet.copyOf( + Objects.requireNonNull(variablesSet, "variablesSet")); assert isValid(Litmus.THROW, null); } @@ -280,7 +280,7 @@ private static int countTrivial(List refs) { @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw) - .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); + .itemIf("correlation", variablesSet, !variablesSet.isEmpty()); // Skip writing field names so the optimizer can reuse the projects that differ in // field names only diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index f2599d5922b8..65eefc6fdc00 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11756,7 +11756,7 @@ from emp SOME($0, { +LogicalProject(correlation=[[$cor0]], EXPR$0=[> SOME($0, { LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($cor0.JOB, $1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) @@ -11798,7 +11798,7 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS TRUE(>($0, $9)), IS NOT TRUE(OR(IS NULL($1 ($cor0.DEPTNO, 0)]) @@ -781,7 +781,7 @@ LogicalProject(DEPTNO=[$7], EXPR$1=[$9]) Date: Wed, 8 Dec 2021 16:36:48 +0300 Subject: [PATCH 05/21] add a test case to sub-query.iq --- core/src/test/resources/sql/sub-query.iq | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 2d4936b696f5..4e534d423334 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -3257,6 +3257,34 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[null:BOOLEAN], proj#0..8=[{exprs}]) EnumerableTableScan(table=[[scott, EMP]]) !plan +# [CALCITE-4913] Correlated variables in a select list are not deduplicated +select e.deptno, + (select count(*) from emp where e.deptno > 0) as c1, + (select count(*) from emp where e.deptno > 0 and e.deptno < 10) as c2 + from emp e; + ++--------+----+----+ +| DEPTNO | C1 | C2 | ++--------+----+----+ +| 10 | 14 | 0 | +| 10 | 14 | 0 | +| 10 | 14 | 0 | +| 20 | 14 | 0 | +| 20 | 14 | 0 | +| 20 | 14 | 0 | +| 20 | 14 | 0 | +| 20 | 14 | 0 | +| 30 | 14 | 0 | +| 30 | 14 | 0 | +| 30 | 14 | 0 | +| 30 | 14 | 0 | +| 30 | 14 | 0 | +| 30 | 14 | 0 | ++--------+----+----+ +(14 rows) + +!ok + # [CALCITE-4844] IN-list that references columns is wrongly converted to Values, and gives incorrect results !set insubquerythreshold 0 From 0c01b1bf6321745453be886e80c527f8f0a7c34b Mon Sep 17 00:00:00 2001 From: korlov42 Date: Mon, 13 Dec 2021 14:41:03 +0300 Subject: [PATCH 06/21] use builder to create correlated project --- .../enumerable/EnumerableRelFactories.java | 6 +- .../calcite/adapter/jdbc/JdbcRules.java | 4 +- .../org/apache/calcite/plan/RelOptUtil.java | 4 +- .../org/apache/calcite/rel/core/Project.java | 16 ++-- .../apache/calcite/rel/core/RelFactories.java | 17 +++- .../calcite/rel/logical/LogicalProject.java | 12 +++ .../calcite/sql2rel/SqlToRelConverter.java | 14 +++- .../org/apache/calcite/tools/RelBuilder.java | 79 ++++++++++++++++++- .../apache/calcite/plan/RelOptUtilTest.java | 7 +- 9 files changed, 135 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index ad109c44d0b4..8d69937c00c7 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -26,6 +26,8 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.validate.SqlValidatorUtil; +import com.google.common.base.Preconditions; + import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -68,7 +70,9 @@ private static class ProjectFactoryImpl implements org.apache.calcite.rel.core.RelFactories.ProjectFactory { @Override public RelNode createProject(RelNode input, List hints, List childExprs, - @Nullable List fieldNames) { + @Nullable List fieldNames, + Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty()); final RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.F_SUGGESTER); diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 37fb747a87d7..cccdbb479b12 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -96,7 +96,9 @@ private JdbcRules() { protected static final Logger LOGGER = CalciteTrace.getPlannerTracer(); static final RelFactories.ProjectFactory PROJECT_FACTORY = - (input, hints, projects, fieldNames) -> { + (input, hints, projects, fieldNames, variablesSet) -> { + Preconditions.checkArgument(variablesSet.isEmpty(), + "JdbcProject does not allow variables"); final RelOptCluster cluster = input.getCluster(); final RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), projects, diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index c7b5c7cbcc78..fd75a1877f05 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -905,11 +905,11 @@ public static RelNode createCastRel( if (rename) { // Use names and types from castRowType. return projectFactory.createProject(input, hints, castExps, - castRowType.getFieldNames()); + castRowType.getFieldNames(), ImmutableSet.of()); } else { // Use names from rowType, types from castRowType. return projectFactory.createProject(input, hints, castExps, - rowType.getFieldNames()); + rowType.getFieldNames(), ImmutableSet.of()); } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 65bc32fcb82c..bb7ab92d918a 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -78,14 +78,14 @@ public abstract class Project extends SingleRel implements Hintable { /** * Creates a Project. * - * @param cluster Cluster that this relational expression belongs to - * @param traits Traits of this relational expression - * @param hints Hints of this relation expression - * @param input Input relational expression - * @param projects List of expressions for the input columns - * @param rowType Output row type - * @param variablesSet Correlation variables set by this relational expression to be used by - * nested expressions + * @param cluster Cluster that this relational expression belongs to + * @param traits Traits of this relational expression + * @param hints Hints of this relation expression + * @param input Input relational expression + * @param projects List of expressions for the input columns + * @param rowType Output row type + * @param variablesSet Correlation variables set by this relational expression + * to be used by nested expressions */ @SuppressWarnings("method.invocation.invalid") protected Project( diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index 6d5e2c0d65ca..87c568a67369 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -162,10 +162,20 @@ public interface ProjectFactory { * @param hints The hints * @param childExprs The projection expressions * @param fieldNames The projection field names + * @param variablesSet Correlating variables that are set when reading + * a row from the input, and which may be referenced from inside the + * projection * @return a project */ RelNode createProject(RelNode input, List hints, - List childExprs, @Nullable List fieldNames); + List childExprs, @Nullable List fieldNames, + Set variablesSet); + + @Deprecated // to be removed before 2.0 + default RelNode createProject(RelNode input, List hints, + List childExprs, @Nullable List fieldNames) { + return createProject(input, hints, childExprs, fieldNames, ImmutableSet.of()); + } } /** @@ -174,8 +184,9 @@ RelNode createProject(RelNode input, List hints, */ private static class ProjectFactoryImpl implements ProjectFactory { @Override public RelNode createProject(RelNode input, List hints, - List childExprs, @Nullable List fieldNames) { - return LogicalProject.create(input, hints, childExprs, fieldNames); + List childExprs, @Nullable List fieldNames, + Set variablesSet) { + return LogicalProject.create(input, hints, childExprs, fieldNames, variablesSet); } } diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index 35bba8c54196..e556c0980c0d 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -122,6 +122,18 @@ public static LogicalProject create(final RelNode input, List hints, return create(input, hints, projects, rowType); } + /** Creates a LogicalProject. */ + public static LogicalProject create(final RelNode input, List hints, + final List projects, + @Nullable List fieldNames, + Set variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelDataType rowType = + RexUtil.createStructType(cluster.getTypeFactory(), projects, + fieldNames, SqlValidatorUtil.F_SUGGESTER); + return create(input, hints, projects, rowType, variablesSet); + } + /** Creates a LogicalProject, specifying row type rather than field names. */ public static LogicalProject create(final RelNode input, List hints, final List projects, RelDataType rowType) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 638f757109c8..da286c67f242 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4393,13 +4393,21 @@ private void convertSelectList( RelNode project = relBuilder.build(); - final RelNode r; + RelNode r; final CorrelationUse p = getCorrelationUse(bb, project); if (p != null) { assert p.r instanceof Project; Project proj = (Project) p.r; - r = LogicalProject.create(proj.getInput(), proj.getHints(), - proj.getProjects(), proj.getRowType(), ImmutableSet.of(p.id)); + r = relBuilder.push(proj.getInput()) + .projectCorrelated(ImmutableSet.of(p.id), proj.getProjects(), + proj.getRowType().getFieldNames(), true) + .build(); + + if (!proj.getHints().isEmpty()) { + assert r instanceof Hintable; + + r = ((Hintable) r).withHints(proj.getHints()); + } } else { r = project; } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 6eb38385b36c..220045dc2ad1 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -1817,7 +1817,74 @@ public RelBuilder project(Iterable nodes, */ public RelBuilder project(Iterable nodes, Iterable fieldNames, boolean force) { - return project_(nodes, fieldNames, ImmutableList.of(), force); + return project_(ImmutableSet.of(), nodes, fieldNames, ImmutableList.of(), force); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given expressions. */ + public RelBuilder projectCorrelated(Iterable variablesSet, + RexNode... nodes) { + return projectCorrelated(variablesSet, ImmutableList.copyOf(nodes)); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions. + * + *

Infers names as would {@link #projectCorrelated(Iterable, Iterable, Iterable)} + * if all suggested names were null. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + */ + public RelBuilder projectCorrelated(Iterable variablesSet, + Iterable nodes) { + return projectCorrelated(variablesSet, nodes, ImmutableList.of()); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions and field names. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + * @param fieldNames field names for expressions + */ + public RelBuilder projectCorrelated( + Iterable variablesSet, + Iterable nodes, + Iterable fieldNames) { + return projectCorrelated(variablesSet, nodes, fieldNames, false); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions, using the given names. + * + *

Names are deduced as follows: + *

    + *
  • If the length of {@code fieldNames} is greater than the index of + * the current entry in {@code nodes}, and the entry in + * {@code fieldNames} is not null, uses it; otherwise + *
  • If an expression projects an input field, + * or is a cast an input field, + * uses the input field name; otherwise + *
  • If an expression is a call to + * {@link SqlStdOperatorTable#AS} + * (see {@link #alias}), removes the call but uses the intended alias. + *
+ * + *

After the field names have been inferred, makes the + * field names unique by appending numeric suffixes. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + * @param fieldNames Suggested field names + * @param force create project even if it is identity + */ + public RelBuilder projectCorrelated( + Iterable variablesSet, + Iterable nodes, + Iterable fieldNames, + boolean force) { + return project_(variablesSet, nodes, fieldNames, ImmutableList.of(), force); } /** Creates a {@link Project} of all original fields, plus the given @@ -1882,12 +1949,14 @@ public RelBuilder projectExcept(Iterable expressions) { *

After the field names have been inferred, makes the * field names unique by appending numeric suffixes. * + * @param variablesSet Set of correlation variables * @param nodes Expressions * @param fieldNames Suggested field names * @param hints Hints * @param force create project even if it is identity */ private RelBuilder project_( + Iterable variablesSet, Iterable nodes, Iterable fieldNames, Iterable hints, @@ -1958,7 +2027,7 @@ private RelBuilder project_( final ImmutableSet.Builder mergedHints = ImmutableSet.builder(); mergedHints.addAll(project.getHints()); mergedHints.addAll(hints); - return project_(newNodes, fieldNameList, mergedHints.build(), force); + return project_(variablesSet, newNodes, fieldNameList, mergedHints.build(), force); } // Simplify expressions. @@ -2043,7 +2112,8 @@ private RelBuilder project_( struct.projectFactory.createProject(frame.rel, ImmutableList.copyOf(hints), ImmutableList.copyOf(nodeList), - fieldNameList); + fieldNameList, + ImmutableSet.copyOf(variablesSet)); stack.pop(); stack.push(new Frame(project, fields.build())); return this; @@ -3278,7 +3348,8 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc struct.projectFactory.createProject(sort, project.getHints(), project.getProjects(), - Pair.right(project.getNamedProjects()))); + Pair.right(project.getNamedProjects()), + project.getVariablesSet())); return this; } } diff --git a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java index 2f22e78ee723..54e9c88d16b2 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java @@ -49,6 +49,7 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -693,7 +694,8 @@ private void splitJoinConditionHelper(RexNode joinCond, List expLeftKey ImmutableList.of( fieldEmpno.getName(), fieldEname.getName(), - "JOB_CNT")); + "JOB_CNT"), + ImmutableSet.of()); assertThat(castNode1.explain(), is(expectNode1.explain())); // Change the field JOB_CNT field name again. // The projection expect to be merged. @@ -716,7 +718,8 @@ private void splitJoinConditionHelper(RexNode joinCond, List expLeftKey ImmutableList.of( fieldEmpno.getName(), fieldEname.getName(), - "JOB_CNT2")); + "JOB_CNT2"), + ImmutableSet.of()); assertThat(castNode2.explain(), is(expectNode2.explain())); } From 005b1aa6a38cf05d0d1d1db21faeec111e687752 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Mon, 13 Dec 2021 15:42:44 +0300 Subject: [PATCH 07/21] revert javadoc formatting --- .../apache/calcite/rel/logical/LogicalProject.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index e556c0980c0d..77175d53a5ce 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -55,12 +55,12 @@ public final class LogicalProject extends Project { * *

Use {@link #create} unless you know what you're doing. * - * @param cluster Cluster this relational expression belongs to - * @param traitSet Traits of this relational expression - * @param hints Hints of this relational expression - * @param input Input relational expression - * @param projects List of expressions for the input columns - * @param rowType Output row type + * @param cluster Cluster this relational expression belongs to + * @param traitSet Traits of this relational expression + * @param hints Hints of this relational expression + * @param input Input relational expression + * @param projects List of expressions for the input columns + * @param rowType Output row type * @param variablesSet Correlation variables set by this relational expression * to be used by nested expressions */ From 8d1714efc671138e30c4d5cef874d31fca6424f7 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Mon, 13 Dec 2021 17:49:06 +0300 Subject: [PATCH 08/21] reuse introduced methods --- .../calcite/rel/logical/LogicalProject.java | 15 ++------------- site/_docs/algebra.md | 1 + 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index 77175d53a5ce..84300ab030bf 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -115,11 +115,7 @@ public LogicalProject(RelInput input) { public static LogicalProject create(final RelNode input, List hints, final List projects, @Nullable List fieldNames) { - final RelOptCluster cluster = input.getCluster(); - final RelDataType rowType = - RexUtil.createStructType(cluster.getTypeFactory(), projects, - fieldNames, SqlValidatorUtil.F_SUGGESTER); - return create(input, hints, projects, rowType); + return create(input, hints, projects, fieldNames, ImmutableSet.of()); } /** Creates a LogicalProject. */ @@ -137,14 +133,7 @@ public static LogicalProject create(final RelNode input, List hints, /** Creates a LogicalProject, specifying row type rather than field names. */ public static LogicalProject create(final RelNode input, List hints, final List projects, RelDataType rowType) { - final RelOptCluster cluster = input.getCluster(); - final RelMetadataQuery mq = cluster.getMetadataQuery(); - final RelTraitSet traitSet = - cluster.traitSet().replace(Convention.NONE) - .replaceIfs(RelCollationTraitDef.INSTANCE, - () -> RelMdCollation.project(mq, input, projects)); - return new LogicalProject(cluster, traitSet, hints, input, projects, rowType, - ImmutableSet.of()); + return create(input, hints, projects, rowType, ImmutableSet.of()); } /** Creates a LogicalProject, specifying row type rather than field names. */ diff --git a/site/_docs/algebra.md b/site/_docs/algebra.md index 66b2792779ea..11342e789f87 100644 --- a/site/_docs/algebra.md +++ b/site/_docs/algebra.md @@ -326,6 +326,7 @@ return the `RelBuilder`. | `values(fieldNames, value...)`
`values(rowType, tupleList)` | Creates a [Values]({{ site.apiRoot }}/org/apache/calcite/rel/core/Values.html). | `filter([variablesSet, ] exprList)`
`filter([variablesSet, ] expr...)` | Creates a [Filter]({{ site.apiRoot }}/org/apache/calcite/rel/core/Filter.html) over the AND of the given predicates; if `variablesSet` is specified, the predicates may reference those variables. | `project(expr...)`
`project(exprList [, fieldNames])` | Creates a [Project]({{ site.apiRoot }}/org/apache/calcite/rel/core/Project.html). To override the default name, wrap expressions using `alias`, or specify the `fieldNames` argument. +| `projectCorrelated(variablesSet, expr...)`
`projectCorrelated(variablesSet, exprList [, fieldNames])` | Variant of `project` that creates correlated project. | `projectPlus(expr...)`
`projectPlus(exprList)` | Variant of `project` that keeps original fields and appends the given expressions. | `projectExcept(expr...)`
`projectExcept(exprList)` | Variant of `project` that keeps original fields and removes the given expressions. | `permute(mapping)` | Creates a [Project]({{ site.apiRoot }}/org/apache/calcite/rel/core/Project.html) that permutes the fields using `mapping`. From 0166561c638d05896d028dca72e57653db5152cb Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 14 Jan 2022 13:03:01 +0300 Subject: [PATCH 09/21] add explanation message and deprecate the constructor in Project --- .../calcite/adapter/enumerable/EnumerableRelFactories.java | 3 ++- core/src/main/java/org/apache/calcite/rel/core/Project.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index 8d69937c00c7..b959f534a09c 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -72,7 +72,8 @@ private static class ProjectFactoryImpl List childExprs, @Nullable List fieldNames, Set variablesSet) { - Preconditions.checkArgument(variablesSet.isEmpty()); + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); final RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.F_SUGGESTER); diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index bb7ab92d918a..d3d63f3801f0 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -106,6 +106,7 @@ protected Project( assert isValid(Litmus.THROW, null); } + @Deprecated // to be removed before 2.0 protected Project(RelOptCluster cluster, RelTraitSet traits, List hints, RelNode input, List projects, RelDataType rowType) { this(cluster, traits, hints, input, projects, rowType, ImmutableSet.of()); From a8f9e81525dd2ab0a26f13d3f6729706b07add38 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 14 Jan 2022 15:41:13 +0300 Subject: [PATCH 10/21] deprecate an old constructor and extend a Project#copy with a new param --- .../adapter/cassandra/CassandraProject.java | 10 ++- .../adapter/cassandra/CassandraRules.java | 3 +- .../adapter/enumerable/EnumerableProject.java | 10 ++- .../enumerable/EnumerableProjectRule.java | 3 +- .../calcite/adapter/jdbc/JdbcRules.java | 7 +- .../apache/calcite/interpreter/Bindables.java | 10 ++- .../org/apache/calcite/rel/core/Project.java | 13 +++- .../calcite/rel/logical/LogicalProject.java | 2 +- .../rel/rules/FilterProjectTransposeRule.java | 2 +- .../rel/rules/ProjectJoinRemoveRule.java | 4 +- .../calcite/rel/rules/ProjectRemoveRule.java | 2 +- .../rel/rules/ProjectWindowTransposeRule.java | 3 +- .../calcite/sql2rel/SqlToRelConverter.java | 12 +-- .../org/apache/calcite/tools/RelBuilder.java | 5 +- .../plan/volcano/TraitPropagationTest.java | 10 ++- .../apache/calcite/test/RelOptRulesTest.java | 10 ++- .../calcite/test/SqlToRelConverterTest.xml | 78 ++++++++++--------- .../calcite/adapter/druid/DruidRules.java | 6 +- .../elasticsearch/ElasticsearchProject.java | 10 ++- .../elasticsearch/ElasticsearchRules.java | 3 +- .../adapter/geode/rel/GeodeProject.java | 10 ++- .../calcite/adapter/geode/rel/GeodeRules.java | 3 +- .../calcite/adapter/innodb/InnodbProject.java | 10 ++- .../calcite/adapter/innodb/InnodbRules.java | 3 +- .../calcite/adapter/mongodb/MongoProject.java | 10 ++- .../calcite/adapter/mongodb/MongoRules.java | 3 +- .../calcite/adapter/pig/PigProject.java | 10 ++- .../apache/calcite/adapter/pig/PigRules.java | 3 +- 28 files changed, 166 insertions(+), 89 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java index b5a77c74f829..bbe5c6766bc5 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java @@ -21,19 +21,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -42,13 +46,15 @@ public class CassandraProject extends Project implements CassandraRel { public CassandraProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == CassandraRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new CassandraProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index ce38ed3d18d2..ec45ec1be45f 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -258,7 +258,8 @@ public interface CassandraFilterRuleConfig extends RelRule.Config { public static class CassandraProjectRule extends CassandraConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, CassandraRel.CONVENTION, "CassandraProjectRule") .withRuleFactory(CassandraProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java index 4bc516c520a8..cd62846cef93 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMdCollation; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -28,11 +29,14 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Set; /** Implementation of {@link org.apache.calcite.rel.core.Project} in * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ @@ -54,7 +58,7 @@ public EnumerableProject( RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof EnumerableConvention; } @@ -80,7 +84,9 @@ public static EnumerableProject create(final RelNode input, } @Override public EnumerableProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new EnumerableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java index b1b69220a629..e7f4df4bf17e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java @@ -32,7 +32,8 @@ class EnumerableProjectRule extends ConverterRule { /** Default configuration. */ static final Config DEFAULT_CONFIG = Config.INSTANCE .as(Config.class) - .withConversion(LogicalProject.class, p -> !p.containsOver(), + .withConversion(LogicalProject.class, p -> !p.containsOver() + && (p.getCorrelVariable() == null || p.getVariablesSet().isEmpty()), Convention.NONE, EnumerableConvention.INSTANCE, "EnumerableProjectRule") .withRuleFactory(EnumerableProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index cccdbb479b12..672a86972690 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -73,6 +73,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -534,7 +535,7 @@ public JdbcProject( RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof JdbcConvention; } @@ -546,7 +547,9 @@ public JdbcProject(RelOptCluster cluster, RelTraitSet traitSet, } @Override public JdbcProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new JdbcProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index 77b5287fc24b..4bd6a3bc71e6 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -78,6 +78,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; @@ -376,7 +377,8 @@ public static BindableFilter create(final RelNode input, public static class BindableProjectRule extends ConverterRule { /** Default configuration. */ public static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, p -> !p.containsOver(), + .withConversion(LogicalProject.class, p -> !p.containsOver() + && (p.getCorrelVariable() == null || p.getVariablesSet().isEmpty()), Convention.NONE, BindableConvention.INSTANCE, "BindableProjectRule") .withRuleFactory(BindableProjectRule::new); @@ -403,12 +405,14 @@ protected BindableProjectRule(Config config) { public static class BindableProject extends Project implements BindableRel { public BindableProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof BindableConvention; } @Override public BindableProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new BindableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index d3d63f3801f0..24c1252c8fc1 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -142,7 +142,7 @@ protected Project(RelInput input) { @Override public final RelNode copy(RelTraitSet traitSet, List inputs) { - return copy(traitSet, sole(inputs), exps, getRowType()); + return copy(traitSet, sole(inputs), exps, getRowType(), variablesSet); } /** @@ -152,6 +152,7 @@ protected Project(RelInput input) { * @param input Input * @param projects Project expressions * @param rowType Output row type + * @param variablesSet Correlation variables * @return New {@code Project} if any parameter differs from the value of this * {@code Project}, or just {@code this} if all the parameters are * the same @@ -159,7 +160,13 @@ protected Project(RelInput input) { * @see #copy(RelTraitSet, List) */ public abstract Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType); + List projects, RelDataType rowType, Set variablesSet); + + @Deprecated // to be removed before 2.0 + public Project copy(RelTraitSet traitSet, RelNode input, + List projects, RelDataType rowType) { + return copy(traitSet, input, projects, rowType, ImmutableSet.of()); + } @Deprecated // to be removed before 2.0 public Project copy(RelTraitSet traitSet, RelNode input, @@ -184,7 +191,7 @@ public boolean isBoxed() { exps, getRowType().getFieldNames(), null); - return copy(traitSet, getInput(), exps, rowType); + return copy(traitSet, getInput(), exps, rowType, variablesSet); } /** diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index 84300ab030bf..bc53ed840d9f 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -150,7 +150,7 @@ public static LogicalProject create(final RelNode input, List hints, } @Override public LogicalProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType, variablesSet); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index 13665c4809e8..6fdbbaf4dee2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -187,7 +187,7 @@ protected FilterProjectTransposeRule( RelNode newProject = config.isCopyProject() ? project.copy(project.getTraitSet(), newFilterRel, - project.getProjects(), project.getRowType()) + project.getProjects(), project.getRowType(), project.getVariablesSet()) : relBuilder.push(newFilterRel) .project(project.getProjects(), project.getRowType().getFieldNames()) .build(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java index 5ecd6ccdfee6..b8ea42b66b6a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java @@ -114,14 +114,14 @@ public ProjectJoinRemoveRule( if (isLeftJoin) { node = project .copy(project.getTraitSet(), join.getLeft(), project.getProjects(), - project.getRowType()); + project.getRowType(), project.getVariablesSet()); } else { final int offset = join.getLeft().getRowType().getFieldCount(); final List newExprs = project.getProjects().stream() .map(expr -> RexUtil.shift(expr, -offset)) .collect(Collectors.toList()); node = project.copy(project.getTraitSet(), join.getRight(), newExprs, - project.getRowType()); + project.getRowType(), project.getVariablesSet()); } call.transformTo(node); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java index 67df2031622e..a1b04f347026 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java @@ -64,7 +64,7 @@ public ProjectRemoveRule(RelBuilderFactory relBuilderFactory) { Project childProject = (Project) stripped; stripped = childProject.copy(childProject.getTraitSet(), childProject.getInput(), childProject.getProjects(), - project.getRowType()); + project.getRowType(), childProject.getVariablesSet()); } stripped = convert(stripped, project.getConvention()); call.transformTo(stripped); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index 5512459076f9..a0c09ea1be9d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -183,7 +183,8 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { newLogicalWindow.getTraitSet(), newLogicalWindow, topProjExps, - project.getRowType()); + project.getRowType(), + projectBelowWindow.getVariablesSet()); if (ProjectRemoveRule.isTrivial(newTopProj)) { call.transformTo(newLogicalWindow); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index da286c67f242..a12c79c4a743 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4398,16 +4398,8 @@ private void convertSelectList( if (p != null) { assert p.r instanceof Project; Project proj = (Project) p.r; - r = relBuilder.push(proj.getInput()) - .projectCorrelated(ImmutableSet.of(p.id), proj.getProjects(), - proj.getRowType().getFieldNames(), true) - .build(); - - if (!proj.getHints().isEmpty()) { - assert r instanceof Hintable; - - r = ((Hintable) r).withHints(proj.getHints()); - } + r = proj.copy(proj.getTraitSet(), proj.getInput(), proj.getProjects(), + proj.getRowType(), ImmutableSet.of(p.id)); } else { r = project; } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 220045dc2ad1..87cfd2724f32 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -2155,7 +2155,8 @@ public RelBuilder projectNamed(Iterable nodes, final Frame frame = stack.pop(); final Project childProject = (Project) frame.rel; final Project newInput = childProject.copy(childProject.getTraitSet(), - childProject.getInput(), childProject.getProjects(), rowType); + childProject.getInput(), childProject.getProjects(), rowType, + childProject.getVariablesSet()); stack.push(new Frame(newInput.attachHints(childProject.getHints()), frame.fields)); } if (input instanceof Values && fieldNameList != null) { @@ -2415,7 +2416,7 @@ public RelBuilder aggregate(GroupKey groupKey, Iterable aggCalls) { builder.add(project.getRowType().getFieldList().get(i)); } r = project.copy(cluster.traitSet(), project.getInput(), newProjects, - builder.build()); + builder.build(), project.getVariablesSet()); } } diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index 825e1434691a..4ceffe26d919 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -43,6 +43,7 @@ import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -71,7 +72,9 @@ import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; @@ -82,6 +85,7 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -370,7 +374,7 @@ public Aggregate copy(RelTraitSet traitSet, RelNode input, private static class PhysProj extends Project implements Phys { PhysProj(RelOptCluster cluster, RelTraitSet traits, RelNode child, List exps, RelDataType rowType) { - super(cluster, traits, ImmutableList.of(), child, exps, rowType); + super(cluster, traits, ImmutableList.of(), child, exps, rowType, ImmutableSet.of()); } public static PhysProj create(final RelNode input, @@ -386,7 +390,9 @@ public static PhysProj create(final RelNode input, } public PhysProj copy(RelTraitSet traitSet, RelNode input, - List exps, RelDataType rowType) { + List exps, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new PhysProj(getCluster(), traitSet, input, exps, rowType); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index dc4b1a41d5e7..ee7b9b2d17d4 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -125,7 +125,10 @@ import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; import org.junit.jupiter.api.Disabled; @@ -136,6 +139,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; @@ -6745,11 +6749,13 @@ private static class MyProject extends Project { RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); } public MyProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new MyProject(getCluster(), traitSet, input, projects, rowType); } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index fef60f528673..8ea11c102196 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -587,40 +587,6 @@ LogicalProject(DEPTNO=[$0], ENAME=[$1]) LogicalProject(ENAME=[$1], SAL=[$5]) LogicalFilter(condition=[=($7, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -]]> - - - - - ($cor0.DEPTNO, 0)]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -})], EXPR$2=[$SCALAR_QUERY({ -LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) - LogicalProject($f0=[0]) - LogicalFilter(condition=[AND(>($cor0.DEPTNO, 0), <($cor0.DEPTNO, 10))]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -})]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -]]> - - - ($cor0.DEPTNO, 0)]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) - LogicalProject($f0=[0]) - LogicalFilter(condition=[AND(>($cor1.DEPTNO, 0), <($cor1.DEPTNO, 10))]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> @@ -698,6 +664,46 @@ LogicalProject(EMPNO=[$0], JOB=[$2]) ]]> + + + ($cor0.DEPTNO, 0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})], EXPR$2=[$SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(>($cor0.DEPTNO, 0), <($cor0.DEPTNO, 10))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + ($cor0.DEPTNO, 0)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(>($cor1.DEPTNO, 0), <($cor1.DEPTNO, 10))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + 0), + (select count(*) from emp where e.deptno > 0 and e.deptno < 10) +from emp e]]> + + projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == ElasticsearchRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet relTraitSet, RelNode input, List projects, - RelDataType relDataType) { + RelDataType relDataType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new ElasticsearchProject(getCluster(), traitSet, input, projects, relDataType); } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 78261b9ae59b..88832094662e 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -284,7 +284,8 @@ protected ElasticsearchAggregateRule(Config config) { */ private static class ElasticsearchProjectRule extends ElasticsearchConverterRule { private static final ElasticsearchProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, ElasticsearchRel.CONVENTION, "ElasticsearchProjectRule") .withRuleFactory(ElasticsearchProjectRule::new) .toRule(ElasticsearchProjectRule.class); diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java index da5cf40a7399..04ae3f8b635b 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java @@ -22,19 +22,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of @@ -45,13 +49,15 @@ public class GeodeProject extends Project implements GeodeRel { GeodeProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == GeodeRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new GeodeProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index 5fd12fa3e67f..b933ded078d2 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -128,7 +128,8 @@ private static String stripQuotes(String s) { */ private static class GeodeProjectRule extends GeodeConverterRule { private static final GeodeProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, GeodeRel.CONVENTION, "GeodeProjectRule") .withRuleFactory(GeodeProjectRule::new) .toRule(GeodeProjectRule.class); diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java index b7e657ac07df..9e0e8dbf60a3 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java @@ -21,19 +21,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -42,13 +46,15 @@ public class InnodbProject extends Project implements InnodbRel { InnodbProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == InnodbRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new InnodbProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java index cf381de24715..66de0afc4198 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java @@ -125,7 +125,8 @@ abstract static class InnodbConverterRule extends ConverterRule { public static class InnodbProjectRule extends InnodbConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, InnodbRel.CONVENTION, "InnodbProjectRule") .withRuleFactory(InnodbProjectRule::new); diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index 55bce98e56ea..f418a759bbb3 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; @@ -29,12 +30,15 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -43,7 +47,7 @@ public class MongoProject extends Project implements MongoRel { public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == MongoRel.CONVENTION; assert getConvention() == input.getConvention(); } @@ -56,7 +60,9 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new MongoProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 9e9ac37fca57..c1e0b1447f70 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -301,7 +301,8 @@ private static class MongoFilterRule extends MongoConverterRule { */ private static class MongoProjectRule extends MongoConverterRule { static final MongoProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, MongoRel.CONVENTION, "MongoProjectRule") .withRuleFactory(MongoProjectRule::new) .toRule(MongoProjectRule.class); diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java index 11d2e90fbd63..0e2ad9fb4327 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java @@ -20,13 +20,17 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Set; /** Implementation of {@link org.apache.calcite.rel.core.Project} in * {@link PigRel#CONVENTION Pig calling convention}. */ @@ -35,12 +39,14 @@ public class PigProject extends Project implements PigRel { /** Creates a PigProject. */ public PigProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == PigRel.CONVENTION; } @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, - RelDataType rowType) { + RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Scalar subqueries is not supported"); return new PigProject(input.getCluster(), traitSet, input, projects, rowType); } diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java index 3def5e0c7203..54abdf01e0a0 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java @@ -99,7 +99,8 @@ protected PigTableScanRule(Config config) { */ private static class PigProjectRule extends ConverterRule { private static final PigProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + || p.getVariablesSet().isEmpty(), Convention.NONE, PigRel.CONVENTION, "PigProjectRule") .withRuleFactory(PigProjectRule::new) .toRule(PigProjectRule.class); From 19f1c0a951b4eb1a4da8b7a9dee0b5eb12feccd7 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 14 Jan 2022 16:10:14 +0300 Subject: [PATCH 11/21] fix missed correlation copy during structured types flattening --- .../adapter/cassandra/CassandraProject.java | 2 +- .../adapter/enumerable/EnumerableProject.java | 2 +- .../adapter/enumerable/EnumerableRelFactories.java | 2 +- .../org/apache/calcite/adapter/jdbc/JdbcRules.java | 2 +- .../org/apache/calcite/interpreter/Bindables.java | 2 +- .../sql2rel/RelStructuredTypeFlattener.java | 14 +++++++++++++- .../calcite/plan/volcano/TraitPropagationTest.java | 2 +- .../org/apache/calcite/test/RelOptRulesTest.java | 2 +- .../apache/calcite/test/SqlToRelConverterTest.xml | 8 ++++---- .../elasticsearch/ElasticsearchProject.java | 2 +- .../calcite/adapter/geode/rel/GeodeProject.java | 2 +- .../calcite/adapter/innodb/InnodbProject.java | 2 +- .../calcite/adapter/mongodb/MongoProject.java | 2 +- .../org/apache/calcite/adapter/pig/PigProject.java | 2 +- 14 files changed, 29 insertions(+), 17 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java index bbe5c6766bc5..6ad07a712d76 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java @@ -54,7 +54,7 @@ public CassandraProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new CassandraProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java index cd62846cef93..379406ec2d32 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java @@ -86,7 +86,7 @@ public static EnumerableProject create(final RelNode input, @Override public EnumerableProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new EnumerableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index b959f534a09c..cca0a56afc5e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -73,7 +73,7 @@ private static class ProjectFactoryImpl @Nullable List fieldNames, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); final RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.F_SUGGESTER); diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 672a86972690..6ba604e8d18e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -549,7 +549,7 @@ public JdbcProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public JdbcProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new JdbcProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index 4bd6a3bc71e6..309058f81998 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -412,7 +412,7 @@ public BindableProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public BindableProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new BindableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index f4bf28aa245c..90c4fc27c8f1 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -27,6 +27,7 @@ import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Collect; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sample; import org.apache.calcite.rel.core.Sort; @@ -534,10 +535,21 @@ public void rewriteRel(LogicalProject rel) { RelNode newInput = getNewForOldRel(rel.getInput()); List newProjects = Pair.left(flattenedExpList); List newNames = Pair.right(flattenedExpList); - final RelNode newRel = relBuilder.push(newInput) + RelNode renamedProject = relBuilder.push(newInput) .projectNamed(newProjects, newNames, true) .hints(rel.getHints()) .build(); + + final RelNode newRel; + // we need to check if builder returns an instance of project + // because in some case a Values relation could be returned instead + if (!rel.getVariablesSet().isEmpty() && renamedProject instanceof Project) { + Project p = (Project) renamedProject; + newRel = p.copy(p.getTraitSet(), p.getInput(), p.getProjects(), + p.getRowType(), rel.getVariablesSet()); + } else { + newRel = renamedProject; + } setNewForOldRel(rel, newRel); } diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index 4ceffe26d919..b34a269fa9ff 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -392,7 +392,7 @@ public static PhysProj create(final RelNode input, public PhysProj copy(RelTraitSet traitSet, RelNode input, List exps, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new PhysProj(getCluster(), traitSet, input, exps, rowType); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index ee7b9b2d17d4..1fa627917e0b 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -6755,7 +6755,7 @@ private static class MyProject extends Project { public MyProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new MyProject(getCluster(), traitSet, input, projects, rowType); } } diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 8ea11c102196..80fd6159aefe 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -618,7 +618,7 @@ from dept]]> projects, RelDataType relDataType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new ElasticsearchProject(getCluster(), traitSet, input, projects, relDataType); } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java index 04ae3f8b635b..9cb14f41aaa6 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java @@ -57,7 +57,7 @@ public class GeodeProject extends Project implements GeodeRel { @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new GeodeProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java index 9e0e8dbf60a3..863d328c0cee 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java @@ -54,7 +54,7 @@ public class InnodbProject extends Project implements InnodbRel { @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new InnodbProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index f418a759bbb3..cf3c10ebe27d 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -62,7 +62,7 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new MongoProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java index 0e2ad9fb4327..96ddae11b2e8 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java @@ -46,7 +46,7 @@ public PigProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Scalar subqueries is not supported"); + "Correlated scalar subqueries is not supported"); return new PigProject(input.getCluster(), traitSet, input, projects, rowType); } From 6e01e6d2e677c0b233a6d50a8a00498188ab290b Mon Sep 17 00:00:00 2001 From: korlov42 Date: Fri, 28 Jan 2022 11:48:02 +0300 Subject: [PATCH 12/21] fixes after merge --- .../java/org/apache/calcite/test/SqlToRelConverterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 3b183a114a78..b35c6af5ac6b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1290,9 +1290,9 @@ class SqlToRelConverterTest extends SqlToRelTestBase { @Test void testCorrelatedScalarSubQueryInSelectList() { Consumer fn = sql -> { - sql(sql).expand(true).decorrelate(false) + sql(sql).withExpand(true).withDecorrelate(false) .convertsTo("${planExpanded}"); - sql(sql).expand(false).decorrelate(false) + sql(sql).withExpand(false).withDecorrelate(false) .convertsTo("${planNotExpanded}"); }; fn.accept("select e.deptno,\n" From fc271487c989d48a1d287de172055196cfa668e3 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Tue, 15 Mar 2022 12:10:53 +0300 Subject: [PATCH 13/21] fix grammar --- .../org/apache/calcite/adapter/cassandra/CassandraProject.java | 2 +- .../apache/calcite/adapter/enumerable/EnumerableProject.java | 2 +- .../calcite/adapter/enumerable/EnumerableRelFactories.java | 2 +- .../main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java | 2 +- .../src/main/java/org/apache/calcite/interpreter/Bindables.java | 2 +- .../org/apache/calcite/plan/volcano/TraitPropagationTest.java | 2 +- core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchProject.java | 2 +- .../java/org/apache/calcite/adapter/geode/rel/GeodeProject.java | 2 +- .../java/org/apache/calcite/adapter/innodb/InnodbProject.java | 2 +- .../java/org/apache/calcite/adapter/mongodb/MongoProject.java | 2 +- .../main/java/org/apache/calcite/adapter/pig/PigProject.java | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java index 6ad07a712d76..fdc094e79c88 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java @@ -54,7 +54,7 @@ public CassandraProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new CassandraProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java index 379406ec2d32..287c96e9ba89 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java @@ -86,7 +86,7 @@ public static EnumerableProject create(final RelNode input, @Override public EnumerableProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new EnumerableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index cca0a56afc5e..b86ea3e9cc28 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -73,7 +73,7 @@ private static class ProjectFactoryImpl @Nullable List fieldNames, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); final RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.F_SUGGESTER); diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 6ba604e8d18e..bcc60715fd24 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -549,7 +549,7 @@ public JdbcProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public JdbcProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new JdbcProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index 309058f81998..954876a7f9ba 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -412,7 +412,7 @@ public BindableProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public BindableProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new BindableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index b34a269fa9ff..d8e0781b4caf 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -392,7 +392,7 @@ public static PhysProj create(final RelNode input, public PhysProj copy(RelTraitSet traitSet, RelNode input, List exps, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new PhysProj(getCluster(), traitSet, input, exps, rowType); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 1fa627917e0b..2ce5a183a1c6 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -6755,7 +6755,7 @@ private static class MyProject extends Project { public MyProject copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new MyProject(getCluster(), traitSet, input, projects, rowType); } } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java index 0d68fba26708..c06295b3efe4 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java @@ -55,7 +55,7 @@ public class ElasticsearchProject extends Project implements ElasticsearchRel { @Override public Project copy(RelTraitSet relTraitSet, RelNode input, List projects, RelDataType relDataType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new ElasticsearchProject(getCluster(), traitSet, input, projects, relDataType); } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java index 9cb14f41aaa6..48f2fbe887d7 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java @@ -57,7 +57,7 @@ public class GeodeProject extends Project implements GeodeRel { @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new GeodeProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java index 863d328c0cee..92c8ec29e743 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java @@ -54,7 +54,7 @@ public class InnodbProject extends Project implements InnodbRel { @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new InnodbProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index cf3c10ebe27d..b6ff2bc9e970 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -62,7 +62,7 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new MongoProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java index 96ddae11b2e8..518c54d8164a 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java @@ -46,7 +46,7 @@ public PigProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, Set variablesSet) { Preconditions.checkArgument(variablesSet.isEmpty(), - "Correlated scalar subqueries is not supported"); + "Correlated scalar subqueries are not supported"); return new PigProject(input.getCluster(), traitSet, input, projects, rowType); } From 807de98092747b00b594d0d828f0c4a958e33017 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Tue, 15 Mar 2022 14:35:48 +0300 Subject: [PATCH 14/21] review fixes - fix corr variables propagation within createCastRel - improve javadocs - fix project (de-)serialization - return back and deprecate constructor for LogicalProject --- .../org/apache/calcite/plan/RelOptUtil.java | 6 +- .../java/org/apache/calcite/rel/RelNode.java | 3 - .../org/apache/calcite/rel/core/Project.java | 15 ++++- .../calcite/rel/externalize/RelJson.java | 6 ++ .../calcite/rel/logical/LogicalProject.java | 11 ++++ .../apache/calcite/plan/RelWriterTest.java | 41 +++++++++++++ .../apache/calcite/test/RelBuilderTest.java | 60 +++++++++++++++++++ 7 files changed, 136 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index fd75a1877f05..d7739ea4ae34 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -885,6 +885,7 @@ public static RelNode createCastRel( List castExps; RelNode input; List hints = ImmutableList.of(); + Set variablesSet = ImmutableSet.of(); if (rel instanceof Project) { // No need to create another project node if the rel // is already a project. @@ -894,6 +895,7 @@ public static RelNode createCastRel( castRowType, ((Project) rel).getProjects()); input = rel.getInput(0); + variablesSet = project.getVariablesSet(); hints = project.getHints(); } else { castExps = RexUtil.generateCastExpressions( @@ -905,11 +907,11 @@ public static RelNode createCastRel( if (rename) { // Use names and types from castRowType. return projectFactory.createProject(input, hints, castExps, - castRowType.getFieldNames(), ImmutableSet.of()); + castRowType.getFieldNames(), variablesSet); } else { // Use names from rowType, types from castRowType. return projectFactory.createProject(input, hints, castExps, - rowType.getFieldNames(), ImmutableSet.of()); + rowType.getFieldNames(), variablesSet); } } diff --git a/core/src/main/java/org/apache/calcite/rel/RelNode.java b/core/src/main/java/org/apache/calcite/rel/RelNode.java index 21f6586cc2fa..ef3b878ce704 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/RelNode.java @@ -152,9 +152,6 @@ public interface RelNode extends RelOptNode, Cloneable { * expression but also used and therefore not available to parents of this * relational expression. * - *

Note: only {@link org.apache.calcite.rel.core.Correlate} should set - * variables. - * * @return Names of variables which are set in this relational * expression */ diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 24c1252c8fc1..00124df7b130 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -54,6 +54,7 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -71,6 +72,12 @@ public abstract class Project extends SingleRel implements Hintable { protected final ImmutableList hints; + /** + * Correlation variables set by this relational expression to be used by + * nested expressions. It's expected to be used in the following way: + * first read the row from input, set it to the appropriate correlation + * variables in the context, then execute the Rex expressions. + */ protected final ImmutableSet variablesSet; //~ Constructors ----------------------------------------------------------- @@ -135,7 +142,13 @@ protected Project(RelInput input) { input.getInput(), requireNonNull(input.getExpressionList("exprs"), "exprs"), input.getRowType("exprs", "fields"), - ImmutableSet.of()); + ImmutableSet.copyOf( + Util.transform( + Optional.ofNullable(input.getIntegerList("correlation")) + .orElse(ImmutableList.of()), + CorrelationId::new + ) + )); } //~ Methods ---------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index d39947dfdfec..d50c77c45483 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -402,6 +402,12 @@ public Object toJson(AggregateCall node) { list.add(toJson(o)); } return list; + } else if (value instanceof Set) { + final List<@Nullable Object> list = jsonBuilder().list(); + for (Object o : (Set) value) { + list.add(toJson(o)); + } + return list; } else if (value instanceof ImmutableBitSet) { final List<@Nullable Object> list = jsonBuilder().list(); for (Integer integer : (ImmutableBitSet) value) { diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index bc53ed840d9f..c12899e010e2 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -76,6 +76,17 @@ public LogicalProject( assert traitSet.containsIfApplicable(Convention.NONE); } + @Deprecated // to be removed before 2.0 + public LogicalProject( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelNode input, + List projects, + RelDataType rowType) { + this(cluster, traitSet, hints, input, projects, rowType, ImmutableSet.of()); + } + @Deprecated // to be removed before 2.0 public LogicalProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index f2277fdc4141..0b310116d965 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttleImpl; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; @@ -612,6 +613,46 @@ private static Fixture relFn(Function relFn) { .assertThatPlan(isLinux(expected)); } + @Test void testProject() { + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder builder = RelBuilder.create(config); + final RelNode rel = builder + .scan("EMP") + .project(builder.alias(builder.field(0), "a")) + .build(); + final String relJson = RelOptUtil.dumpPlan("", rel, + SqlExplainFormat.JSON, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + String s = deserializeAndDumpToTextFormat(getSchema(rel), relJson); + final String expected = "" + + "LogicalProject(a=[$0])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(s, isLinux(expected)); + } + + @Test void testCorrelatedProject() { + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder builder = RelBuilder.create(config); + final CorrelationId corr = builder.getCluster().createCorrel(); + final RelNode rel = builder + .scan("EMP") + .projectCorrelated( + ImmutableSet.of(corr), + builder.alias( + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.VARCHAR), corr), + "a" + ) + ) + .build(); + final String relJson = RelOptUtil.dumpPlan("", rel, + SqlExplainFormat.JSON, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + String s = deserializeAndDumpToTextFormat(getSchema(rel), relJson); + final String expected = "" + + "LogicalProject(correlation=[[$cor0]], a=[$cor0])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(s, isLinux(expected)); + } + @Test public void testExchangeWithDistributionTraitDef() { final Function relFn = b -> b.scan("EMP") diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 88a7cbad4236..943266e57621 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -1191,6 +1191,66 @@ private RexNode caseCall(RelBuilder b, RexNode ref, RexNode... nodes) { assertThat(root, hasTree(expected)); } + @Test void testConvertCorrelatedProject() { + final RelBuilder builder = RelBuilder.create(config().build()); + + CorrelationId cor1 = builder.getCluster().createCorrel(); + CorrelationId cor2 = builder.getCluster().createCorrel(); + + RelDataType rowType = + builder.getTypeFactory().builder() + .add("a", SqlTypeName.BIGINT) + .add("b", SqlTypeName.BIGINT) + .build(); + + RelNode root = + builder + .scan("DEPT") + .projectCorrelated( + ImmutableSet.of(cor1, cor2), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor1), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor2) + ) + .convert(rowType, false) + .build(); + final String expected = "" + + "LogicalProject(correlation=[[$cor0, $cor1]], $f0=[CAST($cor0):BIGINT NOT NULL], $f1=[CAST($cor1):BIGINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); + } + + @Test void testConvertCorrelatedProjectWithRename() { + final RelBuilder builder = RelBuilder.create(config().build()); + + CorrelationId cor1 = builder.getCluster().createCorrel(); + CorrelationId cor2 = builder.getCluster().createCorrel(); + + RelDataType rowType = + builder.getTypeFactory().builder() + .add("a", SqlTypeName.BIGINT) + .add("b", SqlTypeName.BIGINT) + .build(); + + RelNode root = + builder + .scan("DEPT") + .projectCorrelated( + ImmutableSet.of(cor1, cor2), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor1), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor2) + ) + .convert(rowType, true) + .build(); + final String expected = "" + + "LogicalProject(correlation=[[$cor0, $cor1]], a=[CAST($cor0):BIGINT NOT NULL], b=[CAST($cor1):BIGINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); + } + /** Test case for * [CALCITE-4429] * RelOptUtil#createCastRel should throw an exception when the desired row type From 3a56c1f03300591ef775dd21f44d05ccafdcb0d1 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Tue, 15 Mar 2022 17:41:34 +0300 Subject: [PATCH 15/21] review fixes - fixed project convertor rule predicate - fixed lost corr variables propagation in ProjectWindowTransposeRule --- .../org/apache/calcite/adapter/cassandra/CassandraRules.java | 2 +- .../calcite/adapter/enumerable/EnumerableProjectRule.java | 2 +- .../src/main/java/org/apache/calcite/interpreter/Bindables.java | 2 +- .../apache/calcite/rel/rules/ProjectWindowTransposeRule.java | 2 +- .../calcite/adapter/elasticsearch/ElasticsearchRules.java | 2 +- .../java/org/apache/calcite/adapter/geode/rel/GeodeRules.java | 2 +- .../java/org/apache/calcite/adapter/innodb/InnodbRules.java | 2 +- .../java/org/apache/calcite/adapter/mongodb/MongoRules.java | 2 +- pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index ec45ec1be45f..8398dddac515 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -259,7 +259,7 @@ public static class CassandraProjectRule extends CassandraConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, CassandraRel.CONVENTION, "CassandraProjectRule") .withRuleFactory(CassandraProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java index e7f4df4bf17e..b7497ef0217b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java @@ -33,7 +33,7 @@ class EnumerableProjectRule extends ConverterRule { static final Config DEFAULT_CONFIG = Config.INSTANCE .as(Config.class) .withConversion(LogicalProject.class, p -> !p.containsOver() - && (p.getCorrelVariable() == null || p.getVariablesSet().isEmpty()), + && p.getCorrelVariable() == null && p.getVariablesSet().isEmpty(), Convention.NONE, EnumerableConvention.INSTANCE, "EnumerableProjectRule") .withRuleFactory(EnumerableProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index 954876a7f9ba..43a4d69ef730 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -378,7 +378,7 @@ public static class BindableProjectRule extends ConverterRule { /** Default configuration. */ public static final Config DEFAULT_CONFIG = Config.INSTANCE .withConversion(LogicalProject.class, p -> !p.containsOver() - && (p.getCorrelVariable() == null || p.getVariablesSet().isEmpty()), + && p.getCorrelVariable() == null && p.getVariablesSet().isEmpty(), Convention.NONE, BindableConvention.INSTANCE, "BindableProjectRule") .withRuleFactory(BindableProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index a0c09ea1be9d..ed69ef357000 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -184,7 +184,7 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { newLogicalWindow, topProjExps, project.getRowType(), - projectBelowWindow.getVariablesSet()); + project.getVariablesSet()); if (ProjectRemoveRule.isTrivial(newTopProj)) { call.transformTo(newLogicalWindow); diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 88832094662e..b7a8473ad2c5 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -285,7 +285,7 @@ protected ElasticsearchAggregateRule(Config config) { private static class ElasticsearchProjectRule extends ElasticsearchConverterRule { private static final ElasticsearchProjectRule INSTANCE = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, ElasticsearchRel.CONVENTION, "ElasticsearchProjectRule") .withRuleFactory(ElasticsearchProjectRule::new) .toRule(ElasticsearchProjectRule.class); diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index b933ded078d2..4cf376200726 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -129,7 +129,7 @@ private static String stripQuotes(String s) { private static class GeodeProjectRule extends GeodeConverterRule { private static final GeodeProjectRule INSTANCE = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, GeodeRel.CONVENTION, "GeodeProjectRule") .withRuleFactory(GeodeProjectRule::new) .toRule(GeodeProjectRule.class); diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java index 66de0afc4198..c64356ea4c6b 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java @@ -126,7 +126,7 @@ public static class InnodbProjectRule extends InnodbConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, InnodbRel.CONVENTION, "InnodbProjectRule") .withRuleFactory(InnodbProjectRule::new); diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index c1e0b1447f70..43ba8e94608b 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -302,7 +302,7 @@ private static class MongoFilterRule extends MongoConverterRule { private static class MongoProjectRule extends MongoConverterRule { static final MongoProjectRule INSTANCE = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, MongoRel.CONVENTION, "MongoProjectRule") .withRuleFactory(MongoProjectRule::new) .toRule(MongoProjectRule.class); diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java index 54abdf01e0a0..e255e460b633 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java @@ -100,7 +100,7 @@ protected PigTableScanRule(Config config) { private static class PigProjectRule extends ConverterRule { private static final PigProjectRule INSTANCE = Config.INSTANCE .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null - || p.getVariablesSet().isEmpty(), Convention.NONE, + && p.getVariablesSet().isEmpty(), Convention.NONE, PigRel.CONVENTION, "PigProjectRule") .withRuleFactory(PigProjectRule::new) .toRule(PigProjectRule.class); From cfe02dca860421b7319983bda0a1766f8c6f30fa Mon Sep 17 00:00:00 2001 From: korlov42 Date: Wed, 23 Mar 2022 17:46:20 +0300 Subject: [PATCH 16/21] move correlate variables rewritting to field trimmer --- .../calcite/rel/rules/SubQueryRemoveRule.java | 26 +--- .../ChangeTypeOfCorrelateVariables.java | 113 ++++++++++++++++ .../calcite/sql2rel/RelFieldTrimmer.java | 10 +- .../calcite/sql2rel/SqlToRelConverter.java | 3 +- .../java/org/apache/calcite/util/Bug.java | 2 +- .../apache/calcite/test/RelOptRulesTest.java | 1 - .../apache/calcite/test/RelOptRulesTest.xml | 8 +- core/src/test/resources/sql/agg.iq | 10 +- core/src/test/resources/sql/sub-query.iq | 126 +++++++++++------- 9 files changed, 216 insertions(+), 83 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index c57f2853036b..b3877241539b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.rel.rules; -import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelRule; @@ -42,14 +41,12 @@ import org.apache.calcite.sql.fun.SqlQuantifyOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql2rel.DeduplicateCorrelateVariables; import org.apache.calcite.sql2rel.RelDecorrelator; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; @@ -132,25 +129,8 @@ protected RexNode apply(RexSubQuery e, Set variablesSet, */ private static RexNode rewriteScalarQuery(RexSubQuery e, Set variablesSet, RelBuilder builder, int inputCount, int offset) { - // in case the relation has several scalar queries - // referred to the same correlated variable, every - // such a query will be re-written to a particular - // correlate with the common set of variables, - // resulting a different relations will be trying to - // set the same correlated variable. To overcome this - // let's rewrite an every correlated variable with a - // new one created specially for current scalar query - RelNode r = e.rel; - RelOptCluster cluster = r.getCluster(); - ImmutableSet.Builder newVariablesSet = ImmutableSet.builder(); - for (CorrelationId corrId : variablesSet) { - CorrelationId newId = cluster.createCorrel(); - newVariablesSet.add(newId); - r = DeduplicateCorrelateVariables.go(cluster.getRexBuilder(), newId, - ImmutableSet.of(corrId), r); - } - builder.push(r); - final RelMetadataQuery mq = cluster.getMetadataQuery(); + builder.push(e.rel); + final RelMetadataQuery mq = e.rel.getCluster().getMetadataQuery(); final Boolean unique = mq.areColumnsUnique(builder.peek(), ImmutableBitSet.of()); if (unique == null || !unique) { @@ -158,7 +138,7 @@ private static RexNode rewriteScalarQuery(RexSubQuery e, Set vari builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, builder.field(0))); } - builder.join(JoinRelType.LEFT, builder.literal(true), newVariablesSet.build()); + builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet); return field(builder, inputCount, offset); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java b/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java new file mode 100644 index 000000000000..eb62773ae6b2 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql2rel; + +import org.apache.calcite.rel.RelHomogeneousShuttle; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexSubQuery; + +import com.google.common.collect.ImmutableSet; + +import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; +import org.checkerframework.checker.initialization.qual.UnderInitialization; + +/** + * Rewrites relations to ensure a correlation references the right row type after trimming. + */ +public class ChangeTypeOfCorrelateVariables extends RelHomogeneousShuttle { + @NotOnlyInitialized + private final RexShuttle dedupRex; + + /** Creates a ChangeTypeOfCorrelateVariables. */ + private ChangeTypeOfCorrelateVariables(RexBuilder builder, + ImmutableSet corrIds, RelDataType expectedType) { + dedupRex = new ChangeTypeOfCorrelateVariablesShuttle(builder, + corrIds, expectedType, this); + } + + /** + * Rewrites a relational expression, replacing correlation variables + * with a similar one but proper row type. + */ + public static RelNode go(RexBuilder builder, Iterable corrIds, + RelDataType expectedType, RelNode r) { + return r.accept( + new ChangeTypeOfCorrelateVariables(builder, ImmutableSet.copyOf(corrIds), expectedType)); + } + + @Override public RelNode visit(RelNode other) { + RelNode next = super.visit(other); + return next.accept(dedupRex); + } + + /** + * Replaces row type of correlation variable to the expected one. + */ + private static class ChangeTypeOfCorrelateVariablesShuttle extends RexShuttle { + private final RexBuilder builder; + private final ImmutableSet corrIds; + private final RelDataType expectedType; + @NotOnlyInitialized + private final ChangeTypeOfCorrelateVariables shuttle; + + private ChangeTypeOfCorrelateVariablesShuttle(RexBuilder builder, + ImmutableSet corrIds, RelDataType expectedType, + @UnderInitialization ChangeTypeOfCorrelateVariables shuttle) { + this.builder = builder; + this.corrIds = corrIds; + this.expectedType = expectedType; + this.shuttle = shuttle; + } + + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + RexNode before = fieldAccess.getReferenceExpr(); + RexNode after = before.accept(this); + + if (before == after) { + return fieldAccess; + } else { + return builder.makeFieldAccess(after, + fieldAccess.getField().getName(), true); + } + } + + @Override public RexNode visitCorrelVariable(RexCorrelVariable variable) { + if (!corrIds.contains(variable.id) || variable.getType().equals(expectedType)) { + return variable; + } + + return builder.makeCorrel(expectedType, variable.id); + } + + @Override public RexNode visitSubQuery(RexSubQuery subQuery) { + if (shuttle != null) { + RelNode r = subQuery.rel.accept(shuttle); // look inside sub-queries + if (r != subQuery.rel) { + subQuery = subQuery.clone(r); + } + } + return super.visitSubQuery(subQuery); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 2a53e9427cde..77df3eacbcc2 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -531,8 +531,14 @@ public TrimResult trimFields( mapping); relBuilder.push(newInput); - relBuilder.project(newProjects, newRowType.getFieldNames()); - final RelNode newProject = RelOptUtil.propagateRelHints(project, relBuilder.build()); + relBuilder.projectCorrelated(project.getVariablesSet(), + newProjects, newRowType.getFieldNames()); + final RelNode newProject = ChangeTypeOfCorrelateVariables.go( + project.getCluster().getRexBuilder(), + project.getVariablesSet(), + newInput.getRowType(), + RelOptUtil.propagateRelHints(project, relBuilder.build()) + ); return result(newProject, mapping); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index a12c79c4a743..e106ce67ab9a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -6479,7 +6479,8 @@ private class NestedJsonFunctionRelRewriter extends RelShuttleImpl { newInput, project.getHints(), newProjections.build(), - project.getRowType().getFieldNames()); + project.getRowType().getFieldNames(), + project.getVariablesSet()); } private Set requiredJsonOutputFromParent(RelNode relNode) { diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 602984c5cf1b..a92764b5617a 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -150,7 +150,7 @@ public abstract class Bug { /** Whether * [CALCITE-1045] * Decorrelate sub-queries in Project and Join is fixed. */ - public static final boolean CALCITE_1045_FIXED = false; + public static final boolean CALCITE_1045_FIXED = true; /** * Whether diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 2ce5a183a1c6..ee1430cd6d00 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -127,7 +127,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 65eefc6fdc00..f25546e5cabb 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -1704,11 +1704,11 @@ LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject("K0"=[$0], "C1"=[$1], "F1"."A0"=[$2], "F2"."A0"=[$3], "F0"."C0"=[$4], "F1"."C0"=[$5], "F0"."C1"=[$6], "F1"."C2"=[$7], "F2"."C3"=[$8]) LogicalProject("K0"=[$0], "C1"=[$1], "F1"."A0"=[$2], "F2"."A0"=[$3], "F0"."C0"=[$4], "F1"."C0"=[$5], "F0"."C1"=[$6], "F1"."C2"=[$7], "F2"."C3"=[$8]) LogicalFilter(condition=[=($4, $9)]) - LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) LogicalTableScan(table=[[CATALOG, STRUCT, T]]) LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject("F1"."C0"=[$5]) - LogicalFilter(condition=[=($cor1."K0", $0)]) + LogicalFilter(condition=[=($cor0."K0", $0)]) LogicalTableScan(table=[[CATALOG, STRUCT, T]]) ]]> @@ -3763,11 +3763,11 @@ LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject(EMPNO=[$0]) LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[=($5, $9)]) - LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) LogicalAggregate(group=[{}], EXPR$0=[MAX($0)]) LogicalProject(SAL=[$5]) - LogicalFilter(condition=[=($0, $cor1.EMPNO)]) + LogicalFilter(condition=[=($0, $cor0.EMPNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index f8b56a832544..47ba7e71f934 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -1336,8 +1336,16 @@ GROUP BY deptno; # As above, but with correlation !if (fixed.calcite1045) { SELECT SUM( - (select char_length(dname) from "scott".dept where dept.deptno = emp.empno)) as s + (select char_length(dname) from "scott".dept where dept.deptno = emp.deptno)) as s FROM "scott".emp; + ++-----+ +| S | ++-----+ +| 100 | ++-----+ +(1 row) + !ok !} diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 4e534d423334..7bf4d4ad7089 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -316,13 +316,15 @@ select empno from "scott".emp as e join "scott".dept as d using (deptno) where e.job in ( select e2.job from "scott".emp as e2 where e2.deptno > e.deptno); - EMPNO -------- - 7369 - 7566 - 7782 - 7876 - 7934 ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7566 | +| 7782 | +| 7876 | +| 7934 | ++-------+ (5 rows) !ok @@ -330,13 +332,15 @@ EnumerableCalc(expr#0..5=[{inputs}], EMPNO=[$t0]) EnumerableHashJoin(condition=[=($2, $5)], joinType=[inner]) EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t2], JOB=[$t3], DEPTNO=[$t4], JOB0=[$t0], DEPTNO0=[$t1]) EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) - EnumerableCalc(expr#0..1=[{inputs}], JOB=[$t1], DEPTNO=[$t0]) - EnumerableAggregate(group=[{0, 2}]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[>($t3, $t0)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) - EnumerableAggregate(group=[{7}]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableAggregate(group=[{1, 3}]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) @@ -351,7 +355,57 @@ select empno from "scott".emp as e join "scott".dept as d using (deptno) where e.job not in ( select e2.job from "scott".emp as e2 where e2.deptno > e.deptno); + ++-------+ +| EMPNO | ++-------+ +| 7499 | +| 7521 | +| 7654 | +| 7698 | +| 7788 | +| 7839 | +| 7844 | +| 7900 | +| 7902 | ++-------+ +(9 rows) + !ok + +EnumerableCalc(expr#0..9=[{inputs}], expr#10=[0], expr#11=[=($t5, $t10)], expr#12=[IS NULL($t1)], expr#13=[IS NOT NULL($t9)], expr#14=[<($t6, $t5)], expr#15=[OR($t12, $t13, $t14)], expr#16=[IS NOT TRUE($t15)], expr#17=[OR($t11, $t16)], EMPNO=[$t0], $condition=[$t17]) + EnumerableMergeJoin(condition=[AND(=($1, $7), =($2, $8))], joinType=[left]) + EnumerableSort(sort0=[$1], sort1=[$2], dir0=[ASC], dir1=[ASC]) + EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[inner]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{3}], c=[COUNT()], ck=[COUNT($1)]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) + EnumerableAggregate(group=[{1, 3}]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) !plan !} @@ -555,14 +609,14 @@ select deptno, (select min(1) from "scott".emp where empno > d.deptno) as i0, (select min(0) from "scott".emp where deptno = d.deptno and ename = 'SMITH') as i1 from "scott".dept as d; -+--------+----+---+ -| DEPTNO | I0 | I1| -+--------+----+---+ -| 10 | 1 | | -| 20 | 1 | 0 | -| 30 | 1 | | -| 40 | 1 | | -+--------+----+---+ ++--------+----+----+ +| DEPTNO | I0 | I1 | ++--------+----+----+ +| 10 | 1 | | +| 20 | 1 | 0 | +| 30 | 1 | | +| 40 | 1 | | ++--------+----+----+ (4 rows) !ok @@ -3257,34 +3311,6 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[null:BOOLEAN], proj#0..8=[{exprs}]) EnumerableTableScan(table=[[scott, EMP]]) !plan -# [CALCITE-4913] Correlated variables in a select list are not deduplicated -select e.deptno, - (select count(*) from emp where e.deptno > 0) as c1, - (select count(*) from emp where e.deptno > 0 and e.deptno < 10) as c2 - from emp e; - -+--------+----+----+ -| DEPTNO | C1 | C2 | -+--------+----+----+ -| 10 | 14 | 0 | -| 10 | 14 | 0 | -| 10 | 14 | 0 | -| 20 | 14 | 0 | -| 20 | 14 | 0 | -| 20 | 14 | 0 | -| 20 | 14 | 0 | -| 20 | 14 | 0 | -| 30 | 14 | 0 | -| 30 | 14 | 0 | -| 30 | 14 | 0 | -| 30 | 14 | 0 | -| 30 | 14 | 0 | -| 30 | 14 | 0 | -+--------+----+----+ -(14 rows) - -!ok - # [CALCITE-4844] IN-list that references columns is wrongly converted to Values, and gives incorrect results !set insubquerythreshold 0 From c4b5ea3f024f243ae96c2aa24bb8e8e74a085327 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Thu, 2 Jun 2022 14:01:11 +0300 Subject: [PATCH 17/21] minors --- core/src/main/java/org/apache/calcite/rel/core/Project.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index 00124df7b130..4070c62dc7a5 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -146,7 +146,7 @@ protected Project(RelInput input) { Util.transform( Optional.ofNullable(input.getIntegerList("correlation")) .orElse(ImmutableList.of()), - CorrelationId::new + id -> new CorrelationId(id) ) )); } From 3a58c5753fd40dfb9b4aa67ec7e77ad20692cea2 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Thu, 2 Jun 2022 15:11:27 +0300 Subject: [PATCH 18/21] change the query to the one from sub-query.iq --- .../calcite/test/SqlToRelConverterTest.java | 8 ++-- .../calcite/test/SqlToRelConverterTest.xml | 42 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index b35c6af5ac6b..2a5750b3dc67 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1295,10 +1295,10 @@ class SqlToRelConverterTest extends SqlToRelTestBase { sql(sql).withExpand(false).withDecorrelate(false) .convertsTo("${planNotExpanded}"); }; - fn.accept("select e.deptno,\n" - + " (select count(*) from emp where e.deptno > 0),\n" - + " (select count(*) from emp where e.deptno > 0 and e.deptno < 10)\n" - + "from emp e"); + fn.accept("select deptno,\n" + + " (select min(1) from emp where empno > d.deptno) as i0,\n" + + " (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1\n" + + "from dept as d"); } @Test void testCorrelationLateralSubQuery() { diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 80fd6159aefe..e40ca9a3736d 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -667,41 +667,41 @@ LogicalProject(EMPNO=[$0], JOB=[$2]) ($cor0.DEPTNO, 0)]) +LogicalProject(correlation=[[$cor0]], DEPTNO=[$0], I0=[$SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) + LogicalProject($f0=[1]) + LogicalFilter(condition=[>($0, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) -})], EXPR$2=[$SCALAR_QUERY({ -LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) +})], I1=[$SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) LogicalProject($f0=[0]) - LogicalFilter(condition=[AND(>($cor0.DEPTNO, 0), <($cor0.DEPTNO, 10))]) + LogicalFilter(condition=[AND(=($7, $cor0.DEPTNO), =($1, 'SMITH'))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) })]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) ]]> ($cor0.DEPTNO, 0)]) +LogicalProject(DEPTNO=[$0], I0=[$2], I1=[$3]) + LogicalCorrelate(correlation=[$cor1], joinType=[left], requiredColumns=[{0}]) + LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{0}]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) + LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) + LogicalProject($f0=[1]) + LogicalFilter(condition=[>($0, $cor0.DEPTNO)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) LogicalProject($f0=[0]) - LogicalFilter(condition=[AND(>($cor1.DEPTNO, 0), <($cor1.DEPTNO, 10))]) + LogicalFilter(condition=[AND(=($7, $cor1.DEPTNO), =($1, 'SMITH'))]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> - 0), - (select count(*) from emp where e.deptno > 0 and e.deptno < 10) -from emp e]]> + d.deptno) as i0, + (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1 +from dept as d]]> From 53f8f466ec08a7528cca80dc98908b078f63d4a5 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Thu, 2 Jun 2022 15:18:56 +0300 Subject: [PATCH 19/21] fix indentation --- .../java/org/apache/calcite/test/SqlToRelConverterTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 2a5750b3dc67..64183548a5d9 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1296,9 +1296,9 @@ class SqlToRelConverterTest extends SqlToRelTestBase { .convertsTo("${planNotExpanded}"); }; fn.accept("select deptno,\n" - + " (select min(1) from emp where empno > d.deptno) as i0,\n" - + " (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1\n" - + "from dept as d"); + + " (select min(1) from emp where empno > d.deptno) as i0,\n" + + " (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1\n" + + "from dept as d"); } @Test void testCorrelationLateralSubQuery() { From 3b49adcbf5f06073be46f37ba1e64287e31b5f2d Mon Sep 17 00:00:00 2001 From: korlov42 Date: Thu, 2 Jun 2022 15:35:48 +0300 Subject: [PATCH 20/21] added a plan --- core/src/test/resources/sql/sub-query.iq | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 7bf4d4ad7089..254653c31a7d 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -620,6 +620,24 @@ from "scott".dept as d; (4 rows) !ok + +EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}], I1=[$t3]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[=($0, $1)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableAggregate(group=[{0}], EXPR$0=[MIN($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], DEPTNO0=[$t0], $f0=[$t2]) + EnumerableNestedLoopJoin(condition=[>($1, $0)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{0}], EXPR$0=[MIN($1)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=['SMITH':VARCHAR(10)], expr#10=[=($t1, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], DEPTNO=[$t7], $f0=[$t8], $condition=[$t12]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan !} # [CALCITE-1494] Inefficient plan for correlated sub-queries From d06e3e96e10db538c8eed34818cf21a6c4e5ade0 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Thu, 2 Jun 2022 17:21:02 +0300 Subject: [PATCH 21/21] added test for a correlated subquery in collection --- core/src/test/resources/sql/sub-query.iq | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 254653c31a7d..71a62741a768 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -3534,4 +3534,38 @@ where NOT EXISTS (select count(*) from emp e having false); EnumerableTableScan(table=[[scott, DEPT]]) !plan +# Test collection from a correlated subquery + +select dname, + array (select deptno + from emp + where deptno = dept.deptno) as deptno_array, + multiset (select deptno + from emp + where deptno = dept.deptno) as deptno_multiset +from dept; ++------------+--------------------------------------+--------------------------------------+ +| DNAME | DEPTNO_ARRAY | DEPTNO_MULTISET | ++------------+--------------------------------------+--------------------------------------+ +| ACCOUNTING | [{10}, {10}, {10}] | [{10}, {10}, {10}] | +| OPERATIONS | [] | [] | +| RESEARCH | [{20}, {20}, {20}, {20}, {20}] | [{20}, {20}, {20}, {20}, {20}] | +| SALES | [{30}, {30}, {30}, {30}, {30}, {30}] | [{30}, {30}, {30}, {30}, {30}, {30}] | ++------------+--------------------------------------+--------------------------------------+ +(4 rows) + +!ok + +EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], DEPTNO_ARRAY=[$t3], DEPTNO_MULTISET=[$t4]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCollect(field=[x]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[$cor0], expr#9=[$t8.DEPTNO], expr#10=[=($t7, $t9)], DEPTNO=[$t7], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCollect(field=[x]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[$cor0], expr#9=[$t8.DEPTNO], expr#10=[=($t7, $t9)], DEPTNO=[$t7], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + # End sub-query.iq