diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java index b5a77c74f829..fdc094e79c88 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java @@ -21,19 +21,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -42,13 +46,15 @@ public class CassandraProject extends Project implements CassandraRel { public CassandraProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == CassandraRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new CassandraProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java index ce38ed3d18d2..8398dddac515 100644 --- a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java +++ b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraRules.java @@ -258,7 +258,8 @@ public interface CassandraFilterRuleConfig extends RelRule.Config { public static class CassandraProjectRule extends CassandraConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, CassandraRel.CONVENTION, "CassandraProjectRule") .withRuleFactory(CassandraProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java index 4bc516c520a8..287c96e9ba89 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProject.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMdCollation; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -28,11 +29,14 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Set; /** Implementation of {@link org.apache.calcite.rel.core.Project} in * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ @@ -54,7 +58,7 @@ public EnumerableProject( RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof EnumerableConvention; } @@ -80,7 +84,9 @@ public static EnumerableProject create(final RelNode input, } @Override public EnumerableProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new EnumerableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java index b1b69220a629..b7497ef0217b 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableProjectRule.java @@ -32,7 +32,8 @@ class EnumerableProjectRule extends ConverterRule { /** Default configuration. */ static final Config DEFAULT_CONFIG = Config.INSTANCE .as(Config.class) - .withConversion(LogicalProject.class, p -> !p.containsOver(), + .withConversion(LogicalProject.class, p -> !p.containsOver() + && p.getCorrelVariable() == null && p.getVariablesSet().isEmpty(), Convention.NONE, EnumerableConvention.INSTANCE, "EnumerableProjectRule") .withRuleFactory(EnumerableProjectRule::new); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java index ad109c44d0b4..b86ea3e9cc28 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelFactories.java @@ -26,6 +26,8 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.validate.SqlValidatorUtil; +import com.google.common.base.Preconditions; + import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -68,7 +70,10 @@ private static class ProjectFactoryImpl implements org.apache.calcite.rel.core.RelFactories.ProjectFactory { @Override public RelNode createProject(RelNode input, List hints, List childExprs, - @Nullable List fieldNames) { + @Nullable List fieldNames, + Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); final RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.F_SUGGESTER); diff --git a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java index 37fb747a87d7..bcc60715fd24 100644 --- a/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java @@ -73,6 +73,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -96,7 +97,9 @@ private JdbcRules() { protected static final Logger LOGGER = CalciteTrace.getPlannerTracer(); static final RelFactories.ProjectFactory PROJECT_FACTORY = - (input, hints, projects, fieldNames) -> { + (input, hints, projects, fieldNames, variablesSet) -> { + Preconditions.checkArgument(variablesSet.isEmpty(), + "JdbcProject does not allow variables"); final RelOptCluster cluster = input.getCluster(); final RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), projects, @@ -532,7 +535,7 @@ public JdbcProject( RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof JdbcConvention; } @@ -544,7 +547,9 @@ public JdbcProject(RelOptCluster cluster, RelTraitSet traitSet, } @Override public JdbcProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new JdbcProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java index 77b5287fc24b..43a4d69ef730 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Bindables.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Bindables.java @@ -78,6 +78,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; @@ -376,7 +377,8 @@ public static BindableFilter create(final RelNode input, public static class BindableProjectRule extends ConverterRule { /** Default configuration. */ public static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, p -> !p.containsOver(), + .withConversion(LogicalProject.class, p -> !p.containsOver() + && p.getCorrelVariable() == null && p.getVariablesSet().isEmpty(), Convention.NONE, BindableConvention.INSTANCE, "BindableProjectRule") .withRuleFactory(BindableProjectRule::new); @@ -403,12 +405,14 @@ protected BindableProjectRule(Config config) { public static class BindableProject extends Project implements BindableRel { public BindableProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() instanceof BindableConvention; } @Override public BindableProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new BindableProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index c7b5c7cbcc78..d7739ea4ae34 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -885,6 +885,7 @@ public static RelNode createCastRel( List castExps; RelNode input; List hints = ImmutableList.of(); + Set variablesSet = ImmutableSet.of(); if (rel instanceof Project) { // No need to create another project node if the rel // is already a project. @@ -894,6 +895,7 @@ public static RelNode createCastRel( castRowType, ((Project) rel).getProjects()); input = rel.getInput(0); + variablesSet = project.getVariablesSet(); hints = project.getHints(); } else { castExps = RexUtil.generateCastExpressions( @@ -905,11 +907,11 @@ public static RelNode createCastRel( if (rename) { // Use names and types from castRowType. return projectFactory.createProject(input, hints, castExps, - castRowType.getFieldNames()); + castRowType.getFieldNames(), variablesSet); } else { // Use names from rowType, types from castRowType. return projectFactory.createProject(input, hints, castExps, - rowType.getFieldNames()); + rowType.getFieldNames(), variablesSet); } } diff --git a/core/src/main/java/org/apache/calcite/rel/RelNode.java b/core/src/main/java/org/apache/calcite/rel/RelNode.java index 21f6586cc2fa..ef3b878ce704 100644 --- a/core/src/main/java/org/apache/calcite/rel/RelNode.java +++ b/core/src/main/java/org/apache/calcite/rel/RelNode.java @@ -152,9 +152,6 @@ public interface RelNode extends RelOptNode, Cloneable { * expression but also used and therefore not available to parents of this * relational expression. * - *

Note: only {@link org.apache.calcite.rel.core.Correlate} should set - * variables. - * * @return Names of variables which are set in this relational * expression */ diff --git a/core/src/main/java/org/apache/calcite/rel/core/Project.java b/core/src/main/java/org/apache/calcite/rel/core/Project.java index f4a2e3ac3ddc..4070c62dc7a5 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Project.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Project.java @@ -45,6 +45,7 @@ import org.apache.calcite.util.mapping.Mappings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apiguardian.api.API; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; @@ -53,6 +54,7 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -70,6 +72,14 @@ public abstract class Project extends SingleRel implements Hintable { protected final ImmutableList hints; + /** + * Correlation variables set by this relational expression to be used by + * nested expressions. It's expected to be used in the following way: + * first read the row from input, set it to the appropriate correlation + * variables in the context, then execute the Rex expressions. + */ + protected final ImmutableSet variablesSet; + //~ Constructors ----------------------------------------------------------- /** @@ -81,6 +91,8 @@ public abstract class Project extends SingleRel implements Hintable { * @param input Input relational expression * @param projects List of expressions for the input columns * @param rowType Output row type + * @param variablesSet Correlation variables set by this relational expression + * to be used by nested expressions */ @SuppressWarnings("method.invocation.invalid") protected Project( @@ -89,25 +101,34 @@ protected Project( List hints, RelNode input, List projects, - RelDataType rowType) { + RelDataType rowType, + Set variablesSet) { super(cluster, traits, input); assert rowType != null; this.exps = ImmutableList.copyOf(projects); this.hints = ImmutableList.copyOf(hints); this.rowType = rowType; + this.variablesSet = ImmutableSet.copyOf( + Objects.requireNonNull(variablesSet, "variablesSet")); assert isValid(Litmus.THROW, null); } + @Deprecated // to be removed before 2.0 + protected Project(RelOptCluster cluster, RelTraitSet traits, List hints, + RelNode input, List projects, RelDataType rowType) { + this(cluster, traits, hints, input, projects, rowType, ImmutableSet.of()); + } + @Deprecated // to be removed before 2.0 protected Project(RelOptCluster cluster, RelTraitSet traits, RelNode input, List projects, RelDataType rowType) { - this(cluster, traits, ImmutableList.of(), input, projects, rowType); + this(cluster, traits, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); } @Deprecated // to be removed before 2.0 protected Project(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, int flags) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); Util.discard(flags); } @@ -120,14 +141,21 @@ protected Project(RelInput input) { ImmutableList.of(), input.getInput(), requireNonNull(input.getExpressionList("exprs"), "exprs"), - input.getRowType("exprs", "fields")); + input.getRowType("exprs", "fields"), + ImmutableSet.copyOf( + Util.transform( + Optional.ofNullable(input.getIntegerList("correlation")) + .orElse(ImmutableList.of()), + id -> new CorrelationId(id) + ) + )); } //~ Methods ---------------------------------------------------------------- @Override public final RelNode copy(RelTraitSet traitSet, List inputs) { - return copy(traitSet, sole(inputs), exps, getRowType()); + return copy(traitSet, sole(inputs), exps, getRowType(), variablesSet); } /** @@ -137,6 +165,7 @@ protected Project(RelInput input) { * @param input Input * @param projects Project expressions * @param rowType Output row type + * @param variablesSet Correlation variables * @return New {@code Project} if any parameter differs from the value of this * {@code Project}, or just {@code this} if all the parameters are * the same @@ -144,7 +173,13 @@ protected Project(RelInput input) { * @see #copy(RelTraitSet, List) */ public abstract Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType); + List projects, RelDataType rowType, Set variablesSet); + + @Deprecated // to be removed before 2.0 + public Project copy(RelTraitSet traitSet, RelNode input, + List projects, RelDataType rowType) { + return copy(traitSet, input, projects, rowType, ImmutableSet.of()); + } @Deprecated // to be removed before 2.0 public Project copy(RelTraitSet traitSet, RelNode input, @@ -169,7 +204,7 @@ public boolean isBoxed() { exps, getRowType().getFieldNames(), null); - return copy(traitSet, getInput(), exps, rowType); + return copy(traitSet, getInput(), exps, rowType, variablesSet); } /** @@ -265,7 +300,9 @@ private static int countTrivial(List refs) { } @Override public RelWriter explainTerms(RelWriter pw) { - super.explainTerms(pw); + super.explainTerms(pw) + .itemIf("correlation", variablesSet, !variablesSet.isEmpty()); + // Skip writing field names so the optimizer can reuse the projects that differ in // field names only if (pw.getDetailLevel() == SqlExplainLevel.DIGEST_ATTRIBUTES) { @@ -297,6 +334,10 @@ private static int countTrivial(List refs) { return pw; } + @Override public Set getVariablesSet() { + return variablesSet; + } + @API(since = "1.24", status = API.Status.INTERNAL) @EnsuresNonNullIf(expression = "#1", result = true) protected boolean deepEquals0(@Nullable Object obj) { @@ -311,12 +352,13 @@ protected boolean deepEquals0(@Nullable Object obj) { && input.deepEquals(o.input) && exps.equals(o.exps) && hints.equals(o.hints) - && getRowType().equalsSansFieldNames(o.getRowType()); + && getRowType().equalsSansFieldNames(o.getRowType()) + && variablesSet.equals(o.variablesSet); } @API(since = "1.24", status = API.Status.INTERNAL) protected int deepHashCode0() { - return Objects.hash(traitSet, input.deepHashCode(), exps, hints); + return Objects.hash(traitSet, input.deepHashCode(), exps, hints, variablesSet); } /** diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index 6d5e2c0d65ca..87c568a67369 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -162,10 +162,20 @@ public interface ProjectFactory { * @param hints The hints * @param childExprs The projection expressions * @param fieldNames The projection field names + * @param variablesSet Correlating variables that are set when reading + * a row from the input, and which may be referenced from inside the + * projection * @return a project */ RelNode createProject(RelNode input, List hints, - List childExprs, @Nullable List fieldNames); + List childExprs, @Nullable List fieldNames, + Set variablesSet); + + @Deprecated // to be removed before 2.0 + default RelNode createProject(RelNode input, List hints, + List childExprs, @Nullable List fieldNames) { + return createProject(input, hints, childExprs, fieldNames, ImmutableSet.of()); + } } /** @@ -174,8 +184,9 @@ RelNode createProject(RelNode input, List hints, */ private static class ProjectFactoryImpl implements ProjectFactory { @Override public RelNode createProject(RelNode input, List hints, - List childExprs, @Nullable List fieldNames) { - return LogicalProject.create(input, hints, childExprs, fieldNames); + List childExprs, @Nullable List fieldNames, + Set variablesSet) { + return LogicalProject.create(input, hints, childExprs, fieldNames, variablesSet); } } diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index d39947dfdfec..d50c77c45483 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -402,6 +402,12 @@ public Object toJson(AggregateCall node) { list.add(toJson(o)); } return list; + } else if (value instanceof Set) { + final List<@Nullable Object> list = jsonBuilder().list(); + for (Object o : (Set) value) { + list.add(toJson(o)); + } + return list; } else if (value instanceof ImmutableBitSet) { final List<@Nullable Object> list = jsonBuilder().list(); for (Integer integer : (ImmutableBitSet) value) { diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java index cb64b372700c..c12899e010e2 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalProject.java @@ -24,6 +24,7 @@ import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMdCollation; @@ -35,10 +36,12 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Set; /** * Sub-class of {@link org.apache.calcite.rel.core.Project} not @@ -58,6 +61,8 @@ public final class LogicalProject extends Project { * @param input Input relational expression * @param projects List of expressions for the input columns * @param rowType Output row type + * @param variablesSet Correlation variables set by this relational expression + * to be used by nested expressions */ public LogicalProject( RelOptCluster cluster, @@ -65,22 +70,36 @@ public LogicalProject( List hints, RelNode input, List projects, - RelDataType rowType) { - super(cluster, traitSet, hints, input, projects, rowType); + RelDataType rowType, + Set variablesSet) { + super(cluster, traitSet, hints, input, projects, rowType, variablesSet); assert traitSet.containsIfApplicable(Convention.NONE); } + @Deprecated // to be removed before 2.0 + public LogicalProject( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelNode input, + List projects, + RelDataType rowType) { + this(cluster, traitSet, hints, input, projects, rowType, ImmutableSet.of()); + } + @Deprecated // to be removed before 2.0 public LogicalProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, + ImmutableSet.of()); } @Deprecated // to be removed before 2.0 public LogicalProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType, int flags) { - this(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + this(cluster, traitSet, ImmutableList.of(), input, projects, rowType, + ImmutableSet.of()); Util.discard(flags); } @@ -90,7 +109,7 @@ public LogicalProject(RelOptCluster cluster, RelNode input, this(cluster, cluster.traitSetOf(RelCollations.EMPTY), ImmutableList.of(), input, projects, RexUtil.createStructType(cluster.getTypeFactory(), projects, - fieldNames, null)); + fieldNames, null), ImmutableSet.of()); Util.discard(flags); } @@ -107,28 +126,44 @@ public LogicalProject(RelInput input) { public static LogicalProject create(final RelNode input, List hints, final List projects, @Nullable List fieldNames) { + return create(input, hints, projects, fieldNames, ImmutableSet.of()); + } + + /** Creates a LogicalProject. */ + public static LogicalProject create(final RelNode input, List hints, + final List projects, + @Nullable List fieldNames, + Set variablesSet) { final RelOptCluster cluster = input.getCluster(); final RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), projects, fieldNames, SqlValidatorUtil.F_SUGGESTER); - return create(input, hints, projects, rowType); + return create(input, hints, projects, rowType, variablesSet); } /** Creates a LogicalProject, specifying row type rather than field names. */ public static LogicalProject create(final RelNode input, List hints, final List projects, RelDataType rowType) { + return create(input, hints, projects, rowType, ImmutableSet.of()); + } + + /** Creates a LogicalProject, specifying row type rather than field names. */ + public static LogicalProject create(final RelNode input, List hints, + final List projects, RelDataType rowType, + Set variablesSet) { final RelOptCluster cluster = input.getCluster(); final RelMetadataQuery mq = cluster.getMetadataQuery(); final RelTraitSet traitSet = cluster.traitSet().replace(Convention.NONE) .replaceIfs(RelCollationTraitDef.INSTANCE, () -> RelMdCollation.project(mq, input, projects)); - return new LogicalProject(cluster, traitSet, hints, input, projects, rowType); + return new LogicalProject(cluster, traitSet, hints, input, projects, rowType, variablesSet); } @Override public LogicalProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { - return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType); + List projects, RelDataType rowType, Set variablesSet) { + return new LogicalProject(getCluster(), traitSet, hints, input, projects, rowType, + variablesSet); } @Override public RelNode accept(RelShuttle shuttle) { @@ -137,7 +172,7 @@ public static LogicalProject create(final RelNode input, List hints, @Override public RelNode withHints(List hintList) { return new LogicalProject(getCluster(), traitSet, hintList, - input, getProjects(), getRowType()); + input, getProjects(), getRowType(), variablesSet); } @Override public boolean deepEquals(@Nullable Object obj) { diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index 13665c4809e8..6fdbbaf4dee2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -187,7 +187,7 @@ protected FilterProjectTransposeRule( RelNode newProject = config.isCopyProject() ? project.copy(project.getTraitSet(), newFilterRel, - project.getProjects(), project.getRowType()) + project.getProjects(), project.getRowType(), project.getVariablesSet()) : relBuilder.push(newFilterRel) .project(project.getProjects(), project.getRowType().getFieldNames()) .build(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java index 5ecd6ccdfee6..b8ea42b66b6a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectJoinRemoveRule.java @@ -114,14 +114,14 @@ public ProjectJoinRemoveRule( if (isLeftJoin) { node = project .copy(project.getTraitSet(), join.getLeft(), project.getProjects(), - project.getRowType()); + project.getRowType(), project.getVariablesSet()); } else { final int offset = join.getLeft().getRowType().getFieldCount(); final List newExprs = project.getProjects().stream() .map(expr -> RexUtil.shift(expr, -offset)) .collect(Collectors.toList()); node = project.copy(project.getTraitSet(), join.getRight(), newExprs, - project.getRowType()); + project.getRowType(), project.getVariablesSet()); } call.transformTo(node); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java index 67df2031622e..a1b04f347026 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectRemoveRule.java @@ -64,7 +64,7 @@ public ProjectRemoveRule(RelBuilderFactory relBuilderFactory) { Project childProject = (Project) stripped; stripped = childProject.copy(childProject.getTraitSet(), childProject.getInput(), childProject.getProjects(), - project.getRowType()); + project.getRowType(), childProject.getVariablesSet()); } stripped = convert(stripped, project.getConvention()); call.transformTo(stripped); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java index 761dd1df7055..ed69ef357000 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectWindowTransposeRule.java @@ -37,6 +37,7 @@ import org.apache.calcite.util.ImmutableBitSet; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; @@ -99,7 +100,7 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { final LogicalProject projectBelowWindow = new LogicalProject(cluster, window.getTraitSet(), ImmutableList.of(), - window.getInput(), exps, builder.build()); + window.getInput(), exps, builder.build(), ImmutableSet.of()); // Create a new LogicalWindow with necessary inputs only final List groups = new ArrayList<>(); @@ -182,7 +183,8 @@ public ProjectWindowTransposeRule(RelBuilderFactory relBuilderFactory) { newLogicalWindow.getTraitSet(), newLogicalWindow, topProjExps, - project.getRowType()); + project.getRowType(), + project.getVariablesSet()); if (ProjectRemoveRule.isTrivial(newTopProj)) { call.transformTo(newLogicalWindow); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java b/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java new file mode 100644 index 000000000000..eb62773ae6b2 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql2rel/ChangeTypeOfCorrelateVariables.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql2rel; + +import org.apache.calcite.rel.RelHomogeneousShuttle; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexSubQuery; + +import com.google.common.collect.ImmutableSet; + +import org.checkerframework.checker.initialization.qual.NotOnlyInitialized; +import org.checkerframework.checker.initialization.qual.UnderInitialization; + +/** + * Rewrites relations to ensure a correlation references the right row type after trimming. + */ +public class ChangeTypeOfCorrelateVariables extends RelHomogeneousShuttle { + @NotOnlyInitialized + private final RexShuttle dedupRex; + + /** Creates a ChangeTypeOfCorrelateVariables. */ + private ChangeTypeOfCorrelateVariables(RexBuilder builder, + ImmutableSet corrIds, RelDataType expectedType) { + dedupRex = new ChangeTypeOfCorrelateVariablesShuttle(builder, + corrIds, expectedType, this); + } + + /** + * Rewrites a relational expression, replacing correlation variables + * with a similar one but proper row type. + */ + public static RelNode go(RexBuilder builder, Iterable corrIds, + RelDataType expectedType, RelNode r) { + return r.accept( + new ChangeTypeOfCorrelateVariables(builder, ImmutableSet.copyOf(corrIds), expectedType)); + } + + @Override public RelNode visit(RelNode other) { + RelNode next = super.visit(other); + return next.accept(dedupRex); + } + + /** + * Replaces row type of correlation variable to the expected one. + */ + private static class ChangeTypeOfCorrelateVariablesShuttle extends RexShuttle { + private final RexBuilder builder; + private final ImmutableSet corrIds; + private final RelDataType expectedType; + @NotOnlyInitialized + private final ChangeTypeOfCorrelateVariables shuttle; + + private ChangeTypeOfCorrelateVariablesShuttle(RexBuilder builder, + ImmutableSet corrIds, RelDataType expectedType, + @UnderInitialization ChangeTypeOfCorrelateVariables shuttle) { + this.builder = builder; + this.corrIds = corrIds; + this.expectedType = expectedType; + this.shuttle = shuttle; + } + + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + RexNode before = fieldAccess.getReferenceExpr(); + RexNode after = before.accept(this); + + if (before == after) { + return fieldAccess; + } else { + return builder.makeFieldAccess(after, + fieldAccess.getField().getName(), true); + } + } + + @Override public RexNode visitCorrelVariable(RexCorrelVariable variable) { + if (!corrIds.contains(variable.id) || variable.getType().equals(expectedType)) { + return variable; + } + + return builder.makeCorrel(expectedType, variable.id); + } + + @Override public RexNode visitSubQuery(RexSubQuery subQuery) { + if (shuttle != null) { + RelNode r = subQuery.rel.accept(shuttle); // look inside sub-queries + if (r != subQuery.rel) { + subQuery = subQuery.clone(r); + } + } + return super.visitSubQuery(subQuery); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 2a53e9427cde..77df3eacbcc2 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -531,8 +531,14 @@ public TrimResult trimFields( mapping); relBuilder.push(newInput); - relBuilder.project(newProjects, newRowType.getFieldNames()); - final RelNode newProject = RelOptUtil.propagateRelHints(project, relBuilder.build()); + relBuilder.projectCorrelated(project.getVariablesSet(), + newProjects, newRowType.getFieldNames()); + final RelNode newProject = ChangeTypeOfCorrelateVariables.go( + project.getCluster().getRexBuilder(), + project.getVariablesSet(), + newInput.getRowType(), + RelOptUtil.propagateRelHints(project, relBuilder.build()) + ); return result(newProject, mapping); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index f4bf28aa245c..90c4fc27c8f1 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -27,6 +27,7 @@ import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Collect; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sample; import org.apache.calcite.rel.core.Sort; @@ -534,10 +535,21 @@ public void rewriteRel(LogicalProject rel) { RelNode newInput = getNewForOldRel(rel.getInput()); List newProjects = Pair.left(flattenedExpList); List newNames = Pair.right(flattenedExpList); - final RelNode newRel = relBuilder.push(newInput) + RelNode renamedProject = relBuilder.push(newInput) .projectNamed(newProjects, newNames, true) .hints(rel.getHints()) .build(); + + final RelNode newRel; + // we need to check if builder returns an instance of project + // because in some case a Values relation could be returned instead + if (!rel.getVariablesSet().isEmpty() && renamedProject instanceof Project) { + Project p = (Project) renamedProject; + newRel = p.copy(p.getTraitSet(), p.getInput(), p.getProjects(), + p.getRowType(), rel.getVariablesSet()); + } else { + newRel = renamedProject; + } setNewForOldRel(rel, newRel); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 91d223c62bf6..e106ce67ab9a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2696,16 +2696,6 @@ protected void convertCollectionTable( validator().getValidatedNodeType(call), columnMappings); - final SqlValidatorScope selectScope = - ((DelegatingScope) bb.scope()).getParent(); - final Blackboard seekBb = createBlackboard(selectScope, null, false); - - final CorrelationUse p = getCorrelationUse(seekBb, callRel); - if (p != null) { - assert p.r instanceof LogicalTableFunctionScan; - callRel = (LogicalTableFunctionScan) p.r; - } - bb.setRoot(callRel, true); afterTableFunction(bb, call, callRel); } @@ -4400,7 +4390,21 @@ private void convertSelectList( relBuilder.push(bb.root()) .projectNamed(exprs, fieldNames, true); - bb.setRoot(relBuilder.build(), false); + + RelNode project = relBuilder.build(); + + RelNode r; + final CorrelationUse p = getCorrelationUse(bb, project); + if (p != null) { + assert p.r instanceof Project; + Project proj = (Project) p.r; + r = proj.copy(proj.getTraitSet(), proj.getInput(), proj.getProjects(), + proj.getRowType(), ImmutableSet.of(p.id)); + } else { + r = project; + } + + bb.setRoot(r, false); assert bb.columnMonotonicities.isEmpty(); bb.columnMonotonicities.addAll(columnMonotonicityList); @@ -6475,7 +6479,8 @@ private class NestedJsonFunctionRelRewriter extends RelShuttleImpl { newInput, project.getHints(), newProjections.build(), - project.getRowType().getFieldNames()); + project.getRowType().getFieldNames(), + project.getVariablesSet()); } private Set requiredJsonOutputFromParent(RelNode relNode) { diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 6eb38385b36c..87cfd2724f32 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -1817,7 +1817,74 @@ public RelBuilder project(Iterable nodes, */ public RelBuilder project(Iterable nodes, Iterable fieldNames, boolean force) { - return project_(nodes, fieldNames, ImmutableList.of(), force); + return project_(ImmutableSet.of(), nodes, fieldNames, ImmutableList.of(), force); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given expressions. */ + public RelBuilder projectCorrelated(Iterable variablesSet, + RexNode... nodes) { + return projectCorrelated(variablesSet, ImmutableList.copyOf(nodes)); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions. + * + *

Infers names as would {@link #projectCorrelated(Iterable, Iterable, Iterable)} + * if all suggested names were null. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + */ + public RelBuilder projectCorrelated(Iterable variablesSet, + Iterable nodes) { + return projectCorrelated(variablesSet, nodes, ImmutableList.of()); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions and field names. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + * @param fieldNames field names for expressions + */ + public RelBuilder projectCorrelated( + Iterable variablesSet, + Iterable nodes, + Iterable fieldNames) { + return projectCorrelated(variablesSet, nodes, fieldNames, false); + } + + /** Creates a {@link Project} of a set of correlation variables + * and the given list of expressions, using the given names. + * + *

Names are deduced as follows: + *

    + *
  • If the length of {@code fieldNames} is greater than the index of + * the current entry in {@code nodes}, and the entry in + * {@code fieldNames} is not null, uses it; otherwise + *
  • If an expression projects an input field, + * or is a cast an input field, + * uses the input field name; otherwise + *
  • If an expression is a call to + * {@link SqlStdOperatorTable#AS} + * (see {@link #alias}), removes the call but uses the intended alias. + *
+ * + *

After the field names have been inferred, makes the + * field names unique by appending numeric suffixes. + * + * @param variablesSet Set of correlation variables + * @param nodes Expressions + * @param fieldNames Suggested field names + * @param force create project even if it is identity + */ + public RelBuilder projectCorrelated( + Iterable variablesSet, + Iterable nodes, + Iterable fieldNames, + boolean force) { + return project_(variablesSet, nodes, fieldNames, ImmutableList.of(), force); } /** Creates a {@link Project} of all original fields, plus the given @@ -1882,12 +1949,14 @@ public RelBuilder projectExcept(Iterable expressions) { *

After the field names have been inferred, makes the * field names unique by appending numeric suffixes. * + * @param variablesSet Set of correlation variables * @param nodes Expressions * @param fieldNames Suggested field names * @param hints Hints * @param force create project even if it is identity */ private RelBuilder project_( + Iterable variablesSet, Iterable nodes, Iterable fieldNames, Iterable hints, @@ -1958,7 +2027,7 @@ private RelBuilder project_( final ImmutableSet.Builder mergedHints = ImmutableSet.builder(); mergedHints.addAll(project.getHints()); mergedHints.addAll(hints); - return project_(newNodes, fieldNameList, mergedHints.build(), force); + return project_(variablesSet, newNodes, fieldNameList, mergedHints.build(), force); } // Simplify expressions. @@ -2043,7 +2112,8 @@ private RelBuilder project_( struct.projectFactory.createProject(frame.rel, ImmutableList.copyOf(hints), ImmutableList.copyOf(nodeList), - fieldNameList); + fieldNameList, + ImmutableSet.copyOf(variablesSet)); stack.pop(); stack.push(new Frame(project, fields.build())); return this; @@ -2085,7 +2155,8 @@ public RelBuilder projectNamed(Iterable nodes, final Frame frame = stack.pop(); final Project childProject = (Project) frame.rel; final Project newInput = childProject.copy(childProject.getTraitSet(), - childProject.getInput(), childProject.getProjects(), rowType); + childProject.getInput(), childProject.getProjects(), rowType, + childProject.getVariablesSet()); stack.push(new Frame(newInput.attachHints(childProject.getHints()), frame.fields)); } if (input instanceof Values && fieldNameList != null) { @@ -2345,7 +2416,7 @@ public RelBuilder aggregate(GroupKey groupKey, Iterable aggCalls) { builder.add(project.getRowType().getFieldList().get(i)); } r = project.copy(cluster.traitSet(), project.getInput(), newProjects, - builder.build()); + builder.build(), project.getVariablesSet()); } } @@ -3278,7 +3349,8 @@ public RelBuilder sortLimit(@Nullable RexNode offsetNode, @Nullable RexNode fetc struct.projectFactory.createProject(sort, project.getHints(), project.getProjects(), - Pair.right(project.getNamedProjects()))); + Pair.right(project.getNamedProjects()), + project.getVariablesSet())); return this; } } diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index 602984c5cf1b..a92764b5617a 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -150,7 +150,7 @@ public abstract class Bug { /** Whether * [CALCITE-1045] * Decorrelate sub-queries in Project and Join is fixed. */ - public static final boolean CALCITE_1045_FIXED = false; + public static final boolean CALCITE_1045_FIXED = true; /** * Whether diff --git a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java index 2f22e78ee723..54e9c88d16b2 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelOptUtilTest.java @@ -49,6 +49,7 @@ import org.apache.calcite.util.Util; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -693,7 +694,8 @@ private void splitJoinConditionHelper(RexNode joinCond, List expLeftKey ImmutableList.of( fieldEmpno.getName(), fieldEname.getName(), - "JOB_CNT")); + "JOB_CNT"), + ImmutableSet.of()); assertThat(castNode1.explain(), is(expectNode1.explain())); // Change the field JOB_CNT field name again. // The projection expect to be merged. @@ -716,7 +718,8 @@ private void splitJoinConditionHelper(RexNode joinCond, List expLeftKey ImmutableList.of( fieldEmpno.getName(), fieldEname.getName(), - "JOB_CNT2")); + "JOB_CNT2"), + ImmutableSet.of()); assertThat(castNode2.explain(), is(expectNode2.explain())); } diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index f2277fdc4141..0b310116d965 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttleImpl; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; @@ -612,6 +613,46 @@ private static Fixture relFn(Function relFn) { .assertThatPlan(isLinux(expected)); } + @Test void testProject() { + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder builder = RelBuilder.create(config); + final RelNode rel = builder + .scan("EMP") + .project(builder.alias(builder.field(0), "a")) + .build(); + final String relJson = RelOptUtil.dumpPlan("", rel, + SqlExplainFormat.JSON, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + String s = deserializeAndDumpToTextFormat(getSchema(rel), relJson); + final String expected = "" + + "LogicalProject(a=[$0])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(s, isLinux(expected)); + } + + @Test void testCorrelatedProject() { + final FrameworkConfig config = RelBuilderTest.config().build(); + final RelBuilder builder = RelBuilder.create(config); + final CorrelationId corr = builder.getCluster().createCorrel(); + final RelNode rel = builder + .scan("EMP") + .projectCorrelated( + ImmutableSet.of(corr), + builder.alias( + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.VARCHAR), corr), + "a" + ) + ) + .build(); + final String relJson = RelOptUtil.dumpPlan("", rel, + SqlExplainFormat.JSON, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + String s = deserializeAndDumpToTextFormat(getSchema(rel), relJson); + final String expected = "" + + "LogicalProject(correlation=[[$cor0]], a=[$cor0])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + assertThat(s, isLinux(expected)); + } + @Test public void testExchangeWithDistributionTraitDef() { final Function relFn = b -> b.scan("EMP") diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index 825e1434691a..d8e0781b4caf 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -43,6 +43,7 @@ import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -71,7 +72,9 @@ import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; @@ -82,6 +85,7 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -370,7 +374,7 @@ public Aggregate copy(RelTraitSet traitSet, RelNode input, private static class PhysProj extends Project implements Phys { PhysProj(RelOptCluster cluster, RelTraitSet traits, RelNode child, List exps, RelDataType rowType) { - super(cluster, traits, ImmutableList.of(), child, exps, rowType); + super(cluster, traits, ImmutableList.of(), child, exps, rowType, ImmutableSet.of()); } public static PhysProj create(final RelNode input, @@ -386,7 +390,9 @@ public static PhysProj create(final RelNode input, } public PhysProj copy(RelTraitSet traitSet, RelNode input, - List exps, RelDataType rowType) { + List exps, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new PhysProj(getCluster(), traitSet, input, exps, rowType); } diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index 88a7cbad4236..943266e57621 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -1191,6 +1191,66 @@ private RexNode caseCall(RelBuilder b, RexNode ref, RexNode... nodes) { assertThat(root, hasTree(expected)); } + @Test void testConvertCorrelatedProject() { + final RelBuilder builder = RelBuilder.create(config().build()); + + CorrelationId cor1 = builder.getCluster().createCorrel(); + CorrelationId cor2 = builder.getCluster().createCorrel(); + + RelDataType rowType = + builder.getTypeFactory().builder() + .add("a", SqlTypeName.BIGINT) + .add("b", SqlTypeName.BIGINT) + .build(); + + RelNode root = + builder + .scan("DEPT") + .projectCorrelated( + ImmutableSet.of(cor1, cor2), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor1), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor2) + ) + .convert(rowType, false) + .build(); + final String expected = "" + + "LogicalProject(correlation=[[$cor0, $cor1]], $f0=[CAST($cor0):BIGINT NOT NULL], $f1=[CAST($cor1):BIGINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); + } + + @Test void testConvertCorrelatedProjectWithRename() { + final RelBuilder builder = RelBuilder.create(config().build()); + + CorrelationId cor1 = builder.getCluster().createCorrel(); + CorrelationId cor2 = builder.getCluster().createCorrel(); + + RelDataType rowType = + builder.getTypeFactory().builder() + .add("a", SqlTypeName.BIGINT) + .add("b", SqlTypeName.BIGINT) + .build(); + + RelNode root = + builder + .scan("DEPT") + .projectCorrelated( + ImmutableSet.of(cor1, cor2), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor1), + builder.getRexBuilder().makeCorrel(builder.getTypeFactory() + .createSqlType(SqlTypeName.INTEGER), cor2) + ) + .convert(rowType, true) + .build(); + final String expected = "" + + "LogicalProject(correlation=[[$cor0, $cor1]], a=[CAST($cor0):BIGINT NOT NULL], b=[CAST($cor1):BIGINT NOT NULL])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + assertThat(root, hasTree(expected)); + } + /** Test case for * [CALCITE-4429] * RelOptUtil#createCastRel should throw an exception when the desired row type diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index dc4b1a41d5e7..ee1430cd6d00 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -125,7 +125,9 @@ import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.ImmutableBitSet; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.immutables.value.Value; import org.junit.jupiter.api.Disabled; @@ -136,6 +138,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; @@ -6745,11 +6748,13 @@ private static class MyProject extends Project { RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); } public MyProject copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new MyProject(getCluster(), traitSet, input, projects, rowType); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 41b38ec74697..64183548a5d9 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1288,6 +1288,19 @@ class SqlToRelConverterTest extends SqlToRelTestBase { + "from emp e"); } + @Test void testCorrelatedScalarSubQueryInSelectList() { + Consumer fn = sql -> { + sql(sql).withExpand(true).withDecorrelate(false) + .convertsTo("${planExpanded}"); + sql(sql).withExpand(false).withDecorrelate(false) + .convertsTo("${planNotExpanded}"); + }; + fn.accept("select deptno,\n" + + " (select min(1) from emp where empno > d.deptno) as i0,\n" + + " (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1\n" + + "from dept as d"); + } + @Test void testCorrelationLateralSubQuery() { String sql = "SELECT deptno, ename\n" + "FROM\n" diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index fa18df0f9a64..f25546e5cabb 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11756,7 +11756,7 @@ from emp SOME($0, { +LogicalProject(correlation=[[$cor0]], EXPR$0=[> SOME($0, { LogicalProject(DEPTNO=[$0]) LogicalFilter(condition=[=($cor0.JOB, $1)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) @@ -11798,7 +11798,7 @@ LogicalProject(EXPR$0=[CAST(OR(AND(IS TRUE(>($0, $9)), IS NOT TRUE(OR(IS NULL($1 + + + ($0, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})], I1=[$SCALAR_QUERY({ +LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(=($7, $cor0.DEPTNO), =($1, 'SMITH'))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +})]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + ($0, $cor0.DEPTNO)]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalAggregate(group=[{}], EXPR$0=[MIN($0)]) + LogicalProject($f0=[0]) + LogicalFilter(condition=[AND(=($7, $cor1.DEPTNO), =($1, 'SMITH'))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + d.deptno) as i0, + (select min(0) from emp where deptno = d.deptno and ename = 'SMITH') as i1 +from dept as d]]> + + e.deptno); - EMPNO -------- - 7369 - 7566 - 7782 - 7876 - 7934 ++-------+ +| EMPNO | ++-------+ +| 7369 | +| 7566 | +| 7782 | +| 7876 | +| 7934 | ++-------+ (5 rows) !ok @@ -330,13 +332,15 @@ EnumerableCalc(expr#0..5=[{inputs}], EMPNO=[$t0]) EnumerableHashJoin(condition=[=($2, $5)], joinType=[inner]) EnumerableCalc(expr#0..4=[{inputs}], EMPNO=[$t2], JOB=[$t3], DEPTNO=[$t4], JOB0=[$t0], DEPTNO0=[$t1]) EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) - EnumerableCalc(expr#0..1=[{inputs}], JOB=[$t1], DEPTNO=[$t0]) - EnumerableAggregate(group=[{0, 2}]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[>($t3, $t0)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) - EnumerableAggregate(group=[{7}]) - EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableAggregate(group=[{1, 3}]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) @@ -351,7 +355,57 @@ select empno from "scott".emp as e join "scott".dept as d using (deptno) where e.job not in ( select e2.job from "scott".emp as e2 where e2.deptno > e.deptno); + ++-------+ +| EMPNO | ++-------+ +| 7499 | +| 7521 | +| 7654 | +| 7698 | +| 7788 | +| 7839 | +| 7844 | +| 7900 | +| 7902 | ++-------+ +(9 rows) + !ok + +EnumerableCalc(expr#0..9=[{inputs}], expr#10=[0], expr#11=[=($t5, $t10)], expr#12=[IS NULL($t1)], expr#13=[IS NOT NULL($t9)], expr#14=[<($t6, $t5)], expr#15=[OR($t12, $t13, $t14)], expr#16=[IS NOT TRUE($t15)], expr#17=[OR($t11, $t16)], EMPNO=[$t0], $condition=[$t17]) + EnumerableMergeJoin(condition=[AND(=($1, $7), =($2, $8))], joinType=[left]) + EnumerableSort(sort0=[$1], sort1=[$2], dir0=[ASC], dir1=[ASC]) + EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[inner]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{3}], c=[COUNT()], ck=[COUNT($1)]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) + EnumerableAggregate(group=[{1, 3}]) + EnumerableNestedLoopJoin(condition=[>($2, $3)], joinType=[inner]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], JOB=[$t2], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{2}]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], DEPTNO=[$t7]) + EnumerableTableScan(table=[[scott, EMP]]) !plan !} @@ -555,17 +609,35 @@ select deptno, (select min(1) from "scott".emp where empno > d.deptno) as i0, (select min(0) from "scott".emp where deptno = d.deptno and ename = 'SMITH') as i1 from "scott".dept as d; -+--------+----+---+ -| DEPTNO | I0 | I1| -+--------+----+---+ -| 10 | 1 | | -| 20 | 1 | 0 | -| 30 | 1 | | -| 40 | 1 | | -+--------+----+---+ ++--------+----+----+ +| DEPTNO | I0 | I1 | ++--------+----+----+ +| 10 | 1 | | +| 20 | 1 | 0 | +| 30 | 1 | | +| 40 | 1 | | ++--------+----+----+ (4 rows) !ok + +EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}], I1=[$t3]) + EnumerableHashJoin(condition=[=($0, $2)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0], EXPR$0=[$t2]) + EnumerableHashJoin(condition=[=($0, $1)], joinType=[left]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableAggregate(group=[{0}], EXPR$0=[MIN($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], DEPTNO0=[$t0], $f0=[$t2]) + EnumerableNestedLoopJoin(condition=[>($1, $0)], joinType=[inner]) + EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableAggregate(group=[{0}], EXPR$0=[MIN($1)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=['SMITH':VARCHAR(10)], expr#10=[=($t1, $t9)], expr#11=[IS NOT NULL($t7)], expr#12=[AND($t10, $t11)], DEPTNO=[$t7], $f0=[$t8], $condition=[$t12]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan !} # [CALCITE-1494] Inefficient plan for correlated sub-queries @@ -3462,4 +3534,38 @@ where NOT EXISTS (select count(*) from emp e having false); EnumerableTableScan(table=[[scott, DEPT]]) !plan +# Test collection from a correlated subquery + +select dname, + array (select deptno + from emp + where deptno = dept.deptno) as deptno_array, + multiset (select deptno + from emp + where deptno = dept.deptno) as deptno_multiset +from dept; ++------------+--------------------------------------+--------------------------------------+ +| DNAME | DEPTNO_ARRAY | DEPTNO_MULTISET | ++------------+--------------------------------------+--------------------------------------+ +| ACCOUNTING | [{10}, {10}, {10}] | [{10}, {10}, {10}] | +| OPERATIONS | [] | [] | +| RESEARCH | [{20}, {20}, {20}, {20}, {20}] | [{20}, {20}, {20}, {20}, {20}] | +| SALES | [{30}, {30}, {30}, {30}, {30}, {30}] | [{30}, {30}, {30}, {30}, {30}, {30}] | ++------------+--------------------------------------+--------------------------------------+ +(4 rows) + +!ok + +EnumerableCalc(expr#0..4=[{inputs}], DNAME=[$t1], DEPTNO_ARRAY=[$t3], DEPTNO_MULTISET=[$t4]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}]) + EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}]) + EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCollect(field=[x]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[$cor0], expr#9=[$t8.DEPTNO], expr#10=[=($t7, $t9)], DEPTNO=[$t7], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCollect(field=[x]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[$cor0], expr#9=[$t8.DEPTNO], expr#10=[=($t7, $t9)], DEPTNO=[$t7], $condition=[$t10]) + EnumerableTableScan(table=[[scott, EMP]]) +!plan + # End sub-query.iq diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index 3c7800f38044..86389711945c 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -58,6 +58,7 @@ import org.apache.commons.lang3.tuple.Triple; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.immutables.value.Value; @@ -395,10 +396,11 @@ protected DruidProjectRule(DruidProjectRuleConfig config) { } builder.add(name, e.getType()); } - final RelNode newProject = project.copy(project.getTraitSet(), input, below, builder.build()); + final RelNode newProject = project.copy(project.getTraitSet(), input, below, builder.build(), + ImmutableSet.of()); final DruidQuery newQuery = DruidQuery.extendQuery(query, newProject); final RelNode newProject2 = project.copy(project.getTraitSet(), newQuery, above, - project.getRowType()); + project.getRowType(), project.getVariablesSet()); call.transformTo(newProject2); } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java index 0a341c683c30..c06295b3efe4 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java @@ -22,18 +22,22 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -43,13 +47,15 @@ public class ElasticsearchProject extends Project implements ElasticsearchRel { ElasticsearchProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == ElasticsearchRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet relTraitSet, RelNode input, List projects, - RelDataType relDataType) { + RelDataType relDataType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new ElasticsearchProject(getCluster(), traitSet, input, projects, relDataType); } diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java index 78261b9ae59b..b7a8473ad2c5 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchRules.java @@ -284,7 +284,8 @@ protected ElasticsearchAggregateRule(Config config) { */ private static class ElasticsearchProjectRule extends ElasticsearchConverterRule { private static final ElasticsearchProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, ElasticsearchRel.CONVENTION, "ElasticsearchProjectRule") .withRuleFactory(ElasticsearchProjectRule::new) .toRule(ElasticsearchProjectRule.class); diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java index da5cf40a7399..48f2fbe887d7 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeProject.java @@ -22,19 +22,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of @@ -45,13 +49,15 @@ public class GeodeProject extends Project implements GeodeRel { GeodeProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == GeodeRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new GeodeProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java index 5fd12fa3e67f..4cf376200726 100644 --- a/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java +++ b/geode/src/main/java/org/apache/calcite/adapter/geode/rel/GeodeRules.java @@ -128,7 +128,8 @@ private static String stripQuotes(String s) { */ private static class GeodeProjectRule extends GeodeConverterRule { private static final GeodeProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, GeodeRel.CONVENTION, "GeodeProjectRule") .withRuleFactory(GeodeProjectRule::new) .toRule(GeodeProjectRule.class); diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java index b7e657ac07df..92c8ec29e743 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbProject.java @@ -21,19 +21,23 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Pair; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -42,13 +46,15 @@ public class InnodbProject extends Project implements InnodbRel { InnodbProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == InnodbRel.CONVENTION; assert getConvention() == input.getConvention(); } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new InnodbProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java index cf381de24715..c64356ea4c6b 100644 --- a/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java +++ b/innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbRules.java @@ -125,7 +125,8 @@ abstract static class InnodbConverterRule extends ConverterRule { public static class InnodbProjectRule extends InnodbConverterRule { /** Default configuration. */ private static final Config DEFAULT_CONFIG = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, InnodbRel.CONVENTION, "InnodbProjectRule") .withRuleFactory(InnodbProjectRule::new); diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java index 55bce98e56ea..b6ff2bc9e970 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoProject.java @@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; @@ -29,12 +30,15 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Implementation of {@link org.apache.calcite.rel.core.Project} @@ -43,7 +47,7 @@ public class MongoProject extends Project implements MongoRel { public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == MongoRel.CONVENTION; assert getConvention() == input.getConvention(); } @@ -56,7 +60,9 @@ public MongoProject(RelOptCluster cluster, RelTraitSet traitSet, } @Override public Project copy(RelTraitSet traitSet, RelNode input, - List projects, RelDataType rowType) { + List projects, RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new MongoProject(getCluster(), traitSet, input, projects, rowType); } diff --git a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java index 9e9ac37fca57..43ba8e94608b 100644 --- a/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java +++ b/mongodb/src/main/java/org/apache/calcite/adapter/mongodb/MongoRules.java @@ -301,7 +301,8 @@ private static class MongoFilterRule extends MongoConverterRule { */ private static class MongoProjectRule extends MongoConverterRule { static final MongoProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, MongoRel.CONVENTION, "MongoProjectRule") .withRuleFactory(MongoProjectRule::new) .toRule(MongoProjectRule.class); diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java index 11d2e90fbd63..518c54d8164a 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigProject.java @@ -20,13 +20,17 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Set; /** Implementation of {@link org.apache.calcite.rel.core.Project} in * {@link PigRel#CONVENTION Pig calling convention}. */ @@ -35,12 +39,14 @@ public class PigProject extends Project implements PigRel { /** Creates a PigProject. */ public PigProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - super(cluster, traitSet, ImmutableList.of(), input, projects, rowType); + super(cluster, traitSet, ImmutableList.of(), input, projects, rowType, ImmutableSet.of()); assert getConvention() == PigRel.CONVENTION; } @Override public Project copy(RelTraitSet traitSet, RelNode input, List projects, - RelDataType rowType) { + RelDataType rowType, Set variablesSet) { + Preconditions.checkArgument(variablesSet.isEmpty(), + "Correlated scalar subqueries are not supported"); return new PigProject(input.getCluster(), traitSet, input, projects, rowType); } diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java index 3def5e0c7203..e255e460b633 100644 --- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java +++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRules.java @@ -99,7 +99,8 @@ protected PigTableScanRule(Config config) { */ private static class PigProjectRule extends ConverterRule { private static final PigProjectRule INSTANCE = Config.INSTANCE - .withConversion(LogicalProject.class, Convention.NONE, + .withConversion(LogicalProject.class, p -> p.getCorrelVariable() == null + && p.getVariablesSet().isEmpty(), Convention.NONE, PigRel.CONVENTION, "PigProjectRule") .withRuleFactory(PigProjectRule::new) .toRule(PigProjectRule.class); diff --git a/site/_docs/algebra.md b/site/_docs/algebra.md index 66b2792779ea..11342e789f87 100644 --- a/site/_docs/algebra.md +++ b/site/_docs/algebra.md @@ -326,6 +326,7 @@ return the `RelBuilder`. | `values(fieldNames, value...)`
`values(rowType, tupleList)` | Creates a [Values]({{ site.apiRoot }}/org/apache/calcite/rel/core/Values.html). | `filter([variablesSet, ] exprList)`
`filter([variablesSet, ] expr...)` | Creates a [Filter]({{ site.apiRoot }}/org/apache/calcite/rel/core/Filter.html) over the AND of the given predicates; if `variablesSet` is specified, the predicates may reference those variables. | `project(expr...)`
`project(exprList [, fieldNames])` | Creates a [Project]({{ site.apiRoot }}/org/apache/calcite/rel/core/Project.html). To override the default name, wrap expressions using `alias`, or specify the `fieldNames` argument. +| `projectCorrelated(variablesSet, expr...)`
`projectCorrelated(variablesSet, exprList [, fieldNames])` | Variant of `project` that creates correlated project. | `projectPlus(expr...)`
`projectPlus(exprList)` | Variant of `project` that keeps original fields and appends the given expressions. | `projectExcept(expr...)`
`projectExcept(exprList)` | Variant of `project` that keeps original fields and removes the given expressions. | `permute(mapping)` | Creates a [Project]({{ site.apiRoot }}/org/apache/calcite/rel/core/Project.html) that permutes the fields using `mapping`. diff --git a/site/_docs/history.md b/site/_docs/history.md index eaa1a2efdcf8..711901ecec5a 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -210,6 +210,155 @@ mans2singh. * Site: Add syntax highlighting to SQL statements * Site: Improve HTML tables display & update CSV tutorial +## 1.30.0 / 2022-03-04 +{: #v1-30-0} + +This release comes two months after [1.29.0](#v1-29-0), +contains contributions from 29 authors, +and resolves 37 issues. + +Compatibility: This release is tested on Linux, macOS, Microsoft Windows; +using JDK/OpenJDK versions 8 to 17; +Guava versions 19.0 to 31.0.1-jre; +other software versions as specified in gradle.properties. + +Contributors to this release: + +Alessandro Solimando, +Bill Neil, +Chen Kai, +Eugen Stan, +Feng Zhu, +Jacques Nadeau, +Jake Xie, +Jay Narale, +Jiatao Tao, +Jing Zhang, +Julian Hyde, +LM Kang, +Liya Fan (release manager), +Marco Jorge, +Marieke Gueye, +NobiGo, +Roman Puchkovskiy, +Ruben Quesada Lopez, +Scott Reynolds, +Soumyakanti Das, +Stamatis Zampetakis, +Vova Vysotskyi, +Will Noble, +Xiong Duan, +Yanjing Wang, +Yiqun Zhang, +Xurenhe, +Zhe Hu, +mans2singh. + +#### New features +{: #new-features-1-30-0} + +* [CALCITE-4822] + Add `ARRAY_CONCAT`, `ARRAY_REVERSE`, `ARRAY_LENGTH` functions for BigQuery dialect +* [CALCITE-4980] + Make Babel parser support MySQL NULL-safe equal operator '<=>' +* [CALCITE-4967] + Support SQL hints for temporal table join +* [CALCITE-3673] + Support `ListTransientTable` without tables in the schema + +### Improvements +{: #fixes-1-30-0} + +* [CALCITE-5019] + Avoid multiple scans when the table is `ProjectableFilterableTable` +* [CALCITE-5008] + Ignore synthetic and static methods in `MetadataDef` during instrumentation +* [CALCITE-4996] + In `RelJson`, add a `readExpression` method that converts JSON to a RexNode expression +* [CALCITE-4994] + Improve the performance of SQL-to-RelNode conversion when the table contains hundreds of fields +* [CALCITE-4991] + Make RuleEventLogger print input rels in `FULL_PLAN` mode +* [CALCITE-4986] + Make `HepProgram` thread-safe +* [CALCITE-4963] + Improve the extensibility of `SqlDialectFactory` by making the default behavior in + `SqlDialectFactoryImpl` reusable +* [CALCITE-4960] + Enable unit tests in Elasticsearch Adapter +* [CALCITE-4953] + Deprecate `TableAccessMap` class +* [CALCITE-4952] + Enable use of RelMetadataQuery through a simplistic & slow proxy path +* [CALCITE-4885] + Fluent test fixtures so that dependent projects can write parser, validator and rules tests +* [CALCITE-1794] + Simplify expressions with numeric comparisons when CAST is present + +#### Bug-fixes +{: #fixes-1-30-0} + +* [CALCITE-5011] + Fix the initialization error for `CassandraAdapterDataTypesTest` +* [CALCITE-5006] + Make Gradle tasks for launching JDBC integration tests working +* [CALCITE-4997] + Keep `APPROX_COUNT_DISTINCT` in some `SqlDialect`s +* [CALCITE-4995] + Fix `AssertionError` caused by `RelFieldTrimmer` on SEMI/ANTI join +* [CALCITE-4988] + Expression `((A IS NOT NULL OR B) AND A IS NOT NULL)` can't be simplify to `(A IS NOT NULL)` when `A` + is deterministic +* [CALCITE-4968] + Fix the invalid MS SQL queries generated by Calcite for the query with `LIMIT` statement +* [CALCITE-4965] + Fix the failure of IS NOT NULL in Elasticsearch Adapter +* [CALCITE-4912] + Fix the confusing Javadoc of `RexSimplify#simplify` +* [CALCITE-4901] + Fix the problem that JDBC adapter incorrectly adds ORDER BY columns to the SELECT + list of generated SQL query +* [CALCITE-4872] + Fix the problem that UNKNOWN SqlTypeName is erroneously treated as NULL +* [CALCITE-4702] + Fix the error when executing query with GROUP BY constant via JDBC adapter +* [CALCITE-4683] + Fix the type mismatch exception when IN-list is converted to JOIN +* [CALCITE-4323] + If a view definition has an ORDER BY clause, retain the sort if the view is used in a + query at top level +* [CALCITE-4292] + Fix wrong results in ElasticSearch when query contains 'NOT EQUAL' +* [CALCITE-4054] + Fix NPE caused by RepeatUnion containing a Correlate with a transientScan on its RHS +* [CALCITE-3627] + Fix the incorrect null semantic for ROW function + +#### Dependency version upgrade +{: #dependency-1-30-0} + +* [CALCITE-5030] + Upgrade jsonpath version from 2.4.0 to 2.7.0 +* [CALCITE-5025] + Upgrade commons-io version from 2.4 to 2.11.0 +* [CALCITE-5007] + Upgrade H2 database version to 2.1.210 +* [CALCITE-4973] + Upgrade log4j2 version to 2.17.1 + +#### Web site and documentation +{: #site-1-30-0} + +* Site: Update PMC Chair +* Site: Add external resources section in the community page +* Site: Add "calcite-clj - Use Calcite with Clojure" in talks section +* Site: Add Alessandro Solimando as committer +* Site: Fix typo in howto.md +* Site: Change the javadoc title to Apache Calcite API +* Site: For tables that display results, center the content horizontally +* Site: Add syntax highlighting to SQL statements +* Site: Improve HTML tables display & update CSV tutorial + ## 1.29.0 / 2021-12-26 {: #v1-29-0} diff --git a/site/_posts/2022-03-04-release-1.30.0.md b/site/_posts/2022-03-04-release-1.30.0.md new file mode 100644 index 000000000000..8a91ec8659b9 --- /dev/null +++ b/site/_posts/2022-03-04-release-1.30.0.md @@ -0,0 +1,38 @@ +--- +layout: news_item +date: "2022-03-04 00:00:00 +0800" +author: liyafan82 +version: 1.30.0 +categories: [release] +tag: v1-30-0 +sha: +--- + + +The [Apache Calcite PMC]({{ site.baseurl }}) +is pleased to announce +[Apache Calcite release 1.30.0]({{ site.baseurl }}/docs/history.html#v1-30-0). + +This release comes two months after [1.29.0](#v1-29-0), +contains contributions from 29 authors, +and resolves 37 issues. + +This release fixes vulnerability issues +such as CVE-2021-27568.