From 128fab983285954ce38828ee9a418903c1a66882 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Wed, 4 Sep 2019 15:50:32 +0800 Subject: [PATCH 01/24] Customize README.md --- README.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a9cfee9a604f..38938e820f40 100644 --- a/README.md +++ b/README.md @@ -16,21 +16,15 @@ See the License for the specific language governing permissions and limitations under the License. {% endcomment %} --> -[![Travis Build Status](https://travis-ci.org/apache/calcite.svg?branch=master)](https://travis-ci.org/apache/calcite) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/apache/calcite?svg=true&branch=master)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/calcite) +[![Travis Build Status](https://travis-ci.com/yunpengn/calcite.svg?branch=master)](https://travis-ci.com/yunpengn/calcite) +[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/qo30vjfl2rwsapnx?svg=true)](https://ci.appveyor.com/project/yunpengn/calcite) # Apache Calcite -Apache Calcite is a dynamic data management framework. +This is a forked version of the [Apache Calcite](http://calcite.apache.org) framework, with enhancements on outer join reorderability. _We do NOT guarantee compatibility with its upstream version._ -It contains many of the pieces that comprise a typical -database management system but omits the storage primitives. -It provides an industry standard SQL parser and validator, -a customisable optimizer with pluggable rules and cost functions, -logical and physical algebraic operators, various transformation -algorithms from SQL to algebra (and the opposite), and many -adapters for executing SQL queries over Cassandra, Druid, -Elasticsearch, MongoDB, Kafka, and others, with minimal -configuration. +This [repository](https://github.com/yunpengn/calcite) is currently maintained by **[Yunpeng Niu](https://github.com/yunpengn)**. -For more details, see the [home page](http://calcite.apache.org). +## Licence + +[Apache Licence 2.0](LICENSE) From a913801b46e42d6c648351b5b6fa992a7ae15f7e Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sat, 14 Sep 2019 15:04:37 +0800 Subject: [PATCH 02/24] Add nullify operator --- .../main/java/org/apache/calcite/Runner.java | 117 ++++++++++++++++++ .../org/apache/calcite/rel/core/Nullify.java | 93 ++++++++++++++ .../apache/calcite/rel/core/RelFactories.java | 32 +++++ .../calcite/rel/logical/LogicalNullify.java | 90 ++++++++++++++ .../calcite/rel/rules/NullifyJoinRule.java | 108 ++++++++++++++++ .../org/apache/calcite/tools/RelBuilder.java | 44 +++++++ 6 files changed, 484 insertions(+) create mode 100644 core/src/main/java/org/apache/calcite/Runner.java create mode 100644 core/src/main/java/org/apache/calcite/rel/core/Nullify.java create mode 100644 core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java new file mode 100644 index 000000000000..b40d569808eb --- /dev/null +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.NullifyJoinRule; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; + +/** + * A runner class for manual testing. + */ +public class Runner { + public static void main(String[] args) throws Exception { + // Builds the schema. + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + rootSchema.add("c", new ReflectiveSchema(new Company())); + + // Creates the planner. + final Program programs = Programs.ofRules( + NullifyJoinRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(rootSchema) + .programs(programs) + .build(); + final Planner planner = Frameworks.getPlanner(config); + + // Parses, validates and builds the query. + String sqlQuery = "select e.\"name\", d.\"depName\" from " + + " \"c\".\"employees\" e left join \"c\".\"departments\" d " + + " on e.\"depID\" = d.\"depID\" "; + SqlNode parse = planner.parse(sqlQuery); + SqlNode validate = planner.validate(parse); + RelNode relNode = planner.rel(validate).rel; + System.out.println(RelOptUtil.toString(relNode)); + + // Transforms the query. + RelTraitSet traitSet = relNode.getTraitSet().replace(EnumerableConvention.INSTANCE); + RelNode transformedNode = planner.transform(0, traitSet, relNode); + System.out.println(RelOptUtil.toString(transformedNode)); + } + + /** + * Represents the database named company. */ + public static class Company { + public final Employee[] employees = { + new Employee(10, 1, "Daniel"), + new Employee(20, 1, "Mark"), + new Employee(30, 2, "Smith"), + new Employee(40, 3, "Armstrong") + }; + + public final Department[] departments = { + new Department(1, "Engineering"), + new Department(2, "Finance") + }; + } + + /** + * Represents the schema of the employee table. */ + public static class Employee { + public final int empID; + public final int depID; + public final String name; + + Employee(int empID, int depID, String name) { + this.empID = empID; + this.depID = depID; + this.name = name; + } + } + + /** + * Represents the schema of the department table. */ + public static class Department { + public final int depID; + public final String depName; + + Department(int depID, String depName) { + this.depID = depID; + this.depName = depName; + } + } + + private Runner() { + } +} + +// End Runner.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java new file mode 100644 index 000000000000..dc12a512540c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelInput; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rex.RexNode; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Nullify is an unary operation that performs nullification to the given attribute + * list of the input relation based on the given predicate. + */ +public abstract class Nullify extends SingleRel { + //~ Instance fields -------------------------------------------------------- + + protected final RexNode predicate; + protected final ImmutableList attributes; + + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a nullification operator. + * + * @param cluster Cluster that this relational expression belongs to + * @param traits the traits of this rel + * @param child input relational expression + */ + protected Nullify( + RelOptCluster cluster, + RelTraitSet traits, + RelNode child, + RexNode predicate, + List attributes) { + super(cluster, traits, child); + this.predicate = predicate; + this.attributes = ImmutableList.copyOf(attributes); + } + + /** + * Creates a Nullify by parsing serialized output. + */ + protected Nullify(RelInput input) { + this(input.getCluster(), + input.getTraitSet(), + input.getInput(), + input.getExpression("condition"), + input.getExpressionList("exprs")); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public final RelNode copy(RelTraitSet traitSet, + List inputs) { + return copy(traitSet, sole(inputs), predicate, attributes); + } + + public abstract Nullify copy(RelTraitSet traitSet, RelNode input, + RexNode predicate, List attributes); + + @Override public List getChildExps() { + return ImmutableList.of(predicate); + } + + public RelWriter explainTerms(final RelWriter pw) { + return super.explainTerms(pw) + .item("predicate", predicate) + .item("attributes", attributes); + } +} + +// End Nullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index a02e187eb5b4..6ea340e56dac 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -33,6 +33,7 @@ import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.logical.LogicalMatch; import org.apache.calcite.rel.logical.LogicalMinus; +import org.apache.calcite.rel.logical.LogicalNullify; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalRepeatUnion; import org.apache.calcite.rel.logical.LogicalSnapshot; @@ -74,6 +75,9 @@ public class RelFactories { public static final FilterFactory DEFAULT_FILTER_FACTORY = new FilterFactoryImpl(); + public static final NullifyFactory DEFAULT_NULLIFY_FACTORY = + new NullifyFactoryImpl(); + public static final JoinFactory DEFAULT_JOIN_FACTORY = new JoinFactoryImpl(); public static final CorrelateFactory DEFAULT_CORRELATE_FACTORY = @@ -121,6 +125,7 @@ public class RelFactories { RelBuilder.proto( Contexts.of(DEFAULT_PROJECT_FACTORY, DEFAULT_FILTER_FACTORY, + DEFAULT_NULLIFY_FACTORY, DEFAULT_JOIN_FACTORY, DEFAULT_SORT_FACTORY, DEFAULT_EXCHANGE_FACTORY, @@ -335,6 +340,33 @@ public RelNode createFilter(RelNode input, RexNode condition, } } + /** + * Can create a {@link Nullify} of the appropriate type + * for this rule's calling convention. + */ + public interface NullifyFactory { + /** + * Creates a nullification operator. + */ + RelNode createNullify(RelNode input, RexNode predicate, + List attributes, + List fieldNames, + Set variablesSet); + } + + /** + * Implementation of {@link RelFactories.NullifyFactory} that + * returns a vanilla {@link Nullify}. + */ + private static class NullifyFactoryImpl implements NullifyFactory { + public RelNode createNullify(RelNode input, RexNode predicate, + List attributes, + List fieldNames, + Set variablesSet) { + return LogicalNullify.create(input, predicate, attributes, ImmutableSet.copyOf(variablesSet)); + } + } + /** * Can create a join of the appropriate type for a rule's calling convention. * diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java new file mode 100644 index 000000000000..8d1eb91d0841 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistributionTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.metadata.RelMdCollation; +import org.apache.calcite.rel.metadata.RelMdDistribution; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Objects; + +/** + * Sub-class of {@link org.apache.calcite.rel.core.Nullify} + * not targeted at any particular engine or calling convention. + */ +public class LogicalNullify extends Nullify { + private final ImmutableSet variablesSet; + + //~ Constructors ----------------------------------------------------------- + + protected LogicalNullify( + RelOptCluster cluster, + RelTraitSet traits, + RelNode child, + RexNode predicate, + List attributes, + ImmutableSet variablesSet) { + super(cluster, traits, child, predicate, attributes); + this.variablesSet = Objects.requireNonNull(variablesSet); + } + + /** Creates a LogicalNullify. */ + public static LogicalNullify create(final RelNode input, RexNode condition, + List attributes, + ImmutableSet variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.filter(mq, input)) + .replaceIf(RelDistributionTraitDef.INSTANCE, + () -> RelMdDistribution.filter(mq, input)); + return new LogicalNullify(cluster, traitSet, input, condition, attributes, variablesSet); + } + + @Override public Nullify copy(RelTraitSet traitSet, RelNode input, + RexNode predicate, List attributes) { + return new LogicalNullify(getCluster(), traitSet, input, predicate, attributes, variablesSet); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + + @Override public RelWriter explainTerms(final RelWriter pw) { + return super.explainTerms(pw) + .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); + } +} + +// End LogicalNullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java new file mode 100644 index 000000000000..68a673494031 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * NullifyJoinRule + */ +public class NullifyJoinRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the rule that nullifies inner, left outer or right outer join. */ + public static final NullifyJoinRule INSTANCE = + new NullifyJoinRule(operand(LogicalJoin.class, any()), null); + + //~ Constructors ----------------------------------------------------------- + + public NullifyJoinRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyJoinRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + RelBuilder builder = call.builder(); + + // The join operator at the current node. + Join join = call.rel(0); + + // Determines the nullification attribute list based on the join type. + List nullificationList = new ArrayList<>(); + List leftFieldList = join.getLeft().getRowType().getFieldList(); + List rightFieldList = join.getRight().getRowType().getFieldList(); + List leftList = leftFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + List rightList = rightFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + switch (join.getJoinType()) { + case LEFT: + nullificationList.addAll(rightList); + break; + case RIGHT: + nullificationList.addAll(leftList); + break; + case INNER: + nullificationList.addAll(leftList); + nullificationList.addAll(rightList); + break; + default: + throw new AssertionError(join.getJoinType()); + } + + // Determines the nullification condition. + RexNode nullificationCondition = join.getCondition(); + + // Builds the transformed relational tree. + final RelNode cartesianJoin = + join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + join.getLeft(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + builder.push(cartesianJoin).nullify(nullificationCondition, nullificationList); + call.transformTo(builder.build()); + } +} + +// End NullifyJoinRule.java diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index a86a96c47739..1c1f0b48cadb 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -141,6 +141,7 @@ public class RelBuilder { protected final RelOptCluster cluster; protected final RelOptSchema relOptSchema; private final RelFactories.FilterFactory filterFactory; + private final RelFactories.NullifyFactory nullifyFactory; private final RelFactories.ProjectFactory projectFactory; private final RelFactories.AggregateFactory aggregateFactory; private final RelFactories.SortFactory sortFactory; @@ -174,6 +175,9 @@ protected RelBuilder(Context context, RelOptCluster cluster, this.filterFactory = Util.first(context.unwrap(RelFactories.FilterFactory.class), RelFactories.DEFAULT_FILTER_FACTORY); + this.nullifyFactory = + Util.first(context.unwrap(RelFactories.NullifyFactory.class), + RelFactories.DEFAULT_NULLIFY_FACTORY); this.projectFactory = Util.first(context.unwrap(RelFactories.ProjectFactory.class), RelFactories.DEFAULT_PROJECT_FACTORY); @@ -1224,6 +1228,46 @@ public RelBuilder filter(Iterable variablesSet, return this; } + /** Creates a gamma of the given list of + * fields. */ + public RelBuilder gamma(RexNode... nodes) { + return gamma(ImmutableList.copyOf(nodes)); + } + + public RelBuilder gamma(Iterable nodes) { + final List nodeList = Lists.newArrayList(nodes); + final List checkNullList = nodeList + .stream() + .map(node -> call(SqlStdOperatorTable.IS_NULL, node)) + .collect(Collectors.toList()); + return filter(and(checkNullList)); + } + + public RelBuilder nullify(RexNode predicate, RexNode... nodes) { + return nullify(predicate, ImmutableList.copyOf(nodes)); + } + + public RelBuilder nullify(RexNode predicate, Iterable nodes) { + return nullify(predicate, nodes, ImmutableList.of()); + } + + public RelBuilder nullify(RexNode predicate, Iterable nodes, + Iterable fieldNames) { + return nullify(ImmutableSet.of(), predicate, nodes, fieldNames); + } + + public RelBuilder nullify(Set variablesSet, RexNode predicate, + Iterable nodes, Iterable fieldNames) { + final Frame frame = stack.pop(); + final List nodesList = Lists.newArrayList(nodes); + final List fieldNamesList = Lists.newArrayList(fieldNames); + + final RelNode nullify = nullifyFactory.createNullify(frame.rel, predicate, + nodesList, fieldNamesList, variablesSet); + stack.push(new Frame(nullify, frame.fields)); + return this; + } + /** Creates a {@link Project} of the given * expressions. */ public RelBuilder project(RexNode... nodes) { From 0c384e6a560c400cc1e86b0696eac407ca3da037 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sat, 14 Sep 2019 16:32:07 +0800 Subject: [PATCH 03/24] Add best match operator --- .../apache/calcite/rel/core/BestMatch.java | 64 +++++++++++++++ .../apache/calcite/rel/core/RelFactories.java | 26 +++++++ .../calcite/rel/logical/LogicalBestMatch.java | 78 +++++++++++++++++++ .../calcite/rel/rules/NullifyJoinRule.java | 2 +- .../org/apache/calcite/tools/RelBuilder.java | 15 ++++ 5 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/core/BestMatch.java create mode 100644 core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java b/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java new file mode 100644 index 000000000000..81a07d8446b0 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelInput; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.SingleRel; + +import java.util.List; + +/** + * BestMatch is a unary operation that is always at the top of a join tree + * to eliminate spurious tuples. + */ +public abstract class BestMatch extends SingleRel { + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a best-match operator. + * + * @param cluster Cluster this relational expression belongs to + * @param traits the traits of this rel + * @param input Input relational expression + */ + protected BestMatch( + RelOptCluster cluster, + RelTraitSet traits, + RelNode input) { + super(cluster, traits, input); + } + + /** + * Creates a best match by its input. + */ + protected BestMatch(RelInput input) { + this(input.getCluster(), input.getTraitSet(), input.getInput()); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public final RelNode copy(RelTraitSet traitSet, List inputs) { + return copy(traitSet, sole(inputs)); + } + + public abstract BestMatch copy(RelTraitSet traitSet, RelNode input); +} + +// End BestMatch.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index 6ea340e56dac..65087b72f533 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -26,6 +26,7 @@ import org.apache.calcite.rel.RelDistribution; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalBestMatch; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalExchange; import org.apache.calcite.rel.logical.LogicalFilter; @@ -78,6 +79,9 @@ public class RelFactories { public static final NullifyFactory DEFAULT_NULLIFY_FACTORY = new NullifyFactoryImpl(); + public static final BestMatchFactory DEFAULT_BEST_MATCH_FACTORY = + new BestMatchFactoryImpl(); + public static final JoinFactory DEFAULT_JOIN_FACTORY = new JoinFactoryImpl(); public static final CorrelateFactory DEFAULT_CORRELATE_FACTORY = @@ -126,6 +130,7 @@ public class RelFactories { Contexts.of(DEFAULT_PROJECT_FACTORY, DEFAULT_FILTER_FACTORY, DEFAULT_NULLIFY_FACTORY, + DEFAULT_BEST_MATCH_FACTORY, DEFAULT_JOIN_FACTORY, DEFAULT_SORT_FACTORY, DEFAULT_EXCHANGE_FACTORY, @@ -367,6 +372,27 @@ public RelNode createNullify(RelNode input, RexNode predicate, } } + /** + * Can create a {@link BestMatch} of the appropriate type + * for this rule's calling convention. + */ + public interface BestMatchFactory { + /** + * Creates a best-match operator. + */ + RelNode createBestMatch(RelNode input, Set variablesSet); + } + + /** + * Implementation of {@link RelFactories.BestMatchFactory} that + * returns a vanilla {@link BestMatch}. + */ + private static class BestMatchFactoryImpl implements BestMatchFactory { + public RelNode createBestMatch(final RelNode input, final Set variablesSet) { + return LogicalBestMatch.create(input, ImmutableSet.copyOf(variablesSet)); + } + } + /** * Can create a join of the appropriate type for a rule's calling convention. * diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java new file mode 100644 index 000000000000..9a5945715f3c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistributionTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.metadata.RelMdCollation; +import org.apache.calcite.rel.metadata.RelMdDistribution; +import org.apache.calcite.rel.metadata.RelMetadataQuery; + +import com.google.common.collect.ImmutableSet; + +import java.util.Objects; + +/** + * Sub-class of {@link org.apache.calcite.rel.core.BestMatch} + * not targeted at any particular engine or calling convention. + */ +public class LogicalBestMatch extends BestMatch { + private final ImmutableSet variablesSet; + + //~ Constructors ----------------------------------------------------------- + + protected LogicalBestMatch( + RelOptCluster cluster, + RelTraitSet traits, + RelNode input, + ImmutableSet variablesSet) { + super(cluster, traits, input); + this.variablesSet = Objects.requireNonNull(variablesSet); + } + + /** Creates a LogicalBestMatch. */ + public static LogicalBestMatch create(final RelNode input, + ImmutableSet variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.filter(mq, input)) + .replaceIf(RelDistributionTraitDef.INSTANCE, + () -> RelMdDistribution.filter(mq, input)); + return new LogicalBestMatch(cluster, traitSet, input, variablesSet); + } + + @Override public BestMatch copy(final RelTraitSet traitSet, final RelNode input) { + return new LogicalBestMatch(getCluster(), traitSet, input, variablesSet); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } +} + +// End LogicalBestMatch.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java index 68a673494031..f3b2102fa22d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java @@ -100,7 +100,7 @@ public NullifyJoinRule(RelOptRuleOperand operand, String description) { join.getRight(), join.getJoinType(), join.isSemiJoinDone()); - builder.push(cartesianJoin).nullify(nullificationCondition, nullificationList); + builder.push(cartesianJoin).nullify(nullificationCondition, nullificationList).bestMatch(); call.transformTo(builder.build()); } } diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 1c1f0b48cadb..c6cdc02c7e83 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -142,6 +142,7 @@ public class RelBuilder { protected final RelOptSchema relOptSchema; private final RelFactories.FilterFactory filterFactory; private final RelFactories.NullifyFactory nullifyFactory; + private final RelFactories.BestMatchFactory bestMatchFactory; private final RelFactories.ProjectFactory projectFactory; private final RelFactories.AggregateFactory aggregateFactory; private final RelFactories.SortFactory sortFactory; @@ -178,6 +179,9 @@ protected RelBuilder(Context context, RelOptCluster cluster, this.nullifyFactory = Util.first(context.unwrap(RelFactories.NullifyFactory.class), RelFactories.DEFAULT_NULLIFY_FACTORY); + this.bestMatchFactory = + Util.first(context.unwrap(RelFactories.BestMatchFactory.class), + RelFactories.DEFAULT_BEST_MATCH_FACTORY); this.projectFactory = Util.first(context.unwrap(RelFactories.ProjectFactory.class), RelFactories.DEFAULT_PROJECT_FACTORY); @@ -1268,6 +1272,17 @@ public RelBuilder nullify(Set variablesSet, RexNode predicate, return this; } + public RelBuilder bestMatch() { + return bestMatch(ImmutableSet.of()); + } + + public RelBuilder bestMatch(Set variablesSet) { + final Frame frame = stack.pop(); + final RelNode bestMatch = bestMatchFactory.createBestMatch(frame.rel, variablesSet); + stack.push(new Frame(bestMatch, frame.fields)); + return this; + } + /** Creates a {@link Project} of the given * expressions. */ public RelBuilder project(RexNode... nodes) { From b81176ea9db97d6259032285bf4fa26e308df879 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sat, 14 Sep 2019 19:26:11 +0800 Subject: [PATCH 04/24] Add best match reduce rule --- .../main/java/org/apache/calcite/Runner.java | 4 +- .../rel/rules/custom/BestMatchReduceRule.java | 61 +++++++++++++++++++ .../rules/{ => custom}/NullifyJoinRule.java | 5 +- .../rel/rules/custom/package-info.java | 24 ++++++++ 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java rename core/src/main/java/org/apache/calcite/rel/rules/{ => custom}/NullifyJoinRule.java (96%) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index b40d569808eb..1625f6949f5e 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -22,7 +22,8 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.rules.NullifyJoinRule; +import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; +import org.apache.calcite.rel.rules.custom.NullifyJoinRule; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; @@ -44,6 +45,7 @@ public static void main(String[] args) throws Exception { // Creates the planner. final Program programs = Programs.ofRules( NullifyJoinRule.INSTANCE, + BestMatchReduceRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); final FrameworkConfig config = Frameworks.newConfigBuilder() diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java new file mode 100644 index 000000000000..5ad7775bc5a6 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.tools.RelBuilderFactory; + +/** + * BestMatchReduceRule is able to reduce two consecutive best-match operators + * into one. In the current implementation, the outer one will be eliminated. + */ +public class BestMatchReduceRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the rule that reduces two best-match operators. */ + public static final BestMatchReduceRule INSTANCE = new BestMatchReduceRule( + operand(BestMatch.class, operand(BestMatch.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchReduceRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchReduceRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the inner best-match operator. + BestMatch innerBestMatch = call.rel(1); + + // Eliminates the outer one. + RelNode reducedNode = call.builder().push(innerBestMatch).build(); + call.transformTo(reducedNode); + } +} + +// End BestMatchReduceRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java similarity index 96% rename from core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java rename to core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java index f3b2102fa22d..3e1a4cd2c4c1 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/NullifyJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.calcite.rel.rules; +package org.apache.calcite.rel.rules.custom; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; @@ -22,7 +22,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; @@ -41,7 +40,7 @@ public class NullifyJoinRule extends RelOptRule { /** Instance of the rule that nullifies inner, left outer or right outer join. */ public static final NullifyJoinRule INSTANCE = - new NullifyJoinRule(operand(LogicalJoin.class, any()), null); + new NullifyJoinRule(operand(Join.class, any()), null); //~ Constructors ----------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java new file mode 100644 index 000000000000..28eedf25575e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides a customized set of planner rules, primarily used for outer join + * reorder-ability research. + */ +package org.apache.calcite.rel.rules.custom; + +// End package-info.java From 5c7d5d88be99be04f9ffc4609815dcab5293c732 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 15 Sep 2019 17:48:34 +0800 Subject: [PATCH 05/24] Add more rules --- .../main/java/org/apache/calcite/Runner.java | 6 +- .../org/apache/calcite/rel/core/Nullify.java | 8 ++ .../custom/BestMatchOverNullifyRule.java | 109 ++++++++++++++++++ .../rel/rules/custom/BestMatchPullUpRule.java | 84 ++++++++++++++ .../rel/rules/custom/NullifyPullUpRule.java | 88 ++++++++++++++ 5 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 1625f6949f5e..b9a472bfa6e6 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -40,7 +40,7 @@ public class Runner { public static void main(String[] args) throws Exception { // Builds the schema. final SchemaPlus rootSchema = Frameworks.createRootSchema(true); - rootSchema.add("c", new ReflectiveSchema(new Company())); + rootSchema.add("p", new ReflectiveSchema(new People())); // Creates the planner. final Program programs = Programs.ofRules( @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { // Parses, validates and builds the query. String sqlQuery = "select e.\"name\", d.\"depName\" from " - + " \"c\".\"employees\" e left join \"c\".\"departments\" d " + + " \"p\".\"employees\" e left join \"p\".\"departments\" d " + " on e.\"depID\" = d.\"depID\" "; SqlNode parse = planner.parse(sqlQuery); SqlNode validate = planner.validate(parse); @@ -72,7 +72,7 @@ public static void main(String[] args) throws Exception { /** * Represents the database named company. */ - public static class Company { + public static class People { public final Employee[] employees = { new Employee(10, 1, "Daniel"), new Employee(20, 1, "Mark"), diff --git a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java index dc12a512540c..124fc81ef48f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java @@ -88,6 +88,14 @@ public RelWriter explainTerms(final RelWriter pw) { .item("predicate", predicate) .item("attributes", attributes); } + + public RexNode getPredicate() { + return predicate; + } + + public ImmutableList getAttributes() { + return ImmutableList.copyOf(attributes); + } } // End Nullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java new file mode 100644 index 000000000000..e191fac7bfac --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; + +import java.util.List; + +/** + * BestMatchOverNullifyRule eliminates all inner best-match operators (sandwiched + * by a nullification operator) as long as there is a best-match operator at the + * very end. It will check whether the nullification predicate is null-intolerant. + * The conversion is from `B(Nullify(B(R)))` to `B(Nullify(R))`. + */ +public class BestMatchOverNullifyRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the current rule. */ + public static final BestMatchOverNullifyRule INSTANCE = new BestMatchOverNullifyRule( + operand(BestMatch.class, + operand(Nullify.class, + operand(BestMatch.class, any()))), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchOverNullifyRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + RelBuilder builder = call.builder(); + + // Gets the old nullification operator. + Nullify nullify = call.rel(1); + RexNode oldPredicate = nullify.getPredicate(); + List oldAttributes = nullify.getAttributes(); + + // Makes sure the nullification predicate is null-intolerant. + if (isNullTolerant(oldPredicate)) { + throw new AssertionError("The nullification predicate is not null-intolerant."); + } + + // Gets the base relation. + BestMatch innerBestMatch = call.rel(2); + RelNode base = innerBestMatch.getInput(); + + // Constructs the new expression. + RelNode newNode = builder.push(base).nullify(oldPredicate, oldAttributes).bestMatch().build(); + call.transformTo(newNode); + } + + /** + * Checks whether a given predicate tolerates NULL values. A predicate is + * null-intolerant if it cannot evaluate to TRUE when referring a NULL + * value. + * + * @param predicate is the predicate to be tested. + * @return true if null tolerant; false otherwise. + */ + private boolean isNullTolerant(RexNode predicate) { + // An OR connective is null-tolerant if any of its child expressions is null-tolerant. + if (predicate.isA(SqlKind.OR)) { + RexCall call = (RexCall) predicate; + for (RexNode operand: call.getOperands()) { + if (isNullTolerant(operand)) { + return true; + } + } + return false; + } + + // IS NULL and TRUE are both null-tolerant. + return predicate.isA(SqlKind.IS_NULL) || predicate.isA(SqlKind.IS_TRUE); + } +} + +// End BestMatchOverNullifyRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java new file mode 100644 index 000000000000..176455705c68 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; + +/** + * BestMatchPullUpRule pulls up a best-match operator. Basically, the + * conversion is from `B(R) * S` to `B(R * S)`. + */ +public class BestMatchPullUpRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the current rule. */ + public static final BestMatchPullUpRule INSTANCE = new BestMatchPullUpRule( + operand(Join.class, + operand(BestMatch.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchPullUpRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchPullUpRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + final RelBuilder builder = call.builder(); + + // Only applies the rule on cartesian product. + final Join join = call.rel(0); + final RexNode condition = join.getCondition(); + if (!condition.equals(builder.literal(true))) { + throw new AssertionError(condition); + } + + // Constructs the new cartesian product. + final BestMatch bestMatch = call.rel(1); + final RelNode cartesianJoin = + join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + bestMatch.getInput(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + + // Builds the new expression. + RelNode reducedNode = builder.push(cartesianJoin).bestMatch().build(); + call.transformTo(reducedNode); + } +} + +// End BestMatchPullUpRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java new file mode 100644 index 000000000000..9d4e6d1afa3b --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; + +import java.util.List; + +/** + * NullifyPullUpRule pulls up a nullification operator. Basically, the + * conversion is from `Nullify(R) * S` to `Nullify(R * S)`. + */ +public class NullifyPullUpRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the current rule. */ + public static final NullifyPullUpRule INSTANCE = new NullifyPullUpRule( + operand(Join.class, + operand(Nullify.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public NullifyPullUpRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyPullUpRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + final RelBuilder builder = call.builder(); + + // Only applies the rule on cartesian product. + final Join join = call.rel(0); + final RexNode condition = join.getCondition(); + if (!condition.equals(builder.literal(true))) { + throw new AssertionError(condition); + } + + // Constructs the new cartesian product. + final Nullify oldNullify = call.rel(1); + final RelNode cartesianJoin = + join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + oldNullify.getInput(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + + // Builds the new expression. + final RexNode oldPredicate = oldNullify.getPredicate(); + final List oldAttributes = oldNullify.getAttributes(); + RelNode reducedNode = builder.push(cartesianJoin).nullify(oldPredicate, oldAttributes).build(); + call.transformTo(reducedNode); + } +} + +// End NullifyPullUpRule.java From 2650949e65c283af3e38815c8c1f34c5136fd072 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Thu, 19 Sep 2019 16:43:38 +0800 Subject: [PATCH 06/24] Fix null-tolerant checking --- .../rel/rules/custom/BestMatchOverNullifyRule.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java index e191fac7bfac..dfc960c8592a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java @@ -90,15 +90,26 @@ public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { * @return true if null tolerant; false otherwise. */ private boolean isNullTolerant(RexNode predicate) { - // An OR connective is null-tolerant if any of its child expressions is null-tolerant. if (predicate.isA(SqlKind.OR)) { RexCall call = (RexCall) predicate; + + // An OR connective is null-tolerant if any of its child expressions is null-tolerant. for (RexNode operand: call.getOperands()) { if (isNullTolerant(operand)) { return true; } } return false; + } else if (predicate.isA(SqlKind.AND)) { + RexCall call = (RexCall) predicate; + + // An AND connective is null-tolerant if all of its child expressions are null-tolerant. + for (RexNode operand: call.getOperands()) { + if (!isNullTolerant(operand)) { + return false; + } + } + return true; } // IS NULL and TRUE are both null-tolerant. From 07538efc4a304a5a959f6d58db2d6a5cf735c24f Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Thu, 19 Sep 2019 23:11:09 +0800 Subject: [PATCH 07/24] Add queries with 2 joins --- .../main/java/org/apache/calcite/Runner.java | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index b9a472bfa6e6..898e3f89030d 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; import org.apache.calcite.schema.SchemaPlus; @@ -32,6 +33,7 @@ import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RelBuilder; /** * A runner class for manual testing. @@ -40,7 +42,7 @@ public class Runner { public static void main(String[] args) throws Exception { // Builds the schema. final SchemaPlus rootSchema = Frameworks.createRootSchema(true); - rootSchema.add("p", new ReflectiveSchema(new People())); + final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); // Creates the planner. final Program programs = Programs.ofRules( @@ -50,10 +52,11 @@ public static void main(String[] args) throws Exception { EnumerableRules.ENUMERABLE_JOIN_RULE); final FrameworkConfig config = Frameworks.newConfigBuilder() .parserConfig(SqlParser.Config.DEFAULT) - .defaultSchema(rootSchema) + .defaultSchema(defaultSchema) .programs(programs) .build(); final Planner planner = Frameworks.getPlanner(config); + final RelBuilder builder = RelBuilder.create(config); // Parses, validates and builds the query. String sqlQuery = "select e.\"name\", d.\"depName\" from " @@ -68,6 +71,26 @@ public static void main(String[] args) throws Exception { RelTraitSet traitSet = relNode.getTraitSet().replace(EnumerableConvention.INSTANCE); RelNode transformedNode = planner.transform(0, traitSet, relNode); System.out.println(RelOptUtil.toString(transformedNode)); + + // Alternatively, build the relational nodes directly. + final RelBuilder firstScan = builder.scan("employees").as("e").scan("departments").as("d"); + final RelBuilder firstJoin = firstScan.join(JoinRelType.LEFT, + firstScan.equals( + firstScan.field(2, 0, "depID"), + firstScan.field(2, 1, "depID") + )); + final RelBuilder secondScan = firstJoin.scan("companies").as("c"); + final RelBuilder secondJoin = secondScan.join(JoinRelType.INNER, + secondScan.equals( + secondScan.field(2, 0, "cmpID"), + secondScan.field(2, 1, "cmpID") + )); + final RelBuilder afterProject = secondJoin.project( + secondJoin.field("e", "name"), + secondJoin.field("d", "depName"), + secondJoin.field("c", "cmpName")); + final RelNode finalNode = afterProject.build(); + System.out.println(RelOptUtil.toString(finalNode)); } /** @@ -81,8 +104,13 @@ public static class People { }; public final Department[] departments = { - new Department(1, "Engineering"), - new Department(2, "Finance") + new Department(1, "Engineering", 100), + new Department(2, "Finance", 100) + }; + + public final Company[] companies = { + new Company(100, "All Link Pte Ltd"), + new Company(200, "") }; } @@ -105,10 +133,24 @@ public static class Employee { public static class Department { public final int depID; public final String depName; + public final int cmpID; - Department(int depID, String depName) { + Department(int depID, String depName, int cmpID) { this.depID = depID; this.depName = depName; + this.cmpID = cmpID; + } + } + + /** + * Represents the schema of the company table. */ + public static class Company { + public final int cmpID; + public final String cmpName; + + Company(int cmpID, String cmpName) { + this.cmpID = cmpID; + this.cmpName = cmpName; } } From 505b82b6ac3e205105411f28a3d6727c88532496 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Thu, 19 Sep 2019 23:58:27 +0800 Subject: [PATCH 08/24] Refactor for Runner class --- .../main/java/org/apache/calcite/Runner.java | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 898e3f89030d..5151dd1486b6 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -19,10 +19,10 @@ import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.config.Lex; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; import org.apache.calcite.schema.SchemaPlus; @@ -33,7 +33,6 @@ import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; import org.apache.calcite.tools.Programs; -import org.apache.calcite.tools.RelBuilder; /** * A runner class for manual testing. @@ -45,23 +44,41 @@ public static void main(String[] args) throws Exception { final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); // Creates the planner. + final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); final Program programs = Programs.ofRules( NullifyJoinRule.INSTANCE, BestMatchReduceRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); final FrameworkConfig config = Frameworks.newConfigBuilder() - .parserConfig(SqlParser.Config.DEFAULT) + .parserConfig(parserConfig) .defaultSchema(defaultSchema) .programs(programs) .build(); final Planner planner = Frameworks.getPlanner(config); - final RelBuilder builder = RelBuilder.create(config); + // A single left outer join. + String sqlQuery = "select e.name, d.depName " + + "from p.employees e left join p.departments d on e.depID = d.depID"; + buildAndTransformQuery(planner, sqlQuery); + + // Two joins (left outer join + inner join). + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e left join p.departments d on e.depID = d.depID " + + "join p.companies c on d.cmpID = c.cmpID"; + buildAndTransformQuery(planner, sqlQuery); + } + + /** + * This method emulates the whole life cycle of a given SQL query: parse, validate build and + * transform. It will close & reset the planner after usage. + * + * @param planner is the planner to be used during the life cycle. + * @param sqlQuery is the original SQL query in its string representation. + * @throws Exception when there is error during any step. + */ + private static void buildAndTransformQuery(Planner planner, String sqlQuery) throws Exception { // Parses, validates and builds the query. - String sqlQuery = "select e.\"name\", d.\"depName\" from " - + " \"p\".\"employees\" e left join \"p\".\"departments\" d " - + " on e.\"depID\" = d.\"depID\" "; SqlNode parse = planner.parse(sqlQuery); SqlNode validate = planner.validate(parse); RelNode relNode = planner.rel(validate).rel; @@ -72,25 +89,9 @@ public static void main(String[] args) throws Exception { RelNode transformedNode = planner.transform(0, traitSet, relNode); System.out.println(RelOptUtil.toString(transformedNode)); - // Alternatively, build the relational nodes directly. - final RelBuilder firstScan = builder.scan("employees").as("e").scan("departments").as("d"); - final RelBuilder firstJoin = firstScan.join(JoinRelType.LEFT, - firstScan.equals( - firstScan.field(2, 0, "depID"), - firstScan.field(2, 1, "depID") - )); - final RelBuilder secondScan = firstJoin.scan("companies").as("c"); - final RelBuilder secondJoin = secondScan.join(JoinRelType.INNER, - secondScan.equals( - secondScan.field(2, 0, "cmpID"), - secondScan.field(2, 1, "cmpID") - )); - final RelBuilder afterProject = secondJoin.project( - secondJoin.field("e", "name"), - secondJoin.field("d", "depName"), - secondJoin.field("c", "cmpName")); - final RelNode finalNode = afterProject.build(); - System.out.println(RelOptUtil.toString(finalNode)); + // Closes and resets the planner. + planner.close(); + planner.reset(); } /** From 8200c954f6caad3f0d2bc53e9e120e4d1c7950c4 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Fri, 20 Sep 2019 00:10:21 +0800 Subject: [PATCH 09/24] Add print statements --- core/src/main/java/org/apache/calcite/Runner.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 5151dd1486b6..b0682e9257b8 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -78,20 +78,26 @@ public static void main(String[] args) throws Exception { * @throws Exception when there is error during any step. */ private static void buildAndTransformQuery(Planner planner, String sqlQuery) throws Exception { + System.out.println("============================ Start ============================"); + // Parses, validates and builds the query. SqlNode parse = planner.parse(sqlQuery); SqlNode validate = planner.validate(parse); RelNode relNode = planner.rel(validate).rel; + System.out.println("Before transformation:\n"); System.out.println(RelOptUtil.toString(relNode)); // Transforms the query. RelTraitSet traitSet = relNode.getTraitSet().replace(EnumerableConvention.INSTANCE); RelNode transformedNode = planner.transform(0, traitSet, relNode); + System.out.println("After transformation:\n"); System.out.println(RelOptUtil.toString(transformedNode)); // Closes and resets the planner. planner.close(); planner.reset(); + + System.out.println("============================= End =============================\n"); } /** From f25479c21834dc874feda2c9438c335e5fc1afd5 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Fri, 20 Sep 2019 00:22:16 +0800 Subject: [PATCH 10/24] Close planner in the end --- core/src/main/java/org/apache/calcite/Runner.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index b0682e9257b8..6b357d7f2e52 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -67,6 +67,9 @@ public static void main(String[] args) throws Exception { + "from p.employees e left join p.departments d on e.depID = d.depID " + "join p.companies c on d.cmpID = c.cmpID"; buildAndTransformQuery(planner, sqlQuery); + + // Closes the planner eventually. + planner.close(); } /** From 09c4843745078b37d68e3141f28a46289b1356a5 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 22 Sep 2019 11:25:38 +0800 Subject: [PATCH 11/24] Apply all rules and fix doc error --- core/src/main/java/org/apache/calcite/Runner.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 6b357d7f2e52..4a0356e93342 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -23,8 +23,11 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.custom.BestMatchOverNullifyRule; +import org.apache.calcite.rel.rules.custom.BestMatchPullUpRule; import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; +import org.apache.calcite.rel.rules.custom.NullifyPullUpRule; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; @@ -46,8 +49,11 @@ public static void main(String[] args) throws Exception { // Creates the planner. final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); final Program programs = Programs.ofRules( - NullifyJoinRule.INSTANCE, + BestMatchOverNullifyRule.INSTANCE, + BestMatchPullUpRule.INSTANCE, BestMatchReduceRule.INSTANCE, + NullifyJoinRule.INSTANCE, + NullifyPullUpRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); final FrameworkConfig config = Frameworks.newConfigBuilder() @@ -74,7 +80,7 @@ public static void main(String[] args) throws Exception { /** * This method emulates the whole life cycle of a given SQL query: parse, validate build and - * transform. It will close & reset the planner after usage. + * transform. It will close and reset the planner after usage. * * @param planner is the planner to be used during the life cycle. * @param sqlQuery is the original SQL query in its string representation. From b0a45a0e9062d25cf4bfb0a76e774ec120752ed7 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 22 Sep 2019 23:42:15 +0800 Subject: [PATCH 12/24] Convert nullified join to be outer cartesian product --- .../main/java/org/apache/calcite/Runner.java | 12 ++-- .../apache/calcite/rel/core/JoinRelType.java | 21 +++++- .../org/apache/calcite/rel/core/Nullify.java | 7 ++ .../calcite/rel/rules/JoinAssociateRule.java | 24 +++++-- .../calcite/rel/rules/JoinCommuteRule.java | 3 +- .../custom/BestMatchOverNullifyRule.java | 4 +- .../rel/rules/custom/BestMatchPullUpRule.java | 11 ++- .../rel/rules/custom/NullifyJoinRule.java | 72 +++++++++++-------- .../rel/rules/custom/NullifyPullUpRule.java | 12 +++- .../sql/validate/SqlValidatorUtil.java | 30 ++++++++ 10 files changed, 146 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 4a0356e93342..0b10d351d1a0 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -23,6 +23,8 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.JoinAssociateRule; +import org.apache.calcite.rel.rules.JoinCommuteRule; import org.apache.calcite.rel.rules.custom.BestMatchOverNullifyRule; import org.apache.calcite.rel.rules.custom.BestMatchPullUpRule; import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; @@ -49,11 +51,13 @@ public static void main(String[] args) throws Exception { // Creates the planner. final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); final Program programs = Programs.ofRules( - BestMatchOverNullifyRule.INSTANCE, - BestMatchPullUpRule.INSTANCE, - BestMatchReduceRule.INSTANCE, NullifyJoinRule.INSTANCE, NullifyPullUpRule.INSTANCE, + BestMatchReduceRule.INSTANCE, + BestMatchPullUpRule.INSTANCE, + BestMatchOverNullifyRule.INSTANCE, + JoinCommuteRule.INSTANCE, + JoinAssociateRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); final FrameworkConfig config = Frameworks.newConfigBuilder() @@ -71,7 +75,7 @@ public static void main(String[] args) throws Exception { // Two joins (left outer join + inner join). sqlQuery = "select e.name, d.depName, c.cmpName " + "from p.employees e left join p.departments d on e.depID = d.depID " - + "join p.companies c on d.cmpID = c.cmpID"; + + "left join p.companies c on d.cmpID = c.cmpID"; buildAndTransformQuery(planner, sqlQuery); // Closes the planner eventually. diff --git a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java index 1f1e7bb75c6f..b1e39ff0a886 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java +++ b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java @@ -42,6 +42,11 @@ public enum JoinRelType { */ FULL, + /** + * Outer-cartesian join. + */ + OUTER_CARTESIAN, + /** * Semi-join. * @@ -78,7 +83,7 @@ public enum JoinRelType { * right-hand side. */ public boolean generatesNullsOnRight() { - return (this == LEFT) || (this == FULL); + return (this == LEFT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -86,7 +91,7 @@ public boolean generatesNullsOnRight() { * left-hand side. */ public boolean generatesNullsOnLeft() { - return (this == RIGHT) || (this == FULL); + return (this == RIGHT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -94,7 +99,7 @@ public boolean generatesNullsOnLeft() { * generate NULL values, either on the left-hand side or right-hand side. */ public boolean isOuterJoin() { - return (this == LEFT) || (this == RIGHT) || (this == FULL); + return (this == LEFT) || (this == RIGHT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -127,6 +132,7 @@ public boolean generatesNullsOn(int i) { * the left. */ public JoinRelType cancelNullsOnLeft() { switch (this) { + case OUTER_CARTESIAN: case RIGHT: return INNER; case FULL: @@ -140,6 +146,7 @@ public JoinRelType cancelNullsOnLeft() { * the right. */ public JoinRelType cancelNullsOnRight() { switch (this) { + case OUTER_CARTESIAN: case LEFT: return INNER; case FULL: @@ -152,6 +159,14 @@ public JoinRelType cancelNullsOnRight() { public boolean projectsRight() { return this != SEMI && this != ANTI; } + + public boolean isCommutativeAndAssociative() { + return this == INNER || this == OUTER_CARTESIAN; + } + + public boolean canApplyNullify() { + return this == INNER || this == LEFT || this == RIGHT; + } } // End JoinRelType.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java index 124fc81ef48f..4aa296acf041 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java @@ -22,7 +22,9 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.validate.SqlValidatorUtil; import com.google.common.collect.ImmutableList; @@ -96,6 +98,11 @@ public RexNode getPredicate() { public ImmutableList getAttributes() { return ImmutableList.copyOf(attributes); } + + @Override protected RelDataType deriveRowType() { + return SqlValidatorUtil.deriveNullifyRowType(input.getRowType(), attributes, + getCluster().getTypeFactory()); + } } // End Nullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java index 56206d5b78cd..b0c7b68fb467 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java @@ -31,6 +31,9 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mappings; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; import java.util.ArrayList; import java.util.List; @@ -48,6 +51,7 @@ */ public class JoinAssociateRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); /** The singleton. */ public static final JoinAssociateRule INSTANCE = @@ -101,10 +105,18 @@ public void onMatch(final RelOptRuleCall call) { return; } - // If either join is not inner, we cannot proceed. - // (Is this too strict?) - if (topJoin.getJoinType() != JoinRelType.INNER - || bottomJoin.getJoinType() != JoinRelType.INNER) { + // Only proceeds if the 2 joins are both inner or both outer-cartesian product. + boolean isBothInner = topJoin.getJoinType() == JoinRelType.INNER + && bottomJoin.getJoinType() == JoinRelType.INNER; + boolean isBothOuterCartesian = topJoin.getJoinType() == JoinRelType.OUTER_CARTESIAN + && bottomJoin.getJoinType() == JoinRelType.OUTER_CARTESIAN; + JoinRelType newJoinType = null; + if (isBothInner) { + newJoinType = JoinRelType.INNER; + } else if (isBothOuterCartesian) { + newJoinType = JoinRelType.OUTER_CARTESIAN; + } else { + LOGGER.debug("Will not proceed since the 2 join types are not associative."); return; } @@ -141,7 +153,7 @@ public void onMatch(final RelOptRuleCall call) { final Join newBottomJoin = bottomJoin.copy(bottomJoin.getTraitSet(), newBottomCondition, relB, - relC, JoinRelType.INNER, false); + relC, newJoinType, false); // Condition for newTopJoin consists of pieces from bottomJoin and topJoin. // Field ordinals do not need to be changed. @@ -149,7 +161,7 @@ public void onMatch(final RelOptRuleCall call) { @SuppressWarnings("SuspiciousNameCombination") final Join newTopJoin = topJoin.copy(topJoin.getTraitSet(), newTopCondition, relA, - newBottomJoin, JoinRelType.INNER, false); + newBottomJoin, newJoinType, false); call.transformTo(newTopJoin); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java index 315a0cd50084..5ec8b537a7dc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java @@ -114,7 +114,8 @@ public static RelNode swap(Join join, boolean swapOuterJoins) { public static RelNode swap(Join join, boolean swapOuterJoins, RelBuilder relBuilder) { final JoinRelType joinType = join.getJoinType(); - if (!swapOuterJoins && joinType != JoinRelType.INNER) { + if (!swapOuterJoins && !joinType.isCommutativeAndAssociative()) { + // Notice: outer cartesian product is commutative. return null; } final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java index dfc960c8592a..303dd7264668 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java @@ -77,7 +77,9 @@ public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { RelNode base = innerBestMatch.getInput(); // Constructs the new expression. - RelNode newNode = builder.push(base).nullify(oldPredicate, oldAttributes).bestMatch().build(); + RelNode newNode = builder.push(base) + .nullify(oldPredicate, oldAttributes) + .bestMatch().build(); call.transformTo(newNode); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java index 176455705c68..37d63c4f24b2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java @@ -27,6 +27,9 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; /** * BestMatchPullUpRule pulls up a best-match operator. Basically, the @@ -34,6 +37,7 @@ */ public class BestMatchPullUpRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); /** Instance of the current rule. */ public static final BestMatchPullUpRule INSTANCE = new BestMatchPullUpRule( @@ -61,12 +65,13 @@ public BestMatchPullUpRule(RelOptRuleOperand operand, String description) { final Join join = call.rel(0); final RexNode condition = join.getCondition(); if (!condition.equals(builder.literal(true))) { - throw new AssertionError(condition); + LOGGER.debug("The condition is not true"); + return; } // Constructs the new cartesian product. final BestMatch bestMatch = call.rel(1); - final RelNode cartesianJoin = + final RelNode outerCartesianJoin = join.copy( join.getTraitSet(), builder.literal(true), // Uses a literal condition which is always true. @@ -76,7 +81,7 @@ public BestMatchPullUpRule(RelOptRuleOperand operand, String description) { join.isSemiJoinDone()); // Builds the new expression. - RelNode reducedNode = builder.push(cartesianJoin).bestMatch().build(); + RelNode reducedNode = builder.push(outerCartesianJoin).bestMatch().build(); call.transformTo(reducedNode); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java index 3e1a4cd2c4c1..6fc68fa8ec0b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java @@ -21,22 +21,29 @@ import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; /** - * NullifyJoinRule + * NullifyJoinRule nullifies a 1-sided outer join or inner join operator in the + * following way: + * 1) 1-sided outer join: nullify the null-producing side; + * 2) inner join: nullify both sides; */ public class NullifyJoinRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); /** Instance of the rule that nullifies inner, left outer or right outer join. */ public static final NullifyJoinRule INSTANCE = @@ -59,48 +66,55 @@ public NullifyJoinRule(RelOptRuleOperand operand, String description) { RelBuilder builder = call.builder(); // The join operator at the current node. - Join join = call.rel(0); + final Join join = call.rel(0); + final JoinRelType joinType = join.getJoinType(); + if (join.getCondition().equals(builder.literal(true))) { + LOGGER.debug("No need to nullify cartesian product"); + return; + } else if (!joinType.canApplyNullify()) { + LOGGER.debug("Invalid join relation type"); + return; + } - // Determines the nullification attribute list based on the join type. - List nullificationList = new ArrayList<>(); - List leftFieldList = join.getLeft().getRowType().getFieldList(); - List rightFieldList = join.getRight().getRowType().getFieldList(); - List leftList = leftFieldList.stream() - .map(field -> new RexInputRef(field.getIndex(), field.getType())) - .collect(Collectors.toList()); - List rightList = rightFieldList.stream() - .map(field -> new RexInputRef(field.getIndex(), field.getType())) - .collect(Collectors.toList()); + // The new join operator (as outer-cartesian product). + final RelNode outerCartesianJoin = join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + join.getLeft(), + join.getRight(), + JoinRelType.OUTER_CARTESIAN, + join.isSemiJoinDone()); - switch (join.getJoinType()) { + // Determines the nullification attribute list based on the join type. + List joinFieldList = outerCartesianJoin.getRowType().getFieldList(); + List nullifyFieldList; + int leftFieldCount = join.getLeft().getRowType().getFieldCount(); + switch (joinType) { case LEFT: - nullificationList.addAll(rightList); + nullifyFieldList = joinFieldList.subList(leftFieldCount, joinFieldList.size()); break; case RIGHT: - nullificationList.addAll(leftList); + nullifyFieldList = joinFieldList.subList(0, leftFieldCount); break; case INNER: - nullificationList.addAll(leftList); - nullificationList.addAll(rightList); + nullifyFieldList = joinFieldList; break; default: - throw new AssertionError(join.getJoinType()); + throw new AssertionError(joinType); } + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); // Determines the nullification condition. RexNode nullificationCondition = join.getCondition(); // Builds the transformed relational tree. - final RelNode cartesianJoin = - join.copy( - join.getTraitSet(), - builder.literal(true), // Uses a literal condition which is always true. - join.getLeft(), - join.getRight(), - join.getJoinType(), - join.isSemiJoinDone()); - builder.push(cartesianJoin).nullify(nullificationCondition, nullificationList).bestMatch(); - call.transformTo(builder.build()); + final RelNode transformedNode = builder.push(outerCartesianJoin) + .nullify(nullificationCondition, nullificationList) + .bestMatch() + .build(); + call.transformTo(transformedNode); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java index 9d4e6d1afa3b..419f75ce096b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java @@ -27,6 +27,9 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; import java.util.List; @@ -36,6 +39,7 @@ */ public class NullifyPullUpRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); /** Instance of the current rule. */ public static final NullifyPullUpRule INSTANCE = new NullifyPullUpRule( @@ -63,12 +67,13 @@ public NullifyPullUpRule(RelOptRuleOperand operand, String description) { final Join join = call.rel(0); final RexNode condition = join.getCondition(); if (!condition.equals(builder.literal(true))) { - throw new AssertionError(condition); + LOGGER.debug("The condition is not true"); + return; } // Constructs the new cartesian product. final Nullify oldNullify = call.rel(1); - final RelNode cartesianJoin = + final RelNode outerCartesianJoin = join.copy( join.getTraitSet(), builder.literal(true), // Uses a literal condition which is always true. @@ -80,7 +85,8 @@ public NullifyPullUpRule(RelOptRuleOperand operand, String description) { // Builds the new expression. final RexNode oldPredicate = oldNullify.getPredicate(); final List oldAttributes = oldNullify.getAttributes(); - RelNode reducedNode = builder.push(cartesianJoin).nullify(oldPredicate, oldAttributes).build(); + RelNode reducedNode = builder.push(outerCartesianJoin) + .nullify(oldPredicate, oldAttributes).build(); call.transformTo(reducedNode); } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index c5d7316e90b8..9d90636c16f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -31,6 +31,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.CustomColumnResolvingTable; import org.apache.calcite.schema.ExtensibleTable; import org.apache.calcite.schema.Table; @@ -65,6 +66,7 @@ import com.google.common.collect.Lists; import java.nio.charset.Charset; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -464,6 +466,34 @@ public static List uniquify( : newNameList; } + /** + * Derives the type of a nullification operator. + * + * @param inputType is the row type of its input. + * @param attributes is the list of nullification attributes. + * @param typeFactory is the type factory. + * @return the row type. + */ + public static RelDataType deriveNullifyRowType( + RelDataType inputType, + List attributes, + RelDataTypeFactory typeFactory) { + return typeFactory.createStructType(inputType.getStructKind(), new AbstractList() { + @Override public RelDataType get(final int index) { + RelDataType fieldType = inputType.getFieldList().get(index).getType(); + boolean isNullable = attributes.stream().anyMatch(attr -> attr.hashCode() == index); + if (isNullable) { + return typeFactory.createTypeWithNullability(fieldType, true); + } + return fieldType; + } + + @Override public int size() { + return inputType.getFieldCount(); + } + }, inputType.getFieldNames()); + } + /** * Derives the type of a join relational expression. * From 81e3d01de25b7ee2db2fcb809aee7daf75543baa Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Fri, 4 Oct 2019 11:03:25 +0800 Subject: [PATCH 13/24] Add 2 assoc rules --- .../main/java/org/apache/calcite/Runner.java | 2 +- .../rel/rules/custom/AssocInnerOuterRule.java | 109 ++++++++++++++++++ .../rel/rules/custom/AssocOuterInnerRule.java | 97 ++++++++++++++++ .../rel/rules/custom/BestMatchReduceRule.java | 3 + .../rules/custom/NullifyJoinReverseRule.java | 75 ++++++++++++ .../rel/rules/custom/NullifyJoinRule.java | 7 +- 6 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 0b10d351d1a0..5a20981b42de 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -70,7 +70,7 @@ public static void main(String[] args) throws Exception { // A single left outer join. String sqlQuery = "select e.name, d.depName " + "from p.employees e left join p.departments d on e.depID = d.depID"; - buildAndTransformQuery(planner, sqlQuery); + // buildAndTransformQuery(planner, sqlQuery); // Two joins (left outer join + inner join). sqlQuery = "select e.name, d.depName, c.cmpName " diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java new file mode 100644 index 000000000000..a102d5520d9e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AssocInnerOuterRule applies limited associativity on inner join and outer join. + * + * Rule 22. + */ +public class AssocInnerOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AssocInnerOuterRule INSTANCE = new AssocInnerOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AssocInnerOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomInnerJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The bottom join is not a inner join."); + return; + } + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + topLeftJoin.getCondition(), + topLeftJoin.getLeft(), + bottomInnerJoin.getLeft(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomInnerJoin.copy( + bottomInnerJoin.getTraitSet(), + bottomInnerJoin.getCondition(), + newBottomLeftJoin, + bottomInnerJoin.getRight(), + JoinRelType.LEFT, + bottomInnerJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = bottomInnerJoin.getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } +} + +// End AssocInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java new file mode 100644 index 000000000000..6c539367c41d --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +/** + * AssocOuterInnerRule applies limited associativity on outer join and inner join. + * + * Rule 21. + */ +public class AssocOuterInnerRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AssocOuterInnerRule INSTANCE = new AssocOuterInnerRule( + operand(Join.class, + operand(Join.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AssocOuterInnerRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topInnerJoin = call.rel(0); + Join bottomLeftJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The top join is not an inner join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // The new operators. + final Join newBottomInnerJoin = topInnerJoin.copy( + topInnerJoin.getTraitSet(), + topInnerJoin.getCondition(), + bottomLeftJoin.getRight(), + topInnerJoin.getRight(), + JoinRelType.INNER, + topInnerJoin.isSemiJoinDone()); + final Join newTopInnerJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + bottomLeftJoin.getCondition(), + bottomLeftJoin.getLeft(), + newBottomInnerJoin, + JoinRelType.INNER, + bottomLeftJoin.isSemiJoinDone()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); + call.transformTo(transformedNode); + } +} + +// End OuterJoinAssociateRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java index 5ad7775bc5a6..7e7081c12ae2 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java @@ -27,6 +27,9 @@ /** * BestMatchReduceRule is able to reduce two consecutive best-match operators * into one. In the current implementation, the outer one will be eliminated. + * + * There is no reverse rule for this rule because NullifyJoinReverseRule will + * assume an appropriate best-match operator exists. */ public class BestMatchReduceRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java new file mode 100644 index 000000000000..42f9289074d6 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; + +/** + * NullifyJoinReverseRule converts a nullified join (i.e., nullification operator + * + outer cartesian product) backs to either a 1-sided outer join or inner join. + */ +public class NullifyJoinReverseRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the rule that reverses the nullification process. */ + public static final NullifyJoinReverseRule INSTANCE = + new NullifyJoinReverseRule(operand(Nullify.class, operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public NullifyJoinReverseRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyJoinReverseRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // The nullification operator at the top. + final Nullify nullify = call.rel(0); + + // The join operator at the bottom. + final Join join = call.rel(1); + if (join.getJoinType() != JoinRelType.OUTER_CARTESIAN) { + LOGGER.debug("Nullification reverse should only be applied when the join is an outer cartesian product"); + return; + } + + // Checks which join type should be converted back to. + final List joinFieldList = join.getRowType().getFieldList(); + } +} + +// End NullifyJoinReverseRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java index 6fc68fa8ec0b..a2a9b6ad4adb 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java @@ -68,11 +68,8 @@ public NullifyJoinRule(RelOptRuleOperand operand, String description) { // The join operator at the current node. final Join join = call.rel(0); final JoinRelType joinType = join.getJoinType(); - if (join.getCondition().equals(builder.literal(true))) { - LOGGER.debug("No need to nullify cartesian product"); - return; - } else if (!joinType.canApplyNullify()) { - LOGGER.debug("Invalid join relation type"); + if (!joinType.canApplyNullify()) { + LOGGER.debug("Invalid join relation type for nullify: " + joinType.toString()); return; } From 54dedbb5c1cc3c088954f7c8444b987542c6501c Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Fri, 4 Oct 2019 11:27:36 +0800 Subject: [PATCH 14/24] Add 3 asscom rules --- .../rules/custom/AsscomInnerOuterRule.java | 110 ++++++++++++++++++ .../rules/custom/AsscomOuterInnerRule.java | 96 +++++++++++++++ .../rules/custom/AsscomOuterOuterRule.java | 110 ++++++++++++++++++ .../rel/rules/custom/AssocInnerOuterRule.java | 3 +- 4 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java create mode 100644 core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java new file mode 100644 index 000000000000..e00aa530fc31 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AsscomInnerOuterRule applies limited r-asscom property on inner join and outer join. + * + * Rule 24. + */ +public class AsscomInnerOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomInnerOuterRule INSTANCE = new AsscomInnerOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomInnerOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomInnerJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The bottom join is not an inner join."); + return; + } + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + topLeftJoin.getCondition(), + topLeftJoin.getLeft(), + bottomInnerJoin.getRight(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomInnerJoin.copy( + bottomInnerJoin.getTraitSet(), + bottomInnerJoin.getCondition(), + newBottomLeftJoin, + bottomInnerJoin.getLeft(), + JoinRelType.LEFT, + bottomInnerJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = + bottomInnerJoin.getRight().getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } +} + +// End AsscomInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java new file mode 100644 index 000000000000..4eb2bfe4e29e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +/** + * AsscomOuterInnerRule applies limited r-asscom property on outer join and inner join. + * + * Rule 23. + */ +public class AsscomOuterInnerRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomOuterInnerRule INSTANCE = new AsscomOuterInnerRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomOuterInnerRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topInnerJoin = call.rel(0); + Join bottomLeftJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The top join is not an inner join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // The new operators. + final Join newBottomInnerJoin = topInnerJoin.copy( + topInnerJoin.getTraitSet(), + topInnerJoin.getCondition(), + topInnerJoin.getLeft(), + bottomLeftJoin.getRight(), + JoinRelType.INNER, + topInnerJoin.isSemiJoinDone()); + final Join newTopInnerJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + bottomLeftJoin.getCondition(), + bottomLeftJoin.getLeft(), + newBottomInnerJoin, + JoinRelType.INNER, + bottomLeftJoin.isSemiJoinDone()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); + call.transformTo(transformedNode); + } +} + +// End AsscomOuterInnerRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java new file mode 100644 index 000000000000..bc4b027bbdd9 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AsscomOuterOuterRule applies limited r-asscom property on outer join and outer join. + * + * Rule 25. + */ +public class AsscomOuterOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomOuterOuterRule INSTANCE = new AsscomOuterOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomOuterOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomLeftJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + topLeftJoin.getCondition(), + topLeftJoin.getLeft(), + bottomLeftJoin.getRight(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + bottomLeftJoin.getCondition(), + newBottomLeftJoin, + bottomLeftJoin.getLeft(), + JoinRelType.LEFT, + bottomLeftJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = + bottomLeftJoin.getRight().getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomLeftJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } +} + +// End AsscomOuterOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java index a102d5520d9e..5feb19829d85 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -94,7 +94,8 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { bottomInnerJoin.isSemiJoinDone()); // Determines the nullification attribute. - List nullifyFieldList = bottomInnerJoin.getRowType().getFieldList(); + List nullifyFieldList = + bottomInnerJoin.getLeft().getRowType().getFieldList(); List nullificationList = nullifyFieldList.stream() .map(field -> new RexInputRef(field.getIndex(), field.getType())) .collect(Collectors.toList()); From ae34d8606f26ca1958221e86166c607f8bbf8c79 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sat, 5 Oct 2019 23:05:02 +0800 Subject: [PATCH 15/24] Check whether the condition refers to the desired set of attributes --- .../org/apache/calcite/plan/RelOptUtil.java | 40 +++++++++++++++++++ .../rules/custom/AsscomInnerOuterRule.java | 7 ++++ .../rules/custom/AsscomOuterInnerRule.java | 7 ++++ .../rules/custom/AsscomOuterOuterRule.java | 7 ++++ .../rel/rules/custom/AssocInnerOuterRule.java | 7 ++++ .../rel/rules/custom/AssocOuterInnerRule.java | 7 ++++ 6 files changed, 75 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index bccb0f5f537a..508bb5d65ec6 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -1046,6 +1046,46 @@ public static RexNode splitCorrelatedFilterCondition( filter.getCluster().getRexBuilder(), nonEquiList, true); } + /** + * Checks whether a given list of attributes is the subset of another list + * of attributes. + * + * @param a is the given list of attributes. + * @param b is another given list of attributes. + * @return true if a is the subset of b; false otherwise. + */ + private static boolean isSubSet(List a, List b) { + Set set = new HashSet<>(b); + for (RelDataTypeField attribute: a) { + if (!set.contains(attribute)) { + return false; + } + } + + return true; + } + + /** + * Checks whether a given list of attributes is the subset of another list of list + * of attributes. + * + * @param a is the given list of attributes. + * @param others is the list of list of attributes. + * @return true if a is the subset of others; false otherwise. + */ + @SafeVarargs public static boolean isSubSet(List a, List... others) { + if (others.length == 0) { + return false; + } + + // Puts everything together. + List all = new ArrayList<>(others[0]); + for (int i = 1; i < others.length; i++) { + all.addAll(others[i]); + } + return isSubSet(a, all); + } + private static void splitJoinCondition( List sysFieldList, List inputs, diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java index e00aa530fc31..e7da059989db 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; @@ -75,6 +76,12 @@ public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { LOGGER.debug("The bottom join is not an inner join."); return; + } else if (!RelOptUtil.isSubSet( + topLeftJoin.getCondition().getType().getFieldList(), + topLeftJoin.getLeft().getRowType().getFieldList(), + bottomInnerJoin.getRight().getRowType().getFieldList())) { + LOGGER.debug("Not a subset of attributes."); + return; } // The new operators. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java index 4eb2bfe4e29e..a864027d19f0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; @@ -69,6 +70,12 @@ public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; + } else if (!RelOptUtil.isSubSet( + topInnerJoin.getCondition().getType().getFieldList(), + topInnerJoin.getLeft().getRowType().getFieldList(), + bottomLeftJoin.getRight().getRowType().getFieldList())) { + LOGGER.debug("Not a subset of attributes."); + return; } // The new operators. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java index bc4b027bbdd9..94196ec9644d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; @@ -75,6 +76,12 @@ public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; + } else if (!RelOptUtil.isSubSet( + topLeftJoin.getCondition().getType().getFieldList(), + topLeftJoin.getLeft().getRowType().getFieldList(), + bottomLeftJoin.getRight().getRowType().getFieldList())) { + LOGGER.debug("Not a subset of attributes."); + return; } // The new operators. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java index 5feb19829d85..ad675a93cabd 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; @@ -75,6 +76,12 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { LOGGER.debug("The bottom join is not a inner join."); return; + } else if (!RelOptUtil.isSubSet( + topLeftJoin.getCondition().getType().getFieldList(), + topLeftJoin.getLeft().getRowType().getFieldList(), + bottomInnerJoin.getLeft().getRowType().getFieldList())) { + LOGGER.debug("Not a subset of attributes."); + return; } // The new operators. diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java index 6c539367c41d..0641faaf00e0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; @@ -70,6 +71,12 @@ public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; + } else if (!RelOptUtil.isSubSet( + topInnerJoin.getCondition().getType().getFieldList(), + topInnerJoin.getRight().getRowType().getFieldList(), + bottomLeftJoin.getRight().getRowType().getFieldList())) { + LOGGER.debug("Not a subset of attributes."); + return; } // The new operators. From edb952eadd37dc5d16ea6597e8ec54d07251006d Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 6 Oct 2019 12:30:24 +0800 Subject: [PATCH 16/24] Refactor for runner class --- .../main/java/org/apache/calcite/Runner.java | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 5a20981b42de..1ab4b2aa609d 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -44,12 +44,6 @@ */ public class Runner { public static void main(String[] args) throws Exception { - // Builds the schema. - final SchemaPlus rootSchema = Frameworks.createRootSchema(true); - final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); - - // Creates the planner. - final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); final Program programs = Programs.ofRules( NullifyJoinRule.INSTANCE, NullifyPullUpRule.INSTANCE, @@ -60,37 +54,41 @@ public static void main(String[] args) throws Exception { JoinAssociateRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); - final FrameworkConfig config = Frameworks.newConfigBuilder() - .parserConfig(parserConfig) - .defaultSchema(defaultSchema) - .programs(programs) - .build(); - final Planner planner = Frameworks.getPlanner(config); // A single left outer join. String sqlQuery = "select e.name, d.depName " + "from p.employees e left join p.departments d on e.depID = d.depID"; - // buildAndTransformQuery(planner, sqlQuery); + buildAndTransformQuery(programs, sqlQuery); // Two joins (left outer join + inner join). sqlQuery = "select e.name, d.depName, c.cmpName " + "from p.employees e left join p.departments d on e.depID = d.depID " + "left join p.companies c on d.cmpID = c.cmpID"; - buildAndTransformQuery(planner, sqlQuery); - - // Closes the planner eventually. - planner.close(); + buildAndTransformQuery(programs, sqlQuery); } /** * This method emulates the whole life cycle of a given SQL query: parse, validate build and - * transform. It will close and reset the planner after usage. + * transform. It will close the planner after usage. * - * @param planner is the planner to be used during the life cycle. + * @param programs is the set of transformation rules to be used. * @param sqlQuery is the original SQL query in its string representation. * @throws Exception when there is error during any step. */ - private static void buildAndTransformQuery(Planner planner, String sqlQuery) throws Exception { + private static void buildAndTransformQuery(final Program programs, final String sqlQuery) throws Exception { + // Builds the schema. + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); + + // Creates the planner. + final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(parserConfig) + .defaultSchema(defaultSchema) + .programs(programs) + .build(); + final Planner planner = Frameworks.getPlanner(config); + System.out.println("============================ Start ============================"); // Parses, validates and builds the query. @@ -106,11 +104,10 @@ private static void buildAndTransformQuery(Planner planner, String sqlQuery) thr System.out.println("After transformation:\n"); System.out.println(RelOptUtil.toString(transformedNode)); - // Closes and resets the planner. - planner.close(); - planner.reset(); - System.out.println("============================= End =============================\n"); + + // Closes the planner. + planner.close(); } /** From 2ab85692cdead448c7862f7445dfc396c4a0dc5e Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 6 Oct 2019 13:43:37 +0800 Subject: [PATCH 17/24] Fix bug and add isNotReferringTo --- .../main/java/org/apache/calcite/Runner.java | 32 ++++++------ .../org/apache/calcite/plan/RelOptUtil.java | 50 ++++++++----------- .../rules/custom/AsscomInnerOuterRule.java | 12 +++-- .../rules/custom/AsscomOuterInnerRule.java | 15 ++++-- .../rules/custom/AsscomOuterOuterRule.java | 12 +++-- .../rel/rules/custom/AssocInnerOuterRule.java | 11 ++-- .../rel/rules/custom/AssocOuterInnerRule.java | 11 ++-- 7 files changed, 76 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 1ab4b2aa609d..ee9e813da126 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -23,13 +23,8 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.rules.JoinAssociateRule; -import org.apache.calcite.rel.rules.JoinCommuteRule; -import org.apache.calcite.rel.rules.custom.BestMatchOverNullifyRule; -import org.apache.calcite.rel.rules.custom.BestMatchPullUpRule; -import org.apache.calcite.rel.rules.custom.BestMatchReduceRule; +import org.apache.calcite.rel.rules.custom.AssocOuterInnerRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; -import org.apache.calcite.rel.rules.custom.NullifyPullUpRule; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; @@ -44,26 +39,31 @@ */ public class Runner { public static void main(String[] args) throws Exception { - final Program programs = Programs.ofRules( - NullifyJoinRule.INSTANCE, - NullifyPullUpRule.INSTANCE, - BestMatchReduceRule.INSTANCE, - BestMatchPullUpRule.INSTANCE, - BestMatchOverNullifyRule.INSTANCE, - JoinCommuteRule.INSTANCE, - JoinAssociateRule.INSTANCE, + // A single inner join. + String sqlQuery = "select e.name, d.depName " + + "from p.employees e join p.departments d on e.depID = d.depID"; + Program programs = Programs.ofRules( EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery); // A single left outer join. - String sqlQuery = "select e.name, d.depName " + sqlQuery = "select e.name, d.depName " + "from p.employees e left join p.departments d on e.depID = d.depID"; + programs = Programs.ofRules( + NullifyJoinRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); buildAndTransformQuery(programs, sqlQuery); // Two joins (left outer join + inner join). sqlQuery = "select e.name, d.depName, c.cmpName " + "from p.employees e left join p.departments d on e.depID = d.depID " - + "left join p.companies c on d.cmpID = c.cmpID"; + + "inner join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AssocOuterInnerRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); buildAndTransformQuery(programs, sqlQuery); } diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 508bb5d65ec6..639d03ebc4a1 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -125,6 +125,8 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Supplier; +import java.util.stream.Collectors; + import javax.annotation.Nonnull; /** @@ -1047,43 +1049,31 @@ public static RexNode splitCorrelatedFilterCondition( } /** - * Checks whether a given list of attributes is the subset of another list - * of attributes. + * Checks whether a given predicate is referring to any attribute in a given list. * - * @param a is the given list of attributes. - * @param b is another given list of attributes. - * @return true if a is the subset of b; false otherwise. - */ - private static boolean isSubSet(List a, List b) { - Set set = new HashSet<>(b); - for (RelDataTypeField attribute: a) { - if (!set.contains(attribute)) { - return false; - } - } - - return true; - } - - /** - * Checks whether a given list of attributes is the subset of another list of list - * of attributes. * - * @param a is the given list of attributes. - * @param others is the list of list of attributes. - * @return true if a is the subset of others; false otherwise. + * @param condition is the given predicate. + * @param fields is the given list of attributes. + * @return true if not referring to any attribute; false otherwise. */ - @SafeVarargs public static boolean isSubSet(List a, List... others) { - if (others.length == 0) { + public static boolean isNotReferringTo(RexNode condition, List fields) { + if (!(condition instanceof RexCall)) { return false; } - // Puts everything together. - List all = new ArrayList<>(others[0]); - for (int i = 1; i < others.length; i++) { - all.addAll(others[i]); + // Converts to RexNode format. + Set set = fields.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toSet()); + + // Checks whether the set contains each attribute. + RexCall call = (RexCall) condition; + for (RexNode attribute: call.getOperands()) { + if (set.contains(attribute)) { + return false; + } } - return isSubSet(a, all); + return true; } private static void splitJoinCondition( diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java index e7da059989db..62bd60016d22 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java @@ -76,10 +76,14 @@ public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { LOGGER.debug("The bottom join is not an inner join."); return; - } else if (!RelOptUtil.isSubSet( - topLeftJoin.getCondition().getType().getFieldList(), - topLeftJoin.getLeft().getRowType().getFieldList(), - bottomInnerJoin.getRight().getRowType().getFieldList())) { + } + + // Makes sure the join condition is referring to the correct set of fields. + int topLeftJoinLeft = topLeftJoin.getLeft().getRowType().getFieldCount(); + int bottomInnerJoinRight = bottomInnerJoin.getRight().getRowType().getFieldCount(); + List fields = topLeftJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + fields.subList(topLeftJoinLeft, fields.size() - bottomInnerJoinRight))) { LOGGER.debug("Not a subset of attributes."); return; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java index a864027d19f0..df39786b6cc8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java @@ -25,11 +25,14 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; import org.slf4j.Logger; +import java.util.List; + /** * AsscomOuterInnerRule applies limited r-asscom property on outer join and inner join. * @@ -70,10 +73,14 @@ public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; - } else if (!RelOptUtil.isSubSet( - topInnerJoin.getCondition().getType().getFieldList(), - topInnerJoin.getLeft().getRowType().getFieldList(), - bottomLeftJoin.getRight().getRowType().getFieldList())) { + } + + // Makes sure the join condition is referring to the correct set of fields. + int topInnerJoinLeft = topInnerJoin.getLeft().getRowType().getFieldCount(); + int bottomLeftJoinRight = bottomLeftJoin.getRight().getRowType().getFieldCount(); + List fields = topInnerJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topInnerJoin.getCondition(), + fields.subList(topInnerJoinLeft, fields.size() - bottomLeftJoinRight))) { LOGGER.debug("Not a subset of attributes."); return; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java index 94196ec9644d..f1d63b68bcac 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java @@ -76,10 +76,14 @@ public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; - } else if (!RelOptUtil.isSubSet( - topLeftJoin.getCondition().getType().getFieldList(), - topLeftJoin.getLeft().getRowType().getFieldList(), - bottomLeftJoin.getRight().getRowType().getFieldList())) { + } + + // Makes sure the join condition is referring to the correct set of fields. + int topLeftJoinLeft = topLeftJoin.getLeft().getRowType().getFieldCount(); + int bottomLeftJoinRight = bottomLeftJoin.getRight().getRowType().getFieldCount(); + List fields = topLeftJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + fields.subList(topLeftJoinLeft, fields.size() - bottomLeftJoinRight))) { LOGGER.debug("Not a subset of attributes."); return; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java index ad675a93cabd..97474a8950d5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -76,10 +76,13 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { LOGGER.debug("The bottom join is not a inner join."); return; - } else if (!RelOptUtil.isSubSet( - topLeftJoin.getCondition().getType().getFieldList(), - topLeftJoin.getLeft().getRowType().getFieldList(), - bottomInnerJoin.getLeft().getRowType().getFieldList())) { + } + + // Makes sure the join condition is referring to the correct set of fields. + int bottomInnerJoinRight = bottomInnerJoin.getRight().getRowType().getFieldCount(); + int total = topLeftJoin.getRowType().getFieldCount(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + topLeftJoin.getRowType().getFieldList().subList(total - bottomInnerJoinRight, total))) { LOGGER.debug("Not a subset of attributes."); return; } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java index 0641faaf00e0..5f5130f7c887 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -25,7 +25,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; @@ -71,10 +70,12 @@ public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { LOGGER.debug("The bottom join is not a left outer join."); return; - } else if (!RelOptUtil.isSubSet( - topInnerJoin.getCondition().getType().getFieldList(), - topInnerJoin.getRight().getRowType().getFieldList(), - bottomLeftJoin.getRight().getRowType().getFieldList())) { + } + + // Makes sure the join condition is referring to the correct set of fields. + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + if (!RelOptUtil.isNotReferringTo(topInnerJoin.getCondition(), + topInnerJoin.getRowType().getFieldList().subList(0, bottomLeftJoinLeft))) { LOGGER.debug("Not a subset of attributes."); return; } From 8a4001287ce72caf230fb9316c4dd27152edce61 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 6 Oct 2019 14:36:29 +0800 Subject: [PATCH 18/24] Fix lint errors --- core/src/main/java/org/apache/calcite/Runner.java | 3 ++- core/src/main/java/org/apache/calcite/plan/RelOptUtil.java | 1 - .../apache/calcite/rel/rules/custom/AssocOuterInnerRule.java | 2 +- .../calcite/rel/rules/custom/NullifyJoinReverseRule.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index ee9e813da126..8fbb506def3b 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -75,7 +75,8 @@ public static void main(String[] args) throws Exception { * @param sqlQuery is the original SQL query in its string representation. * @throws Exception when there is error during any step. */ - private static void buildAndTransformQuery(final Program programs, final String sqlQuery) throws Exception { + private static void buildAndTransformQuery( + final Program programs, final String sqlQuery) throws Exception { // Builds the schema. final SchemaPlus rootSchema = Frameworks.createRootSchema(true); final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 639d03ebc4a1..a22849416acf 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -126,7 +126,6 @@ import java.util.TreeSet; import java.util.function.Supplier; import java.util.stream.Collectors; - import javax.annotation.Nonnull; /** diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java index 5f5130f7c887..8715d44ea89f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -102,4 +102,4 @@ public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { } } -// End OuterJoinAssociateRule.java +// End AssocOuterInnerRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java index 42f9289074d6..7fb6f57ad3f6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java @@ -63,7 +63,7 @@ public NullifyJoinReverseRule(RelOptRuleOperand operand, String description) { // The join operator at the bottom. final Join join = call.rel(1); if (join.getJoinType() != JoinRelType.OUTER_CARTESIAN) { - LOGGER.debug("Nullification reverse should only be applied when the join is an outer cartesian product"); + LOGGER.debug("Should only be applied when the join is an outer cartesian product"); return; } From 5dd48fe85ed7dcae1339beec3399216623dbc721 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 6 Oct 2019 17:19:26 +0800 Subject: [PATCH 19/24] Add feature flag to disable type check --- core/src/main/java/org/apache/calcite/Runner.java | 2 ++ core/src/main/java/org/apache/calcite/plan/RelOptUtil.java | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 8fbb506def3b..27a59adc8463 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -64,7 +64,9 @@ public static void main(String[] args) throws Exception { AssocOuterInnerRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); + RelOptUtil.disableTypeCheck = true; buildAndTransformQuery(programs, sqlQuery); + RelOptUtil.disableTypeCheck = false; } /** diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index a22849416acf..ae7f0104d484 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -139,6 +139,11 @@ public abstract class RelOptUtil { public static final double EPSILON = 1.0e-5; + /** + * A feature flag to decide whether we should disable typeCheck temporarily. + */ + public static boolean disableTypeCheck = false; + @SuppressWarnings("Guava") @Deprecated // to be removed before 2.0 public static final com.google.common.base.Predicate @@ -349,7 +354,7 @@ public static boolean areRowTypesEqual( || type2.getSqlTypeName() == SqlTypeName.ANY) { continue; } - if (!type1.equals(type2)) { + if (!type1.equals(type2) && !disableTypeCheck) { return false; } } From 6ca67d50291e37a2271cc8ae2b522eaf6c9aa4d9 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Mon, 7 Oct 2019 10:41:53 +0800 Subject: [PATCH 20/24] Replace variable indices when necessary --- .../main/java/org/apache/calcite/Runner.java | 91 ++++++++++++++++--- .../rules/custom/AsscomInnerOuterRule.java | 63 ++++++++++++- .../rules/custom/AsscomOuterInnerRule.java | 65 ++++++++++++- .../rules/custom/AsscomOuterOuterRule.java | 63 ++++++++++++- .../rel/rules/custom/AssocInnerOuterRule.java | 2 +- 5 files changed, 262 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index 27a59adc8463..de785201ef4e 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -23,6 +23,11 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.JoinCommuteRule; +import org.apache.calcite.rel.rules.custom.AsscomInnerOuterRule; +import org.apache.calcite.rel.rules.custom.AsscomOuterInnerRule; +import org.apache.calcite.rel.rules.custom.AsscomOuterOuterRule; +import org.apache.calcite.rel.rules.custom.AssocInnerOuterRule; import org.apache.calcite.rel.rules.custom.AssocOuterInnerRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; import org.apache.calcite.schema.SchemaPlus; @@ -38,35 +43,84 @@ * A runner class for manual testing. */ public class Runner { + private static int count = 1; + public static void main(String[] args) throws Exception { - // A single inner join. + // 1. A single inner join. String sqlQuery = "select e.name, d.depName " + "from p.employees e join p.departments d on e.depID = d.depID"; Program programs = Programs.ofRules( EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); - buildAndTransformQuery(programs, sqlQuery); + buildAndTransformQuery(programs, sqlQuery, false); - // A single left outer join. + // 2. A single left outer join. sqlQuery = "select e.name, d.depName " + "from p.employees e left join p.departments d on e.depID = d.depID"; programs = Programs.ofRules( NullifyJoinRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); - buildAndTransformQuery(programs, sqlQuery); + buildAndTransformQuery(programs, sqlQuery, false); - // Two joins (left outer join + inner join). + // 3. Two joins (left outer join + inner join) - for Rule 21. sqlQuery = "select e.name, d.depName, c.cmpName " - + "from p.employees e left join p.departments d on e.depID = d.depID " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + "inner join p.companies c on d.cmpID = c.cmpID"; programs = Programs.ofRules( AssocOuterInnerRule.INSTANCE, EnumerableRules.ENUMERABLE_PROJECT_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE); - RelOptUtil.disableTypeCheck = true; - buildAndTransformQuery(programs, sqlQuery); - RelOptUtil.disableTypeCheck = false; + buildAndTransformQuery(programs, sqlQuery, true); + + // 4. Two joins (inner join + left outer join) - for Rule 22. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.departments d " + + "inner join p.employees e on d.depID = e.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AssocInnerOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 5. Two joins (left outer join + inner join) - for Rule 23. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + + "inner join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomOuterInnerRule.INSTANCE, + JoinCommuteRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 6. Two joins (inner join + left outer join) - for Rule 24. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "inner join p.departments d on e.depID = d.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomInnerOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 7. Two joins (left outer join + left outer join) - for Rule 25. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomOuterOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); } /** @@ -75,10 +129,11 @@ public static void main(String[] args) throws Exception { * * @param programs is the set of transformation rules to be used. * @param sqlQuery is the original SQL query in its string representation. + * @param ignoreTypeCheck indicates whether type check should be turned off. * @throws Exception when there is error during any step. */ - private static void buildAndTransformQuery( - final Program programs, final String sqlQuery) throws Exception { + private static void buildAndTransformQuery(final Program programs, + final String sqlQuery, final boolean ignoreTypeCheck) throws Exception { // Builds the schema. final SchemaPlus rootSchema = Frameworks.createRootSchema(true); final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); @@ -93,6 +148,10 @@ private static void buildAndTransformQuery( final Planner planner = Frameworks.getPlanner(config); System.out.println("============================ Start ============================"); + System.out.println("Transaction ID: " + count++); + if (ignoreTypeCheck) { + RelOptUtil.disableTypeCheck = true; + } // Parses, validates and builds the query. SqlNode parse = planner.parse(sqlQuery); @@ -107,6 +166,9 @@ private static void buildAndTransformQuery( System.out.println("After transformation:\n"); System.out.println(RelOptUtil.toString(transformedNode)); + if (ignoreTypeCheck) { + RelOptUtil.disableTypeCheck = false; + } System.out.println("============================= End =============================\n"); // Closes the planner. @@ -120,7 +182,12 @@ public static class People { new Employee(10, 1, "Daniel"), new Employee(20, 1, "Mark"), new Employee(30, 2, "Smith"), - new Employee(40, 3, "Armstrong") + new Employee(40, 3, "Armstrong"), + new Employee(50, 2, "Gabriel"), + new Employee(60, 5, "Daniel"), + new Employee(70, 7, "Joe"), + new Employee(80, 2, "Kim"), + new Employee(90, 1, "Gino") }; public final Department[] departments = { diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java index 62bd60016d22..6ef46ea07570 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java @@ -26,11 +26,15 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; + import org.slf4j.Logger; import java.util.List; @@ -67,7 +71,7 @@ public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { @Override public void onMatch(final RelOptRuleCall call) { // Gets the two original join operators. Join topLeftJoin = call.rel(0); - Join bottomInnerJoin = call.rel(1); + Join bottomInnerJoin = call.rel(2); // Makes sure the join types match the rule. if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { @@ -88,17 +92,23 @@ public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { return; } + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + int bottomInnerJoinLeft = bottomInnerJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topLeftJoinLeft, bottomInnerJoinLeft, bottomInnerJoinRight); + // The new operators. final Join newBottomLeftJoin = topLeftJoin.copy( topLeftJoin.getTraitSet(), - topLeftJoin.getCondition(), + replacer.replace(topLeftJoin.getCondition(), 0), topLeftJoin.getLeft(), bottomInnerJoin.getRight(), JoinRelType.LEFT, topLeftJoin.isSemiJoinDone()); final Join newTopLeftJoin = bottomInnerJoin.copy( bottomInnerJoin.getTraitSet(), - bottomInnerJoin.getCondition(), + replacer.replace(bottomInnerJoin.getCondition(), 0), newBottomLeftJoin, bottomInnerJoin.getLeft(), JoinRelType.LEFT, @@ -116,6 +126,53 @@ public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); call.transformTo(transformedNode); } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex(); + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() + bottomRight; + } else { + newIndex = var.getIndex() - bottomLeft; + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } } // End AsscomInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java index df39786b6cc8..ae1f4d96bd1a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java @@ -26,9 +26,15 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; + import org.slf4j.Logger; import java.util.List; @@ -64,7 +70,7 @@ public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { @Override public void onMatch(final RelOptRuleCall call) { // Gets the two original join operators. Join topInnerJoin = call.rel(0); - Join bottomLeftJoin = call.rel(1); + Join bottomLeftJoin = call.rel(2); // Makes sure the join types match the rule. if (topInnerJoin.getJoinType() != JoinRelType.INNER) { @@ -85,17 +91,23 @@ public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { return; } + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topInnerJoin.getCluster().getRexBuilder(); + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topInnerJoinLeft, bottomLeftJoinLeft, bottomLeftJoinRight); + // The new operators. final Join newBottomInnerJoin = topInnerJoin.copy( topInnerJoin.getTraitSet(), - topInnerJoin.getCondition(), + replacer.replace(topInnerJoin.getCondition(), bottomLeftJoinLeft), topInnerJoin.getLeft(), bottomLeftJoin.getRight(), JoinRelType.INNER, topInnerJoin.isSemiJoinDone()); final Join newTopInnerJoin = bottomLeftJoin.copy( bottomLeftJoin.getTraitSet(), - bottomLeftJoin.getCondition(), + replacer.replace(bottomLeftJoin.getCondition(), 0), bottomLeftJoin.getLeft(), newBottomInnerJoin, JoinRelType.INNER, @@ -105,6 +117,53 @@ public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); call.transformTo(transformedNode); } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex() + bottomLeft; + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() - topLeft; + } else { + newIndex = var.getIndex(); + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } } // End AsscomOuterInnerRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java index f1d63b68bcac..a09ab9a3569d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java @@ -26,11 +26,15 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; + import org.slf4j.Logger; import java.util.List; @@ -67,7 +71,7 @@ public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { @Override public void onMatch(final RelOptRuleCall call) { // Gets the two original join operators. Join topLeftJoin = call.rel(0); - Join bottomLeftJoin = call.rel(1); + Join bottomLeftJoin = call.rel(2); // Makes sure the join types match the rule. if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { @@ -88,17 +92,23 @@ public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { return; } + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topLeftJoinLeft, bottomLeftJoinLeft, bottomLeftJoinRight); + // The new operators. final Join newBottomLeftJoin = topLeftJoin.copy( topLeftJoin.getTraitSet(), - topLeftJoin.getCondition(), + replacer.replace(topLeftJoin.getCondition(), 0), topLeftJoin.getLeft(), bottomLeftJoin.getRight(), JoinRelType.LEFT, topLeftJoin.isSemiJoinDone()); final Join newTopLeftJoin = bottomLeftJoin.copy( bottomLeftJoin.getTraitSet(), - bottomLeftJoin.getCondition(), + replacer.replace(bottomLeftJoin.getCondition(), 0), newBottomLeftJoin, bottomLeftJoin.getLeft(), JoinRelType.LEFT, @@ -116,6 +126,53 @@ public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { .nullify(bottomLeftJoin.getCondition(), nullificationList).bestMatch().build(); call.transformTo(transformedNode); } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex(); + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() + bottomRight; + } else { + newIndex = var.getIndex() - bottomLeft; + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } } // End AsscomOuterOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java index 97474a8950d5..d7caa0bf2328 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -67,7 +67,7 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { @Override public void onMatch(final RelOptRuleCall call) { // Gets the two original join operators. Join topLeftJoin = call.rel(0); - Join bottomInnerJoin = call.rel(1); + Join bottomInnerJoin = call.rel(2); // Makes sure the join types match the rule. if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { From b3de83fef963e3aa60fc0cc6be044838d23aa42a Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 13 Oct 2019 17:59:48 +0800 Subject: [PATCH 21/24] Populate nullification sets for all relations --- .../rel/rules/custom/NullifyJoinRule.java | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java index a2a9b6ad4adb..5a185b64eca9 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java @@ -19,10 +19,12 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; @@ -30,9 +32,17 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + import org.slf4j.Logger; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; import java.util.stream.Collectors; /** @@ -49,6 +59,17 @@ public class NullifyJoinRule extends RelOptRule { public static final NullifyJoinRule INSTANCE = new NullifyJoinRule(operand(Join.class, any()), null); + /** + * A map from table name to a set of predicates, which becomes the relation's + * nullification set. This map shall only be filled once. + */ + private static final Map> NULLIFICATION_SET_MAP = new HashMap<>(); + + /** + * A boolean flag to indicate whether we have filled in the above map. + */ + private static boolean hasFilledMap = false; + //~ Constructors ----------------------------------------------------------- public NullifyJoinRule(RelOptRuleOperand operand, @@ -63,6 +84,11 @@ public NullifyJoinRule(RelOptRuleOperand operand, String description) { //~ Methods ---------------------------------------------------------------- @Override public void onMatch(final RelOptRuleCall call) { + if (!hasFilledMap) { + fillNullificationSetMap(call.getPlanner().getRoot()); + hasFilledMap = true; + } + RelBuilder builder = call.builder(); // The join operator at the current node. @@ -113,6 +139,102 @@ public NullifyJoinRule(RelOptRuleOperand operand, String description) { .build(); call.transformTo(transformedNode); } + + /** + * Fills the nullification set map of this query. + * + * @param root is the root node of the query. + * @return a list consisting of the full names of all tables under this node. + */ + private static List fillNullificationSetMap(RelNode root) { + if (root instanceof TableScan) { + TableScan tableScan = (TableScan) root; + String tableName = getTableFullName(tableScan); + + // Initializes the nullification set of the base relation to be an empty set. + NULLIFICATION_SET_MAP.put(tableName, new HashSet<>()); + + // Returns this base relation's full name. + return ImmutableList.of(tableName); + } else if (root instanceof RelSubset) { + RelSubset relSubset = (RelSubset) root; + return fillNullificationSetMap(relSubset.getOriginal()); + } else if (root instanceof Join) { + Join join = (Join) root; + + // Traverses its left and right children first (the algorithm requires postfix order). + List leftNames = fillNullificationSetMap(join.getLeft()); + List rightNames = fillNullificationSetMap(join.getRight()); + + // The current join predicate will always be added. + Set toAdd = new HashSet<>(); + toAdd.add(join.getCondition()); + + // Populates the nullification set. + switch (join.getJoinType()) { + case LEFT: + // Prepares all the predicates that need to be added. + for (String name: leftNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set. + for (String name: rightNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + break; + case RIGHT: + // Prepares all the predicates that need to be added. + for (String name: rightNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set. + for (String name: leftNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + break; + case INNER: + // Prepares all the predicates that need to be added. + for (String name: leftNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + Set toAdd2 = new HashSet<>(); + toAdd2.add(join.getCondition()); + for (String name: rightNames) { + toAdd2.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set for both sides. + for (String name: rightNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + for (String name: leftNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd2); + } + break; + default: + throw new AssertionError("Unsupported join type: " + join.getJoinType()); + } + + // Merges the table names from both sides together. + return ImmutableList.copyOf(Iterables.concat(leftNames, rightNames)); + } else { + return fillNullificationSetMap(root.getInput(0)); + } + } + + private static String getTableFullName(TableScan tableScan) { + StringJoiner joiner = new StringJoiner("."); + for (String name: tableScan.getTable().getQualifiedName()) { + joiner.add(name); + } + return joiner.toString(); + } } // End NullifyJoinRule.java From 2251db37df6c1eaaebe2b9c8e901ebd14171cd48 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Mon, 21 Oct 2019 20:03:02 +0800 Subject: [PATCH 22/24] Convert transformed node back to SQL and print --- .../main/java/org/apache/calcite/Runner.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java index de785201ef4e..f00104ab4b28 100644 --- a/core/src/main/java/org/apache/calcite/Runner.java +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -23,6 +23,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; import org.apache.calcite.rel.rules.JoinCommuteRule; import org.apache.calcite.rel.rules.custom.AsscomInnerOuterRule; import org.apache.calcite.rel.rules.custom.AsscomOuterInnerRule; @@ -31,6 +32,7 @@ import org.apache.calcite.rel.rules.custom.AssocOuterInnerRule; import org.apache.calcite.rel.rules.custom.NullifyJoinRule; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.tools.FrameworkConfig; @@ -43,6 +45,11 @@ * A runner class for manual testing. */ public class Runner { + // Defines the default dialect used in this class. + private static final SqlDialect DEFAULT_DIALECT + = SqlDialect.DatabaseProduct.MYSQL.getDialect(); + + // A counter for the number of transactions executed so far. private static int count = 1; public static void main(String[] args) throws Exception { @@ -148,27 +155,37 @@ private static void buildAndTransformQuery(final Program programs, final Planner planner = Frameworks.getPlanner(config); System.out.println("============================ Start ============================"); - System.out.println("Transaction ID: " + count++); + System.out.println("Transaction ID: " + count++ + "\n"); + + // Prints the original SQL query string. + System.out.println("Input query:"); + System.out.println(sqlQuery + "\n"); if (ignoreTypeCheck) { RelOptUtil.disableTypeCheck = true; } // Parses, validates and builds the query. - SqlNode parse = planner.parse(sqlQuery); - SqlNode validate = planner.validate(parse); - RelNode relNode = planner.rel(validate).rel; - System.out.println("Before transformation:\n"); + final SqlNode parse = planner.parse(sqlQuery); + final SqlNode validate = planner.validate(parse); + final RelNode relNode = planner.rel(validate).rel; + System.out.println("Before transformation:"); System.out.println(RelOptUtil.toString(relNode)); // Transforms the query. RelTraitSet traitSet = relNode.getTraitSet().replace(EnumerableConvention.INSTANCE); RelNode transformedNode = planner.transform(0, traitSet, relNode); - System.out.println("After transformation:\n"); + System.out.println("After transformation:"); System.out.println(RelOptUtil.toString(transformedNode)); - if (ignoreTypeCheck) { RelOptUtil.disableTypeCheck = false; } + + // Converts the transformed relational expression back to SQL query string. + final RelToSqlConverter converter = new RelToSqlConverter(DEFAULT_DIALECT); + final SqlNode transformedSqlNode = converter.visitChild(0, transformedNode).asStatement(); + final String transformedSqlQuery = transformedSqlNode.toSqlString(DEFAULT_DIALECT).getSql(); + System.out.println("Output query:"); + System.out.println(transformedSqlQuery + "\n"); System.out.println("============================= End =============================\n"); // Closes the planner. From a7f6a0e0832ee065ab4a49bd4f0d56e2d965cc54 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Mon, 21 Oct 2019 20:29:53 +0800 Subject: [PATCH 23/24] Add variable replacers for rule 21 & 22 --- .../rel/rules/custom/AssocInnerOuterRule.java | 44 +++++++++++++++++- .../rel/rules/custom/AssocOuterInnerRule.java | 46 ++++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java index d7caa0bf2328..8b1ca8ca3f96 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -26,11 +26,15 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; + import org.slf4j.Logger; import java.util.List; @@ -87,17 +91,21 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { return; } + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + final VariableReplacer replacer = new VariableReplacer(rexBuilder); + // The new operators. final Join newBottomLeftJoin = topLeftJoin.copy( topLeftJoin.getTraitSet(), - topLeftJoin.getCondition(), + replacer.replace(topLeftJoin.getCondition(), 0), topLeftJoin.getLeft(), bottomInnerJoin.getLeft(), JoinRelType.LEFT, topLeftJoin.isSemiJoinDone()); final Join newTopLeftJoin = bottomInnerJoin.copy( bottomInnerJoin.getTraitSet(), - bottomInnerJoin.getCondition(), + replacer.replace(bottomInnerJoin.getCondition(), 0), newBottomLeftJoin, bottomInnerJoin.getRight(), JoinRelType.LEFT, @@ -115,6 +123,38 @@ public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); call.transformTo(transformedNode); } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + + VariableReplacer(RexBuilder rexBuilder) { + this.rexBuilder = rexBuilder; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Re-builds the attribute. + int newIndex = var.getIndex() - offset; + return rexBuilder.makeInputRef(var.getType(), newIndex); + } else { + return rex; + } + } + } } // End AssocInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java index 8715d44ea89f..6bf63eacdb18 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -25,9 +25,15 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.trace.CalciteTrace; +import com.google.common.collect.ImmutableList; + import org.slf4j.Logger; /** @@ -80,17 +86,21 @@ public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { return; } + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topInnerJoin.getCluster().getRexBuilder(); + final VariableReplacer replacer = new VariableReplacer(rexBuilder); + // The new operators. final Join newBottomInnerJoin = topInnerJoin.copy( topInnerJoin.getTraitSet(), - topInnerJoin.getCondition(), + replacer.replace(topInnerJoin.getCondition(), bottomLeftJoinLeft), bottomLeftJoin.getRight(), topInnerJoin.getRight(), JoinRelType.INNER, topInnerJoin.isSemiJoinDone()); final Join newTopInnerJoin = bottomLeftJoin.copy( bottomLeftJoin.getTraitSet(), - bottomLeftJoin.getCondition(), + replacer.replace(bottomLeftJoin.getCondition(), 0), bottomLeftJoin.getLeft(), newBottomInnerJoin, JoinRelType.INNER, @@ -100,6 +110,38 @@ public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); call.transformTo(transformedNode); } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + + VariableReplacer(RexBuilder rexBuilder) { + this.rexBuilder = rexBuilder; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Re-builds the attribute. + int newIndex = var.getIndex() - offset; + return rexBuilder.makeInputRef(var.getType(), newIndex); + } else { + return rex; + } + } + } } // End AssocOuterInnerRule.java From 2c68dc6d36c626d79f2106f3e2e249893d623986 Mon Sep 17 00:00:00 2001 From: Yunpeng Niu Date: Sun, 27 Oct 2019 11:54:52 +0800 Subject: [PATCH 24/24] Improve README and null-tolerance checking --- README.md | 10 +++ .../custom/BestMatchOverNullifyRule.java | 70 ++++++++++++------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 38938e820f40..5e89135fc159 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,16 @@ This is a forked version of the [Apache Calcite](http://calcite.apache.org) fram This [repository](https://github.com/yunpengn/calcite) is currently maintained by **[Yunpeng Niu](https://github.com/yunpengn)**. +## Development Environment Setup + +- Install the latest version of [IntelliJ IDEA](https://www.jetbrains.com/idea/) by [JetBrains](https://www.jetbrains.com/). +- Clone the repository by `git clone git@github.com:yunpengn/calcite.git`. +- Navigate to the cloned folder by `cd calcite/`. +- Import all dependencies by `./mvnw -DskipTests clean install`. + - This step may take a long time and need stable Internet connection. Please be patient. +- Open the IDE and import the project. +- Start coding! + ## Licence [Apache Licence 2.0](LICENSE) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java index 303dd7264668..ac2efa189a6f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java @@ -29,6 +29,8 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import com.google.common.collect.ImmutableList; + import java.util.List; /** @@ -67,8 +69,8 @@ public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { RexNode oldPredicate = nullify.getPredicate(); List oldAttributes = nullify.getAttributes(); - // Makes sure the nullification predicate is null-intolerant. - if (isNullTolerant(oldPredicate)) { + // Makes sure the nullification predicate is null-intolerant (cannot evaluate to TRUE). + if (valueForNull(oldPredicate) == 1) { throw new AssertionError("The nullification predicate is not null-intolerant."); } @@ -84,38 +86,58 @@ public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { } /** - * Checks whether a given predicate tolerates NULL values. A predicate is - * null-intolerant if it cannot evaluate to TRUE when referring a NULL - * value. + * Evaluates the value of a given predicate when referring to null values. * * @param predicate is the predicate to be tested. - * @return true if null tolerant; false otherwise. + * @return 1 if the predicate evaluates to TRUE, 0 if UNKNOWN, -1 if false. */ - private boolean isNullTolerant(RexNode predicate) { - if (predicate.isA(SqlKind.OR)) { + private int valueForNull(RexNode predicate) { + if (predicate.isA(SqlKind.AND)) { RexCall call = (RexCall) predicate; - - // An OR connective is null-tolerant if any of its child expressions is null-tolerant. - for (RexNode operand: call.getOperands()) { - if (isNullTolerant(operand)) { - return true; + boolean isAllTrue = true; + boolean hasAtLeastOneFalse = false; + + // Iterates through each child. + for (RexNode child: call.getOperands()) { + int childValue = valueForNull(child); + if (childValue != 1) { + isAllTrue = false; + } + if (childValue == -1) { + hasAtLeastOneFalse = true; } } - return false; - } else if (predicate.isA(SqlKind.AND)) { - RexCall call = (RexCall) predicate; - // An AND connective is null-tolerant if all of its child expressions are null-tolerant. - for (RexNode operand: call.getOperands()) { - if (!isNullTolerant(operand)) { - return false; + return isAllTrue ? 1 : (hasAtLeastOneFalse ? -1 : 0); + } else if (predicate.isA(SqlKind.OR)) { + RexCall call = (RexCall) predicate; + boolean isAllFalse = true; + boolean hasAtLeastOneTrue = false; + + // Iterates through each child. + for (RexNode child: call.getOperands()) { + int childValue = valueForNull(child); + if (childValue != -1) { + isAllFalse = false; + } + if (childValue == 1) { + hasAtLeastOneTrue = true; } } - return true; - } - // IS NULL and TRUE are both null-tolerant. - return predicate.isA(SqlKind.IS_NULL) || predicate.isA(SqlKind.IS_TRUE); + return isAllFalse ? -1 : (hasAtLeastOneTrue ? 1 : 0); + } else if (predicate.isA(SqlKind.NOT)) { + RexCall call = (RexCall) predicate; + RexNode child = call.getOperands().get(0); + int childValue = valueForNull(child); + return childValue == 0 ? 0 : -childValue; + } else if (predicate.isA(ImmutableList.of(SqlKind.IS_NULL, SqlKind.IS_TRUE))) { + return 1; + } else if (predicate.isA(ImmutableList.of(SqlKind.IS_NULL, SqlKind.IS_TRUE))) { + return -1; + } else { + return 0; + } } }