diff --git a/README.md b/README.md index a9cfee9a604f..5e89135fc159 100644 --- a/README.md +++ b/README.md @@ -16,21 +16,25 @@ See the License for the specific language governing permissions and limitations under the License. {% endcomment %} --> -[![Travis Build Status](https://travis-ci.org/apache/calcite.svg?branch=master)](https://travis-ci.org/apache/calcite) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/apache/calcite?svg=true&branch=master)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/calcite) +[![Travis Build Status](https://travis-ci.com/yunpengn/calcite.svg?branch=master)](https://travis-ci.com/yunpengn/calcite) +[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/qo30vjfl2rwsapnx?svg=true)](https://ci.appveyor.com/project/yunpengn/calcite) # Apache Calcite -Apache Calcite is a dynamic data management framework. +This is a forked version of the [Apache Calcite](http://calcite.apache.org) framework, with enhancements on outer join reorderability. _We do NOT guarantee compatibility with its upstream version._ -It contains many of the pieces that comprise a typical -database management system but omits the storage primitives. -It provides an industry standard SQL parser and validator, -a customisable optimizer with pluggable rules and cost functions, -logical and physical algebraic operators, various transformation -algorithms from SQL to algebra (and the opposite), and many -adapters for executing SQL queries over Cassandra, Druid, -Elasticsearch, MongoDB, Kafka, and others, with minimal -configuration. +This [repository](https://github.com/yunpengn/calcite) is currently maintained by **[Yunpeng Niu](https://github.com/yunpengn)**. -For more details, see the [home page](http://calcite.apache.org). +## Development Environment Setup + +- Install the latest version of [IntelliJ IDEA](https://www.jetbrains.com/idea/) by [JetBrains](https://www.jetbrains.com/). +- Clone the repository by `git clone git@github.com:yunpengn/calcite.git`. +- Navigate to the cloned folder by `cd calcite/`. +- Import all dependencies by `./mvnw -DskipTests clean install`. + - This step may take a long time and need stable Internet connection. Please be patient. +- Open the IDE and import the project. +- Start coding! + +## Licence + +[Apache Licence 2.0](LICENSE) diff --git a/core/src/main/java/org/apache/calcite/Runner.java b/core/src/main/java/org/apache/calcite/Runner.java new file mode 100644 index 000000000000..f00104ab4b28 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/Runner.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.config.Lex; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; +import org.apache.calcite.rel.rules.JoinCommuteRule; +import org.apache.calcite.rel.rules.custom.AsscomInnerOuterRule; +import org.apache.calcite.rel.rules.custom.AsscomOuterInnerRule; +import org.apache.calcite.rel.rules.custom.AsscomOuterOuterRule; +import org.apache.calcite.rel.rules.custom.AssocInnerOuterRule; +import org.apache.calcite.rel.rules.custom.AssocOuterInnerRule; +import org.apache.calcite.rel.rules.custom.NullifyJoinRule; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Program; +import org.apache.calcite.tools.Programs; + +/** + * A runner class for manual testing. + */ +public class Runner { + // Defines the default dialect used in this class. + private static final SqlDialect DEFAULT_DIALECT + = SqlDialect.DatabaseProduct.MYSQL.getDialect(); + + // A counter for the number of transactions executed so far. + private static int count = 1; + + public static void main(String[] args) throws Exception { + // 1. A single inner join. + String sqlQuery = "select e.name, d.depName " + + "from p.employees e join p.departments d on e.depID = d.depID"; + Program programs = Programs.ofRules( + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, false); + + // 2. A single left outer join. + sqlQuery = "select e.name, d.depName " + + "from p.employees e left join p.departments d on e.depID = d.depID"; + programs = Programs.ofRules( + NullifyJoinRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, false); + + // 3. Two joins (left outer join + inner join) - for Rule 21. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + + "inner join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AssocOuterInnerRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 4. Two joins (inner join + left outer join) - for Rule 22. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.departments d " + + "inner join p.employees e on d.depID = e.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AssocInnerOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 5. Two joins (left outer join + inner join) - for Rule 23. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + + "inner join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomOuterInnerRule.INSTANCE, + JoinCommuteRule.INSTANCE, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 6. Two joins (inner join + left outer join) - for Rule 24. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "inner join p.departments d on e.depID = d.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomInnerOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + + // 7. Two joins (left outer join + left outer join) - for Rule 25. + sqlQuery = "select e.name, d.depName, c.cmpName " + + "from p.employees e " + + "left join p.departments d on e.depID = d.depID " + + "right join p.companies c on d.cmpID = c.cmpID"; + programs = Programs.ofRules( + AsscomOuterOuterRule.INSTANCE, + JoinCommuteRule.SWAP_OUTER, + EnumerableRules.ENUMERABLE_PROJECT_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE); + buildAndTransformQuery(programs, sqlQuery, true); + } + + /** + * This method emulates the whole life cycle of a given SQL query: parse, validate build and + * transform. It will close the planner after usage. + * + * @param programs is the set of transformation rules to be used. + * @param sqlQuery is the original SQL query in its string representation. + * @param ignoreTypeCheck indicates whether type check should be turned off. + * @throws Exception when there is error during any step. + */ + private static void buildAndTransformQuery(final Program programs, + final String sqlQuery, final boolean ignoreTypeCheck) throws Exception { + // Builds the schema. + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus defaultSchema = rootSchema.add("p", new ReflectiveSchema(new People())); + + // Creates the planner. + final SqlParser.Config parserConfig = SqlParser.configBuilder().setLex(Lex.MYSQL).build(); + final FrameworkConfig config = Frameworks.newConfigBuilder() + .parserConfig(parserConfig) + .defaultSchema(defaultSchema) + .programs(programs) + .build(); + final Planner planner = Frameworks.getPlanner(config); + + System.out.println("============================ Start ============================"); + System.out.println("Transaction ID: " + count++ + "\n"); + + // Prints the original SQL query string. + System.out.println("Input query:"); + System.out.println(sqlQuery + "\n"); + if (ignoreTypeCheck) { + RelOptUtil.disableTypeCheck = true; + } + + // Parses, validates and builds the query. + final SqlNode parse = planner.parse(sqlQuery); + final SqlNode validate = planner.validate(parse); + final RelNode relNode = planner.rel(validate).rel; + System.out.println("Before transformation:"); + System.out.println(RelOptUtil.toString(relNode)); + + // Transforms the query. + RelTraitSet traitSet = relNode.getTraitSet().replace(EnumerableConvention.INSTANCE); + RelNode transformedNode = planner.transform(0, traitSet, relNode); + System.out.println("After transformation:"); + System.out.println(RelOptUtil.toString(transformedNode)); + if (ignoreTypeCheck) { + RelOptUtil.disableTypeCheck = false; + } + + // Converts the transformed relational expression back to SQL query string. + final RelToSqlConverter converter = new RelToSqlConverter(DEFAULT_DIALECT); + final SqlNode transformedSqlNode = converter.visitChild(0, transformedNode).asStatement(); + final String transformedSqlQuery = transformedSqlNode.toSqlString(DEFAULT_DIALECT).getSql(); + System.out.println("Output query:"); + System.out.println(transformedSqlQuery + "\n"); + System.out.println("============================= End =============================\n"); + + // Closes the planner. + planner.close(); + } + + /** + * Represents the database named company. */ + public static class People { + public final Employee[] employees = { + new Employee(10, 1, "Daniel"), + new Employee(20, 1, "Mark"), + new Employee(30, 2, "Smith"), + new Employee(40, 3, "Armstrong"), + new Employee(50, 2, "Gabriel"), + new Employee(60, 5, "Daniel"), + new Employee(70, 7, "Joe"), + new Employee(80, 2, "Kim"), + new Employee(90, 1, "Gino") + }; + + public final Department[] departments = { + new Department(1, "Engineering", 100), + new Department(2, "Finance", 100) + }; + + public final Company[] companies = { + new Company(100, "All Link Pte Ltd"), + new Company(200, "") + }; + } + + /** + * Represents the schema of the employee table. */ + public static class Employee { + public final int empID; + public final int depID; + public final String name; + + Employee(int empID, int depID, String name) { + this.empID = empID; + this.depID = depID; + this.name = name; + } + } + + /** + * Represents the schema of the department table. */ + public static class Department { + public final int depID; + public final String depName; + public final int cmpID; + + Department(int depID, String depName, int cmpID) { + this.depID = depID; + this.depName = depName; + this.cmpID = cmpID; + } + } + + /** + * Represents the schema of the company table. */ + public static class Company { + public final int cmpID; + public final String cmpName; + + Company(int cmpID, String cmpName) { + this.cmpID = cmpID; + this.cmpName = cmpName; + } + } + + private Runner() { + } +} + +// End Runner.java diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index bccb0f5f537a..ae7f0104d484 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -125,6 +125,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.function.Supplier; +import java.util.stream.Collectors; import javax.annotation.Nonnull; /** @@ -138,6 +139,11 @@ public abstract class RelOptUtil { public static final double EPSILON = 1.0e-5; + /** + * A feature flag to decide whether we should disable typeCheck temporarily. + */ + public static boolean disableTypeCheck = false; + @SuppressWarnings("Guava") @Deprecated // to be removed before 2.0 public static final com.google.common.base.Predicate @@ -348,7 +354,7 @@ public static boolean areRowTypesEqual( || type2.getSqlTypeName() == SqlTypeName.ANY) { continue; } - if (!type1.equals(type2)) { + if (!type1.equals(type2) && !disableTypeCheck) { return false; } } @@ -1046,6 +1052,34 @@ public static RexNode splitCorrelatedFilterCondition( filter.getCluster().getRexBuilder(), nonEquiList, true); } + /** + * Checks whether a given predicate is referring to any attribute in a given list. + * + * + * @param condition is the given predicate. + * @param fields is the given list of attributes. + * @return true if not referring to any attribute; false otherwise. + */ + public static boolean isNotReferringTo(RexNode condition, List fields) { + if (!(condition instanceof RexCall)) { + return false; + } + + // Converts to RexNode format. + Set set = fields.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toSet()); + + // Checks whether the set contains each attribute. + RexCall call = (RexCall) condition; + for (RexNode attribute: call.getOperands()) { + if (set.contains(attribute)) { + return false; + } + } + return true; + } + private static void splitJoinCondition( List sysFieldList, List inputs, diff --git a/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java b/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java new file mode 100644 index 000000000000..81a07d8446b0 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/core/BestMatch.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelInput; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.SingleRel; + +import java.util.List; + +/** + * BestMatch is a unary operation that is always at the top of a join tree + * to eliminate spurious tuples. + */ +public abstract class BestMatch extends SingleRel { + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a best-match operator. + * + * @param cluster Cluster this relational expression belongs to + * @param traits the traits of this rel + * @param input Input relational expression + */ + protected BestMatch( + RelOptCluster cluster, + RelTraitSet traits, + RelNode input) { + super(cluster, traits, input); + } + + /** + * Creates a best match by its input. + */ + protected BestMatch(RelInput input) { + this(input.getCluster(), input.getTraitSet(), input.getInput()); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public final RelNode copy(RelTraitSet traitSet, List inputs) { + return copy(traitSet, sole(inputs)); + } + + public abstract BestMatch copy(RelTraitSet traitSet, RelNode input); +} + +// End BestMatch.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java index 1f1e7bb75c6f..b1e39ff0a886 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java +++ b/core/src/main/java/org/apache/calcite/rel/core/JoinRelType.java @@ -42,6 +42,11 @@ public enum JoinRelType { */ FULL, + /** + * Outer-cartesian join. + */ + OUTER_CARTESIAN, + /** * Semi-join. * @@ -78,7 +83,7 @@ public enum JoinRelType { * right-hand side. */ public boolean generatesNullsOnRight() { - return (this == LEFT) || (this == FULL); + return (this == LEFT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -86,7 +91,7 @@ public boolean generatesNullsOnRight() { * left-hand side. */ public boolean generatesNullsOnLeft() { - return (this == RIGHT) || (this == FULL); + return (this == RIGHT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -94,7 +99,7 @@ public boolean generatesNullsOnLeft() { * generate NULL values, either on the left-hand side or right-hand side. */ public boolean isOuterJoin() { - return (this == LEFT) || (this == RIGHT) || (this == FULL); + return (this == LEFT) || (this == RIGHT) || (this == FULL) || (this == OUTER_CARTESIAN); } /** @@ -127,6 +132,7 @@ public boolean generatesNullsOn(int i) { * the left. */ public JoinRelType cancelNullsOnLeft() { switch (this) { + case OUTER_CARTESIAN: case RIGHT: return INNER; case FULL: @@ -140,6 +146,7 @@ public JoinRelType cancelNullsOnLeft() { * the right. */ public JoinRelType cancelNullsOnRight() { switch (this) { + case OUTER_CARTESIAN: case LEFT: return INNER; case FULL: @@ -152,6 +159,14 @@ public JoinRelType cancelNullsOnRight() { public boolean projectsRight() { return this != SEMI && this != ANTI; } + + public boolean isCommutativeAndAssociative() { + return this == INNER || this == OUTER_CARTESIAN; + } + + public boolean canApplyNullify() { + return this == INNER || this == LEFT || this == RIGHT; + } } // End JoinRelType.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/Nullify.java b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java new file mode 100644 index 000000000000..4aa296acf041 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/core/Nullify.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.core; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelInput; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.validate.SqlValidatorUtil; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Nullify is an unary operation that performs nullification to the given attribute + * list of the input relation based on the given predicate. + */ +public abstract class Nullify extends SingleRel { + //~ Instance fields -------------------------------------------------------- + + protected final RexNode predicate; + protected final ImmutableList attributes; + + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a nullification operator. + * + * @param cluster Cluster that this relational expression belongs to + * @param traits the traits of this rel + * @param child input relational expression + */ + protected Nullify( + RelOptCluster cluster, + RelTraitSet traits, + RelNode child, + RexNode predicate, + List attributes) { + super(cluster, traits, child); + this.predicate = predicate; + this.attributes = ImmutableList.copyOf(attributes); + } + + /** + * Creates a Nullify by parsing serialized output. + */ + protected Nullify(RelInput input) { + this(input.getCluster(), + input.getTraitSet(), + input.getInput(), + input.getExpression("condition"), + input.getExpressionList("exprs")); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public final RelNode copy(RelTraitSet traitSet, + List inputs) { + return copy(traitSet, sole(inputs), predicate, attributes); + } + + public abstract Nullify copy(RelTraitSet traitSet, RelNode input, + RexNode predicate, List attributes); + + @Override public List getChildExps() { + return ImmutableList.of(predicate); + } + + public RelWriter explainTerms(final RelWriter pw) { + return super.explainTerms(pw) + .item("predicate", predicate) + .item("attributes", attributes); + } + + public RexNode getPredicate() { + return predicate; + } + + public ImmutableList getAttributes() { + return ImmutableList.copyOf(attributes); + } + + @Override protected RelDataType deriveRowType() { + return SqlValidatorUtil.deriveNullifyRowType(input.getRowType(), attributes, + getCluster().getTypeFactory()); + } +} + +// End Nullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java index a02e187eb5b4..65087b72f533 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java +++ b/core/src/main/java/org/apache/calcite/rel/core/RelFactories.java @@ -26,6 +26,7 @@ import org.apache.calcite.rel.RelDistribution; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalBestMatch; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalExchange; import org.apache.calcite.rel.logical.LogicalFilter; @@ -33,6 +34,7 @@ import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.logical.LogicalMatch; import org.apache.calcite.rel.logical.LogicalMinus; +import org.apache.calcite.rel.logical.LogicalNullify; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalRepeatUnion; import org.apache.calcite.rel.logical.LogicalSnapshot; @@ -74,6 +76,12 @@ public class RelFactories { public static final FilterFactory DEFAULT_FILTER_FACTORY = new FilterFactoryImpl(); + public static final NullifyFactory DEFAULT_NULLIFY_FACTORY = + new NullifyFactoryImpl(); + + public static final BestMatchFactory DEFAULT_BEST_MATCH_FACTORY = + new BestMatchFactoryImpl(); + public static final JoinFactory DEFAULT_JOIN_FACTORY = new JoinFactoryImpl(); public static final CorrelateFactory DEFAULT_CORRELATE_FACTORY = @@ -121,6 +129,8 @@ public class RelFactories { RelBuilder.proto( Contexts.of(DEFAULT_PROJECT_FACTORY, DEFAULT_FILTER_FACTORY, + DEFAULT_NULLIFY_FACTORY, + DEFAULT_BEST_MATCH_FACTORY, DEFAULT_JOIN_FACTORY, DEFAULT_SORT_FACTORY, DEFAULT_EXCHANGE_FACTORY, @@ -335,6 +345,54 @@ public RelNode createFilter(RelNode input, RexNode condition, } } + /** + * Can create a {@link Nullify} of the appropriate type + * for this rule's calling convention. + */ + public interface NullifyFactory { + /** + * Creates a nullification operator. + */ + RelNode createNullify(RelNode input, RexNode predicate, + List attributes, + List fieldNames, + Set variablesSet); + } + + /** + * Implementation of {@link RelFactories.NullifyFactory} that + * returns a vanilla {@link Nullify}. + */ + private static class NullifyFactoryImpl implements NullifyFactory { + public RelNode createNullify(RelNode input, RexNode predicate, + List attributes, + List fieldNames, + Set variablesSet) { + return LogicalNullify.create(input, predicate, attributes, ImmutableSet.copyOf(variablesSet)); + } + } + + /** + * Can create a {@link BestMatch} of the appropriate type + * for this rule's calling convention. + */ + public interface BestMatchFactory { + /** + * Creates a best-match operator. + */ + RelNode createBestMatch(RelNode input, Set variablesSet); + } + + /** + * Implementation of {@link RelFactories.BestMatchFactory} that + * returns a vanilla {@link BestMatch}. + */ + private static class BestMatchFactoryImpl implements BestMatchFactory { + public RelNode createBestMatch(final RelNode input, final Set variablesSet) { + return LogicalBestMatch.create(input, ImmutableSet.copyOf(variablesSet)); + } + } + /** * Can create a join of the appropriate type for a rule's calling convention. * diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java new file mode 100644 index 000000000000..9a5945715f3c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalBestMatch.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistributionTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.metadata.RelMdCollation; +import org.apache.calcite.rel.metadata.RelMdDistribution; +import org.apache.calcite.rel.metadata.RelMetadataQuery; + +import com.google.common.collect.ImmutableSet; + +import java.util.Objects; + +/** + * Sub-class of {@link org.apache.calcite.rel.core.BestMatch} + * not targeted at any particular engine or calling convention. + */ +public class LogicalBestMatch extends BestMatch { + private final ImmutableSet variablesSet; + + //~ Constructors ----------------------------------------------------------- + + protected LogicalBestMatch( + RelOptCluster cluster, + RelTraitSet traits, + RelNode input, + ImmutableSet variablesSet) { + super(cluster, traits, input); + this.variablesSet = Objects.requireNonNull(variablesSet); + } + + /** Creates a LogicalBestMatch. */ + public static LogicalBestMatch create(final RelNode input, + ImmutableSet variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.filter(mq, input)) + .replaceIf(RelDistributionTraitDef.INSTANCE, + () -> RelMdDistribution.filter(mq, input)); + return new LogicalBestMatch(cluster, traitSet, input, variablesSet); + } + + @Override public BestMatch copy(final RelTraitSet traitSet, final RelNode input) { + return new LogicalBestMatch(getCluster(), traitSet, input, variablesSet); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } +} + +// End LogicalBestMatch.java diff --git a/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java b/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java new file mode 100644 index 000000000000..8d1eb91d0841 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/logical/LogicalNullify.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistributionTraitDef; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.metadata.RelMdCollation; +import org.apache.calcite.rel.metadata.RelMdDistribution; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Objects; + +/** + * Sub-class of {@link org.apache.calcite.rel.core.Nullify} + * not targeted at any particular engine or calling convention. + */ +public class LogicalNullify extends Nullify { + private final ImmutableSet variablesSet; + + //~ Constructors ----------------------------------------------------------- + + protected LogicalNullify( + RelOptCluster cluster, + RelTraitSet traits, + RelNode child, + RexNode predicate, + List attributes, + ImmutableSet variablesSet) { + super(cluster, traits, child, predicate, attributes); + this.variablesSet = Objects.requireNonNull(variablesSet); + } + + /** Creates a LogicalNullify. */ + public static LogicalNullify create(final RelNode input, RexNode condition, + List attributes, + ImmutableSet variablesSet) { + final RelOptCluster cluster = input.getCluster(); + final RelMetadataQuery mq = cluster.getMetadataQuery(); + final RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE) + .replaceIfs(RelCollationTraitDef.INSTANCE, + () -> RelMdCollation.filter(mq, input)) + .replaceIf(RelDistributionTraitDef.INSTANCE, + () -> RelMdDistribution.filter(mq, input)); + return new LogicalNullify(cluster, traitSet, input, condition, attributes, variablesSet); + } + + @Override public Nullify copy(RelTraitSet traitSet, RelNode input, + RexNode predicate, List attributes) { + return new LogicalNullify(getCluster(), traitSet, input, predicate, attributes, variablesSet); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public RelNode accept(RelShuttle shuttle) { + return shuttle.visit(this); + } + + @Override public RelWriter explainTerms(final RelWriter pw) { + return super.explainTerms(pw) + .itemIf("variablesSet", variablesSet, !variablesSet.isEmpty()); + } +} + +// End LogicalNullify.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java index 56206d5b78cd..b0c7b68fb467 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinAssociateRule.java @@ -31,6 +31,9 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mappings; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; import java.util.ArrayList; import java.util.List; @@ -48,6 +51,7 @@ */ public class JoinAssociateRule extends RelOptRule { //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); /** The singleton. */ public static final JoinAssociateRule INSTANCE = @@ -101,10 +105,18 @@ public void onMatch(final RelOptRuleCall call) { return; } - // If either join is not inner, we cannot proceed. - // (Is this too strict?) - if (topJoin.getJoinType() != JoinRelType.INNER - || bottomJoin.getJoinType() != JoinRelType.INNER) { + // Only proceeds if the 2 joins are both inner or both outer-cartesian product. + boolean isBothInner = topJoin.getJoinType() == JoinRelType.INNER + && bottomJoin.getJoinType() == JoinRelType.INNER; + boolean isBothOuterCartesian = topJoin.getJoinType() == JoinRelType.OUTER_CARTESIAN + && bottomJoin.getJoinType() == JoinRelType.OUTER_CARTESIAN; + JoinRelType newJoinType = null; + if (isBothInner) { + newJoinType = JoinRelType.INNER; + } else if (isBothOuterCartesian) { + newJoinType = JoinRelType.OUTER_CARTESIAN; + } else { + LOGGER.debug("Will not proceed since the 2 join types are not associative."); return; } @@ -141,7 +153,7 @@ public void onMatch(final RelOptRuleCall call) { final Join newBottomJoin = bottomJoin.copy(bottomJoin.getTraitSet(), newBottomCondition, relB, - relC, JoinRelType.INNER, false); + relC, newJoinType, false); // Condition for newTopJoin consists of pieces from bottomJoin and topJoin. // Field ordinals do not need to be changed. @@ -149,7 +161,7 @@ public void onMatch(final RelOptRuleCall call) { @SuppressWarnings("SuspiciousNameCombination") final Join newTopJoin = topJoin.copy(topJoin.getTraitSet(), newTopCondition, relA, - newBottomJoin, JoinRelType.INNER, false); + newBottomJoin, newJoinType, false); call.transformTo(newTopJoin); } diff --git a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java index 315a0cd50084..5ec8b537a7dc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/JoinCommuteRule.java @@ -114,7 +114,8 @@ public static RelNode swap(Join join, boolean swapOuterJoins) { public static RelNode swap(Join join, boolean swapOuterJoins, RelBuilder relBuilder) { final JoinRelType joinType = join.getJoinType(); - if (!swapOuterJoins && joinType != JoinRelType.INNER) { + if (!swapOuterJoins && !joinType.isCommutativeAndAssociative()) { + // Notice: outer cartesian product is commutative. return null; } final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java new file mode 100644 index 000000000000..6ef46ea07570 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomInnerOuterRule.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AsscomInnerOuterRule applies limited r-asscom property on inner join and outer join. + * + * Rule 24. + */ +public class AsscomInnerOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomInnerOuterRule INSTANCE = new AsscomInnerOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomInnerOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomInnerOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomInnerJoin = call.rel(2); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The bottom join is not an inner join."); + return; + } + + // Makes sure the join condition is referring to the correct set of fields. + int topLeftJoinLeft = topLeftJoin.getLeft().getRowType().getFieldCount(); + int bottomInnerJoinRight = bottomInnerJoin.getRight().getRowType().getFieldCount(); + List fields = topLeftJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + fields.subList(topLeftJoinLeft, fields.size() - bottomInnerJoinRight))) { + LOGGER.debug("Not a subset of attributes."); + return; + } + + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + int bottomInnerJoinLeft = bottomInnerJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topLeftJoinLeft, bottomInnerJoinLeft, bottomInnerJoinRight); + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + replacer.replace(topLeftJoin.getCondition(), 0), + topLeftJoin.getLeft(), + bottomInnerJoin.getRight(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomInnerJoin.copy( + bottomInnerJoin.getTraitSet(), + replacer.replace(bottomInnerJoin.getCondition(), 0), + newBottomLeftJoin, + bottomInnerJoin.getLeft(), + JoinRelType.LEFT, + bottomInnerJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = + bottomInnerJoin.getRight().getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex(); + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() + bottomRight; + } else { + newIndex = var.getIndex() - bottomLeft; + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } +} + +// End AsscomInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java new file mode 100644 index 000000000000..ae1f4d96bd1a --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterInnerRule.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; + +import org.slf4j.Logger; + +import java.util.List; + +/** + * AsscomOuterInnerRule applies limited r-asscom property on outer join and inner join. + * + * Rule 23. + */ +public class AsscomOuterInnerRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomOuterInnerRule INSTANCE = new AsscomOuterInnerRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomOuterInnerRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomOuterInnerRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topInnerJoin = call.rel(0); + Join bottomLeftJoin = call.rel(2); + + // Makes sure the join types match the rule. + if (topInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The top join is not an inner join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // Makes sure the join condition is referring to the correct set of fields. + int topInnerJoinLeft = topInnerJoin.getLeft().getRowType().getFieldCount(); + int bottomLeftJoinRight = bottomLeftJoin.getRight().getRowType().getFieldCount(); + List fields = topInnerJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topInnerJoin.getCondition(), + fields.subList(topInnerJoinLeft, fields.size() - bottomLeftJoinRight))) { + LOGGER.debug("Not a subset of attributes."); + return; + } + + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topInnerJoin.getCluster().getRexBuilder(); + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topInnerJoinLeft, bottomLeftJoinLeft, bottomLeftJoinRight); + + // The new operators. + final Join newBottomInnerJoin = topInnerJoin.copy( + topInnerJoin.getTraitSet(), + replacer.replace(topInnerJoin.getCondition(), bottomLeftJoinLeft), + topInnerJoin.getLeft(), + bottomLeftJoin.getRight(), + JoinRelType.INNER, + topInnerJoin.isSemiJoinDone()); + final Join newTopInnerJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + replacer.replace(bottomLeftJoin.getCondition(), 0), + bottomLeftJoin.getLeft(), + newBottomInnerJoin, + JoinRelType.INNER, + bottomLeftJoin.isSemiJoinDone()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); + call.transformTo(transformedNode); + } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex() + bottomLeft; + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() - topLeft; + } else { + newIndex = var.getIndex(); + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } +} + +// End AsscomOuterInnerRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java new file mode 100644 index 000000000000..a09ab9a3569d --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AsscomOuterOuterRule.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AsscomOuterOuterRule applies limited r-asscom property on outer join and outer join. + * + * Rule 25. + */ +public class AsscomOuterOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AsscomOuterOuterRule INSTANCE = new AsscomOuterOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AsscomOuterOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AsscomOuterOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomLeftJoin = call.rel(2); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // Makes sure the join condition is referring to the correct set of fields. + int topLeftJoinLeft = topLeftJoin.getLeft().getRowType().getFieldCount(); + int bottomLeftJoinRight = bottomLeftJoin.getRight().getRowType().getFieldCount(); + List fields = topLeftJoin.getRowType().getFieldList(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + fields.subList(topLeftJoinLeft, fields.size() - bottomLeftJoinRight))) { + LOGGER.debug("Not a subset of attributes."); + return; + } + + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + final VariableReplacer replacer = new VariableReplacer( + rexBuilder, topLeftJoinLeft, bottomLeftJoinLeft, bottomLeftJoinRight); + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + replacer.replace(topLeftJoin.getCondition(), 0), + topLeftJoin.getLeft(), + bottomLeftJoin.getRight(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + replacer.replace(bottomLeftJoin.getCondition(), 0), + newBottomLeftJoin, + bottomLeftJoin.getLeft(), + JoinRelType.LEFT, + bottomLeftJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = + bottomLeftJoin.getRight().getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomLeftJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + private final int topLeft; + private final int bottomLeft; + private final int bottomRight; + + VariableReplacer(RexBuilder rexBuilder, int topLeft, int bottomLeft, int bottomRight) { + this.rexBuilder = rexBuilder; + this.topLeft = topLeft; + this.bottomLeft = bottomLeft; + this.bottomRight = bottomRight; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Computes its index after transformation. + int newIndex; + if (var.getIndex() < topLeft) { + newIndex = var.getIndex(); + } else if (var.getIndex() < topLeft + bottomLeft) { + newIndex = var.getIndex() + bottomRight; + } else { + newIndex = var.getIndex() - bottomLeft; + } + + // Re-builds the attribute. + return rexBuilder.makeInputRef(var.getType(), newIndex - offset); + } else { + return rex; + } + } + } +} + +// End AsscomOuterOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java new file mode 100644 index 000000000000..8b1ca8ca3f96 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocInnerOuterRule.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; + +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * AssocInnerOuterRule applies limited associativity on inner join and outer join. + * + * Rule 22. + */ +public class AssocInnerOuterRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AssocInnerOuterRule INSTANCE = new AssocInnerOuterRule( + operand(Join.class, + operand(RelSubset.class, any()), + operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AssocInnerOuterRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AssocInnerOuterRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topLeftJoin = call.rel(0); + Join bottomInnerJoin = call.rel(2); + + // Makes sure the join types match the rule. + if (topLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The top join is not an left outer join."); + return; + } else if (bottomInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The bottom join is not a inner join."); + return; + } + + // Makes sure the join condition is referring to the correct set of fields. + int bottomInnerJoinRight = bottomInnerJoin.getRight().getRowType().getFieldCount(); + int total = topLeftJoin.getRowType().getFieldCount(); + if (!RelOptUtil.isNotReferringTo(topLeftJoin.getCondition(), + topLeftJoin.getRowType().getFieldList().subList(total - bottomInnerJoinRight, total))) { + LOGGER.debug("Not a subset of attributes."); + return; + } + + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topLeftJoin.getCluster().getRexBuilder(); + final VariableReplacer replacer = new VariableReplacer(rexBuilder); + + // The new operators. + final Join newBottomLeftJoin = topLeftJoin.copy( + topLeftJoin.getTraitSet(), + replacer.replace(topLeftJoin.getCondition(), 0), + topLeftJoin.getLeft(), + bottomInnerJoin.getLeft(), + JoinRelType.LEFT, + topLeftJoin.isSemiJoinDone()); + final Join newTopLeftJoin = bottomInnerJoin.copy( + bottomInnerJoin.getTraitSet(), + replacer.replace(bottomInnerJoin.getCondition(), 0), + newBottomLeftJoin, + bottomInnerJoin.getRight(), + JoinRelType.LEFT, + bottomInnerJoin.isSemiJoinDone()); + + // Determines the nullification attribute. + List nullifyFieldList = + bottomInnerJoin.getLeft().getRowType().getFieldList(); + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopLeftJoin) + .nullify(bottomInnerJoin.getCondition(), nullificationList).bestMatch().build(); + call.transformTo(transformedNode); + } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + + VariableReplacer(RexBuilder rexBuilder) { + this.rexBuilder = rexBuilder; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Re-builds the attribute. + int newIndex = var.getIndex() - offset; + return rexBuilder.makeInputRef(var.getType(), newIndex); + } else { + return rex; + } + } + } +} + +// End AssocInnerOuterRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java new file mode 100644 index 000000000000..6bf63eacdb18 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/AssocOuterInnerRule.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; + +import org.slf4j.Logger; + +/** + * AssocOuterInnerRule applies limited associativity on outer join and inner join. + * + * Rule 21. + */ +public class AssocOuterInnerRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final AssocOuterInnerRule INSTANCE = new AssocOuterInnerRule( + operand(Join.class, + operand(Join.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public AssocOuterInnerRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public AssocOuterInnerRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the two original join operators. + Join topInnerJoin = call.rel(0); + Join bottomLeftJoin = call.rel(1); + + // Makes sure the join types match the rule. + if (topInnerJoin.getJoinType() != JoinRelType.INNER) { + LOGGER.debug("The top join is not an inner join."); + return; + } else if (bottomLeftJoin.getJoinType() != JoinRelType.LEFT) { + LOGGER.debug("The bottom join is not a left outer join."); + return; + } + + // Makes sure the join condition is referring to the correct set of fields. + int bottomLeftJoinLeft = bottomLeftJoin.getLeft().getRowType().getFieldCount(); + if (!RelOptUtil.isNotReferringTo(topInnerJoin.getCondition(), + topInnerJoin.getRowType().getFieldList().subList(0, bottomLeftJoinLeft))) { + LOGGER.debug("Not a subset of attributes."); + return; + } + + // Replaces the variables in the predicates later. + final RexBuilder rexBuilder = topInnerJoin.getCluster().getRexBuilder(); + final VariableReplacer replacer = new VariableReplacer(rexBuilder); + + // The new operators. + final Join newBottomInnerJoin = topInnerJoin.copy( + topInnerJoin.getTraitSet(), + replacer.replace(topInnerJoin.getCondition(), bottomLeftJoinLeft), + bottomLeftJoin.getRight(), + topInnerJoin.getRight(), + JoinRelType.INNER, + topInnerJoin.isSemiJoinDone()); + final Join newTopInnerJoin = bottomLeftJoin.copy( + bottomLeftJoin.getTraitSet(), + replacer.replace(bottomLeftJoin.getCondition(), 0), + bottomLeftJoin.getLeft(), + newBottomInnerJoin, + JoinRelType.INNER, + bottomLeftJoin.isSemiJoinDone()); + + // Builds the transformed relational tree. + final RelNode transformedNode = call.builder().push(newTopInnerJoin).build(); + call.transformTo(transformedNode); + } + + /** + * A utility inner class to replace the index of attributes. + */ + private static class VariableReplacer { + private final RexBuilder rexBuilder; + + VariableReplacer(RexBuilder rexBuilder) { + this.rexBuilder = rexBuilder; + } + + RexNode replace(RexNode rex, int offset) { + if (rex instanceof RexCall) { + final RexCall call = (RexCall) rex; + + // Converts each operand in the predicate. + ImmutableList.Builder builder = ImmutableList.builder(); + call.operands.forEach(operand -> builder.add(replace(operand, offset))); + + // Re-builds the predicate. + return call.clone(call.getType(), builder.build()); + } else if (rex instanceof RexInputRef) { + final RexInputRef var = (RexInputRef) rex; + + // Re-builds the attribute. + int newIndex = var.getIndex() - offset; + return rexBuilder.makeInputRef(var.getType(), newIndex); + } else { + return rex; + } + } + } +} + +// End AssocOuterInnerRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java new file mode 100644 index 000000000000..ac2efa189a6f --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchOverNullifyRule.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * BestMatchOverNullifyRule eliminates all inner best-match operators (sandwiched + * by a nullification operator) as long as there is a best-match operator at the + * very end. It will check whether the nullification predicate is null-intolerant. + * The conversion is from `B(Nullify(B(R)))` to `B(Nullify(R))`. + */ +public class BestMatchOverNullifyRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the current rule. */ + public static final BestMatchOverNullifyRule INSTANCE = new BestMatchOverNullifyRule( + operand(BestMatch.class, + operand(Nullify.class, + operand(BestMatch.class, any()))), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchOverNullifyRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchOverNullifyRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + RelBuilder builder = call.builder(); + + // Gets the old nullification operator. + Nullify nullify = call.rel(1); + RexNode oldPredicate = nullify.getPredicate(); + List oldAttributes = nullify.getAttributes(); + + // Makes sure the nullification predicate is null-intolerant (cannot evaluate to TRUE). + if (valueForNull(oldPredicate) == 1) { + throw new AssertionError("The nullification predicate is not null-intolerant."); + } + + // Gets the base relation. + BestMatch innerBestMatch = call.rel(2); + RelNode base = innerBestMatch.getInput(); + + // Constructs the new expression. + RelNode newNode = builder.push(base) + .nullify(oldPredicate, oldAttributes) + .bestMatch().build(); + call.transformTo(newNode); + } + + /** + * Evaluates the value of a given predicate when referring to null values. + * + * @param predicate is the predicate to be tested. + * @return 1 if the predicate evaluates to TRUE, 0 if UNKNOWN, -1 if false. + */ + private int valueForNull(RexNode predicate) { + if (predicate.isA(SqlKind.AND)) { + RexCall call = (RexCall) predicate; + boolean isAllTrue = true; + boolean hasAtLeastOneFalse = false; + + // Iterates through each child. + for (RexNode child: call.getOperands()) { + int childValue = valueForNull(child); + if (childValue != 1) { + isAllTrue = false; + } + if (childValue == -1) { + hasAtLeastOneFalse = true; + } + } + + return isAllTrue ? 1 : (hasAtLeastOneFalse ? -1 : 0); + } else if (predicate.isA(SqlKind.OR)) { + RexCall call = (RexCall) predicate; + boolean isAllFalse = true; + boolean hasAtLeastOneTrue = false; + + // Iterates through each child. + for (RexNode child: call.getOperands()) { + int childValue = valueForNull(child); + if (childValue != -1) { + isAllFalse = false; + } + if (childValue == 1) { + hasAtLeastOneTrue = true; + } + } + + return isAllFalse ? -1 : (hasAtLeastOneTrue ? 1 : 0); + } else if (predicate.isA(SqlKind.NOT)) { + RexCall call = (RexCall) predicate; + RexNode child = call.getOperands().get(0); + int childValue = valueForNull(child); + return childValue == 0 ? 0 : -childValue; + } else if (predicate.isA(ImmutableList.of(SqlKind.IS_NULL, SqlKind.IS_TRUE))) { + return 1; + } else if (predicate.isA(ImmutableList.of(SqlKind.IS_NULL, SqlKind.IS_TRUE))) { + return -1; + } else { + return 0; + } + } +} + +// End BestMatchOverNullifyRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java new file mode 100644 index 000000000000..37d63c4f24b2 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchPullUpRule.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +/** + * BestMatchPullUpRule pulls up a best-match operator. Basically, the + * conversion is from `B(R) * S` to `B(R * S)`. + */ +public class BestMatchPullUpRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final BestMatchPullUpRule INSTANCE = new BestMatchPullUpRule( + operand(Join.class, + operand(BestMatch.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchPullUpRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchPullUpRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + final RelBuilder builder = call.builder(); + + // Only applies the rule on cartesian product. + final Join join = call.rel(0); + final RexNode condition = join.getCondition(); + if (!condition.equals(builder.literal(true))) { + LOGGER.debug("The condition is not true"); + return; + } + + // Constructs the new cartesian product. + final BestMatch bestMatch = call.rel(1); + final RelNode outerCartesianJoin = + join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + bestMatch.getInput(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + + // Builds the new expression. + RelNode reducedNode = builder.push(outerCartesianJoin).bestMatch().build(); + call.transformTo(reducedNode); + } +} + +// End BestMatchPullUpRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java new file mode 100644 index 000000000000..7e7081c12ae2 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/BestMatchReduceRule.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.BestMatch; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.tools.RelBuilderFactory; + +/** + * BestMatchReduceRule is able to reduce two consecutive best-match operators + * into one. In the current implementation, the outer one will be eliminated. + * + * There is no reverse rule for this rule because NullifyJoinReverseRule will + * assume an appropriate best-match operator exists. + */ +public class BestMatchReduceRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** Instance of the rule that reduces two best-match operators. */ + public static final BestMatchReduceRule INSTANCE = new BestMatchReduceRule( + operand(BestMatch.class, operand(BestMatch.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public BestMatchReduceRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public BestMatchReduceRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // Gets the inner best-match operator. + BestMatch innerBestMatch = call.rel(1); + + // Eliminates the outer one. + RelNode reducedNode = call.builder().push(innerBestMatch).build(); + call.transformTo(reducedNode); + } +} + +// End BestMatchReduceRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java new file mode 100644 index 000000000000..7fb6f57ad3f6 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinReverseRule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; + +/** + * NullifyJoinReverseRule converts a nullified join (i.e., nullification operator + * + outer cartesian product) backs to either a 1-sided outer join or inner join. + */ +public class NullifyJoinReverseRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the rule that reverses the nullification process. */ + public static final NullifyJoinReverseRule INSTANCE = + new NullifyJoinReverseRule(operand(Nullify.class, operand(Join.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public NullifyJoinReverseRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyJoinReverseRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + // The nullification operator at the top. + final Nullify nullify = call.rel(0); + + // The join operator at the bottom. + final Join join = call.rel(1); + if (join.getJoinType() != JoinRelType.OUTER_CARTESIAN) { + LOGGER.debug("Should only be applied when the join is an outer cartesian product"); + return; + } + + // Checks which join type should be converted back to. + final List joinFieldList = join.getRowType().getFieldList(); + } +} + +// End NullifyJoinReverseRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java new file mode 100644 index 000000000000..5a185b64eca9 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyJoinRule.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import org.slf4j.Logger; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +/** + * NullifyJoinRule nullifies a 1-sided outer join or inner join operator in the + * following way: + * 1) 1-sided outer join: nullify the null-producing side; + * 2) inner join: nullify both sides; + */ +public class NullifyJoinRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the rule that nullifies inner, left outer or right outer join. */ + public static final NullifyJoinRule INSTANCE = + new NullifyJoinRule(operand(Join.class, any()), null); + + /** + * A map from table name to a set of predicates, which becomes the relation's + * nullification set. This map shall only be filled once. + */ + private static final Map> NULLIFICATION_SET_MAP = new HashMap<>(); + + /** + * A boolean flag to indicate whether we have filled in the above map. + */ + private static boolean hasFilledMap = false; + + //~ Constructors ----------------------------------------------------------- + + public NullifyJoinRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyJoinRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + if (!hasFilledMap) { + fillNullificationSetMap(call.getPlanner().getRoot()); + hasFilledMap = true; + } + + RelBuilder builder = call.builder(); + + // The join operator at the current node. + final Join join = call.rel(0); + final JoinRelType joinType = join.getJoinType(); + if (!joinType.canApplyNullify()) { + LOGGER.debug("Invalid join relation type for nullify: " + joinType.toString()); + return; + } + + // The new join operator (as outer-cartesian product). + final RelNode outerCartesianJoin = join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + join.getLeft(), + join.getRight(), + JoinRelType.OUTER_CARTESIAN, + join.isSemiJoinDone()); + + // Determines the nullification attribute list based on the join type. + List joinFieldList = outerCartesianJoin.getRowType().getFieldList(); + List nullifyFieldList; + int leftFieldCount = join.getLeft().getRowType().getFieldCount(); + switch (joinType) { + case LEFT: + nullifyFieldList = joinFieldList.subList(leftFieldCount, joinFieldList.size()); + break; + case RIGHT: + nullifyFieldList = joinFieldList.subList(0, leftFieldCount); + break; + case INNER: + nullifyFieldList = joinFieldList; + break; + default: + throw new AssertionError(joinType); + } + List nullificationList = nullifyFieldList.stream() + .map(field -> new RexInputRef(field.getIndex(), field.getType())) + .collect(Collectors.toList()); + + // Determines the nullification condition. + RexNode nullificationCondition = join.getCondition(); + + // Builds the transformed relational tree. + final RelNode transformedNode = builder.push(outerCartesianJoin) + .nullify(nullificationCondition, nullificationList) + .bestMatch() + .build(); + call.transformTo(transformedNode); + } + + /** + * Fills the nullification set map of this query. + * + * @param root is the root node of the query. + * @return a list consisting of the full names of all tables under this node. + */ + private static List fillNullificationSetMap(RelNode root) { + if (root instanceof TableScan) { + TableScan tableScan = (TableScan) root; + String tableName = getTableFullName(tableScan); + + // Initializes the nullification set of the base relation to be an empty set. + NULLIFICATION_SET_MAP.put(tableName, new HashSet<>()); + + // Returns this base relation's full name. + return ImmutableList.of(tableName); + } else if (root instanceof RelSubset) { + RelSubset relSubset = (RelSubset) root; + return fillNullificationSetMap(relSubset.getOriginal()); + } else if (root instanceof Join) { + Join join = (Join) root; + + // Traverses its left and right children first (the algorithm requires postfix order). + List leftNames = fillNullificationSetMap(join.getLeft()); + List rightNames = fillNullificationSetMap(join.getRight()); + + // The current join predicate will always be added. + Set toAdd = new HashSet<>(); + toAdd.add(join.getCondition()); + + // Populates the nullification set. + switch (join.getJoinType()) { + case LEFT: + // Prepares all the predicates that need to be added. + for (String name: leftNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set. + for (String name: rightNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + break; + case RIGHT: + // Prepares all the predicates that need to be added. + for (String name: rightNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set. + for (String name: leftNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + break; + case INNER: + // Prepares all the predicates that need to be added. + for (String name: leftNames) { + toAdd.addAll(NULLIFICATION_SET_MAP.get(name)); + } + Set toAdd2 = new HashSet<>(); + toAdd2.add(join.getCondition()); + for (String name: rightNames) { + toAdd2.addAll(NULLIFICATION_SET_MAP.get(name)); + } + + // Updates the nullification set for both sides. + for (String name: rightNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd); + } + for (String name: leftNames) { + Set current = NULLIFICATION_SET_MAP.get(name); + current.addAll(toAdd2); + } + break; + default: + throw new AssertionError("Unsupported join type: " + join.getJoinType()); + } + + // Merges the table names from both sides together. + return ImmutableList.copyOf(Iterables.concat(leftNames, rightNames)); + } else { + return fillNullificationSetMap(root.getInput(0)); + } + } + + private static String getTableFullName(TableScan tableScan) { + StringJoiner joiner = new StringJoiner("."); + for (String name: tableScan.getTable().getQualifiedName()) { + joiner.add(name); + } + return joiner.toString(); + } +} + +// End NullifyJoinRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java new file mode 100644 index 000000000000..419f75ce096b --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/NullifyPullUpRule.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules.custom; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Nullify; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.trace.CalciteTrace; + +import org.slf4j.Logger; + +import java.util.List; + +/** + * NullifyPullUpRule pulls up a nullification operator. Basically, the + * conversion is from `Nullify(R) * S` to `Nullify(R * S)`. + */ +public class NullifyPullUpRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + private static final Logger LOGGER = CalciteTrace.getPlannerTracer(); + + /** Instance of the current rule. */ + public static final NullifyPullUpRule INSTANCE = new NullifyPullUpRule( + operand(Join.class, + operand(Nullify.class, any()), + operand(RelSubset.class, any())), null); + + //~ Constructors ----------------------------------------------------------- + + public NullifyPullUpRule(RelOptRuleOperand operand, + String description, RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, description); + } + + public NullifyPullUpRule(RelOptRuleOperand operand, String description) { + this(operand, description, RelFactories.LOGICAL_BUILDER); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public void onMatch(final RelOptRuleCall call) { + final RelBuilder builder = call.builder(); + + // Only applies the rule on cartesian product. + final Join join = call.rel(0); + final RexNode condition = join.getCondition(); + if (!condition.equals(builder.literal(true))) { + LOGGER.debug("The condition is not true"); + return; + } + + // Constructs the new cartesian product. + final Nullify oldNullify = call.rel(1); + final RelNode outerCartesianJoin = + join.copy( + join.getTraitSet(), + builder.literal(true), // Uses a literal condition which is always true. + oldNullify.getInput(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone()); + + // Builds the new expression. + final RexNode oldPredicate = oldNullify.getPredicate(); + final List oldAttributes = oldNullify.getAttributes(); + RelNode reducedNode = builder.push(outerCartesianJoin) + .nullify(oldPredicate, oldAttributes).build(); + call.transformTo(reducedNode); + } +} + +// End NullifyPullUpRule.java diff --git a/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java b/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java new file mode 100644 index 000000000000..28eedf25575e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/custom/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides a customized set of planner rules, primarily used for outer join + * reorder-ability research. + */ +package org.apache.calcite.rel.rules.custom; + +// End package-info.java diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java index c5d7316e90b8..9d90636c16f8 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorUtil.java @@ -31,6 +31,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.CustomColumnResolvingTable; import org.apache.calcite.schema.ExtensibleTable; import org.apache.calcite.schema.Table; @@ -65,6 +66,7 @@ import com.google.common.collect.Lists; import java.nio.charset.Charset; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -464,6 +466,34 @@ public static List uniquify( : newNameList; } + /** + * Derives the type of a nullification operator. + * + * @param inputType is the row type of its input. + * @param attributes is the list of nullification attributes. + * @param typeFactory is the type factory. + * @return the row type. + */ + public static RelDataType deriveNullifyRowType( + RelDataType inputType, + List attributes, + RelDataTypeFactory typeFactory) { + return typeFactory.createStructType(inputType.getStructKind(), new AbstractList() { + @Override public RelDataType get(final int index) { + RelDataType fieldType = inputType.getFieldList().get(index).getType(); + boolean isNullable = attributes.stream().anyMatch(attr -> attr.hashCode() == index); + if (isNullable) { + return typeFactory.createTypeWithNullability(fieldType, true); + } + return fieldType; + } + + @Override public int size() { + return inputType.getFieldCount(); + } + }, inputType.getFieldNames()); + } + /** * Derives the type of a join relational expression. * diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index a86a96c47739..c6cdc02c7e83 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -141,6 +141,8 @@ public class RelBuilder { protected final RelOptCluster cluster; protected final RelOptSchema relOptSchema; private final RelFactories.FilterFactory filterFactory; + private final RelFactories.NullifyFactory nullifyFactory; + private final RelFactories.BestMatchFactory bestMatchFactory; private final RelFactories.ProjectFactory projectFactory; private final RelFactories.AggregateFactory aggregateFactory; private final RelFactories.SortFactory sortFactory; @@ -174,6 +176,12 @@ protected RelBuilder(Context context, RelOptCluster cluster, this.filterFactory = Util.first(context.unwrap(RelFactories.FilterFactory.class), RelFactories.DEFAULT_FILTER_FACTORY); + this.nullifyFactory = + Util.first(context.unwrap(RelFactories.NullifyFactory.class), + RelFactories.DEFAULT_NULLIFY_FACTORY); + this.bestMatchFactory = + Util.first(context.unwrap(RelFactories.BestMatchFactory.class), + RelFactories.DEFAULT_BEST_MATCH_FACTORY); this.projectFactory = Util.first(context.unwrap(RelFactories.ProjectFactory.class), RelFactories.DEFAULT_PROJECT_FACTORY); @@ -1224,6 +1232,57 @@ public RelBuilder filter(Iterable variablesSet, return this; } + /** Creates a gamma of the given list of + * fields. */ + public RelBuilder gamma(RexNode... nodes) { + return gamma(ImmutableList.copyOf(nodes)); + } + + public RelBuilder gamma(Iterable nodes) { + final List nodeList = Lists.newArrayList(nodes); + final List checkNullList = nodeList + .stream() + .map(node -> call(SqlStdOperatorTable.IS_NULL, node)) + .collect(Collectors.toList()); + return filter(and(checkNullList)); + } + + public RelBuilder nullify(RexNode predicate, RexNode... nodes) { + return nullify(predicate, ImmutableList.copyOf(nodes)); + } + + public RelBuilder nullify(RexNode predicate, Iterable nodes) { + return nullify(predicate, nodes, ImmutableList.of()); + } + + public RelBuilder nullify(RexNode predicate, Iterable nodes, + Iterable fieldNames) { + return nullify(ImmutableSet.of(), predicate, nodes, fieldNames); + } + + public RelBuilder nullify(Set variablesSet, RexNode predicate, + Iterable nodes, Iterable fieldNames) { + final Frame frame = stack.pop(); + final List nodesList = Lists.newArrayList(nodes); + final List fieldNamesList = Lists.newArrayList(fieldNames); + + final RelNode nullify = nullifyFactory.createNullify(frame.rel, predicate, + nodesList, fieldNamesList, variablesSet); + stack.push(new Frame(nullify, frame.fields)); + return this; + } + + public RelBuilder bestMatch() { + return bestMatch(ImmutableSet.of()); + } + + public RelBuilder bestMatch(Set variablesSet) { + final Frame frame = stack.pop(); + final RelNode bestMatch = bestMatchFactory.createBestMatch(frame.rel, variablesSet); + stack.push(new Frame(bestMatch, frame.fields)); + return this; + } + /** Creates a {@link Project} of the given * expressions. */ public RelBuilder project(RexNode... nodes) {