[CALCITE-2914] Add a new statistic provider, to improve how LatticeSuggester deduces foreign keys

Statistic provider now generates SQL statements to look at a join
condition and figure out whether either side is a unique key, and
whether the other side is a foreign key (i.e. does an anti-join to
verify referential integrity).

Create new package org.apache.calcite.statistic, and move some
existing classes such as MapSqlStatisticProvider into it.

In JDBC adapter, when generating SQL for JDBC tables, use the foreign
catalog, schema and table name.

In Frameworks, use a query provider with a 30 minute, 1,000 element
cache, rather than map provider as default provider.

In LatticeSuggesterTest we continue to use a MapSqlStatisticProvider,
for performance reasons.

Fix deprecated calls to AggregateCall.create added in [CALCITE-1172].

Close #1141

Author: julianhyde

core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcRules.java

@@ -18,6 +18,7 @@
 
 import org.apache.calcite.linq4j.Queryable;
 import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.plan.Contexts;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptCost;
@@ -60,12 +61,15 @@
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.schema.ModifiableTable;
 import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.SqlDialect;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.tools.RelBuilderFactory;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Util;
@@ -91,6 +95,126 @@ private JdbcRules() {
 
   protected static final Logger LOGGER = CalciteTrace.getPlannerTracer();
 
+  static final RelFactories.ProjectFactory PROJECT_FACTORY =
+      (input, projects, fieldNames) -> {
+        final RelOptCluster cluster = input.getCluster();
+        final RelDataType rowType =
+            RexUtil.createStructType(cluster.getTypeFactory(), projects,
+                fieldNames, SqlValidatorUtil.F_SUGGESTER);
+        return new JdbcProject(cluster, input.getTraitSet(), input, projects,
+            rowType);
+      };
+
+  static final RelFactories.FilterFactory FILTER_FACTORY =
+      (input, condition) -> new JdbcRules.JdbcFilter(input.getCluster(),
+          input.getTraitSet(), input, condition);
+
+  static final RelFactories.JoinFactory JOIN_FACTORY =
+      (left, right, condition, variablesSet, joinType, semiJoinDone) -> {
+        final RelOptCluster cluster = left.getCluster();
+        final RelTraitSet traitSet = cluster.traitSetOf(left.getConvention());
+        try {
+          return new JdbcJoin(cluster, traitSet, left, right, condition,
+              variablesSet, joinType);
+        } catch (InvalidRelException e) {
+          throw new AssertionError(e);
+        }
+      };
+
+  static final RelFactories.CorrelateFactory CORRELATE_FACTORY =
+      (left, right, correlationId, requiredColumns, joinType) -> {
+        throw new UnsupportedOperationException("JdbcCorrelate");
+      };
+
+  public static final RelFactories.SemiJoinFactory SEMI_JOIN_FACTORY =
+      (left, right, condition) -> {
+        throw new UnsupportedOperationException("JdbcSemiJoin");
+      };
+
+  public static final RelFactories.SortFactory SORT_FACTORY =
+      (input, collation, offset, fetch) -> {
+        throw new UnsupportedOperationException("JdbcSort");
+      };
+
+  public static final RelFactories.ExchangeFactory EXCHANGE_FACTORY =
+      (input, distribution) -> {
+        throw new UnsupportedOperationException("JdbcExchange");
+      };
+
+  public static final RelFactories.SortExchangeFactory SORT_EXCHANGE_FACTORY =
+      (input, distribution, collation) -> {
+        throw new UnsupportedOperationException("JdbcSortExchange");
+      };
+
+  public static final RelFactories.AggregateFactory AGGREGATE_FACTORY =
+      (input, indicator, groupSet, groupSets, aggCalls) -> {
+        final RelOptCluster cluster = input.getCluster();
+        final RelTraitSet traitSet = cluster.traitSetOf(input.getConvention());
+        try {
+          return new JdbcAggregate(cluster, traitSet, input, false, groupSet,
+              groupSets, aggCalls);
+        } catch (InvalidRelException e) {
+          throw new AssertionError(e);
+        }
+      };
+
+  public static final RelFactories.MatchFactory MATCH_FACTORY =
+      (input, pattern, rowType, strictStart, strictEnd, patternDefinitions,
+          measures, after, subsets, allRows, partitionKeys, orderKeys,
+          interval) -> {
+        throw new UnsupportedOperationException("JdbcMatch");
+      };
+
+  public static final RelFactories.SetOpFactory SET_OP_FACTORY =
+      (kind, inputs, all) -> {
+        RelNode input = inputs.get(0);
+        RelOptCluster cluster = input.getCluster();
+        final RelTraitSet traitSet = cluster.traitSetOf(input.getConvention());
+        switch (kind) {
+        case UNION:
+          return new JdbcUnion(cluster, traitSet, inputs, all);
+        case INTERSECT:
+          return new JdbcIntersect(cluster, traitSet, inputs, all);
+        case EXCEPT:
+          return new JdbcMinus(cluster, traitSet, inputs, all);
+        default:
+          throw new AssertionError("unknown: " + kind);
+        }
+      };
+
+  public static final RelFactories.ValuesFactory VALUES_FACTORY =
+      (cluster, rowType, tuples) -> {
+        throw new UnsupportedOperationException();
+      };
+
+  public static final RelFactories.TableScanFactory TABLE_SCAN_FACTORY =
+      (cluster, table) -> {
+        throw new UnsupportedOperationException();
+      };
+
+  public static final RelFactories.SnapshotFactory SNAPSHOT_FACTORY =
+      (input, period) -> {
+        throw new UnsupportedOperationException();
+      };
+
+  /** A {@link RelBuilderFactory} that creates a {@link RelBuilder} that will
+   * create JDBC relational expressions for everything. */
+  public static final RelBuilderFactory JDBC_BUILDER =
+      RelBuilder.proto(
+          Contexts.of(PROJECT_FACTORY,
+              FILTER_FACTORY,
+              JOIN_FACTORY,
+              SEMI_JOIN_FACTORY,
+              SORT_FACTORY,
+              EXCHANGE_FACTORY,
+              SORT_EXCHANGE_FACTORY,
+              AGGREGATE_FACTORY,
+              MATCH_FACTORY,
+              SET_OP_FACTORY,
+              VALUES_FACTORY,
+              TABLE_SCAN_FACTORY,
+              SNAPSHOT_FACTORY));
+
   public static List<RelOptRule> rules(JdbcConvention out) {
     return rules(out, RelFactories.LOGICAL_BUILDER);
   }

core/src/main/java/org/apache/calcite/adapter/jdbc/JdbcTable.java

@@ -75,19 +75,20 @@
 public class JdbcTable extends AbstractQueryableTable
     implements TranslatableTable, ScannableTable, ModifiableTable {
   private RelProtoDataType protoRowType;
-  private final JdbcSchema jdbcSchema;
-  private final String jdbcCatalogName;
-  private final String jdbcSchemaName;
-  private final String jdbcTableName;
-  private final Schema.TableType jdbcTableType;
+  public final JdbcSchema jdbcSchema;
+  public final String jdbcCatalogName;
+  public final String jdbcSchemaName;
+  public final String jdbcTableName;
+  public final Schema.TableType jdbcTableType;
 
   JdbcTable(JdbcSchema jdbcSchema, String jdbcCatalogName,
-      String jdbcSchemaName, String tableName, Schema.TableType jdbcTableType) {
+      String jdbcSchemaName, String jdbcTableName,
+      Schema.TableType jdbcTableType) {
     super(Object[].class);
-    this.jdbcSchema = jdbcSchema;
+    this.jdbcSchema = Objects.requireNonNull(jdbcSchema);
     this.jdbcCatalogName = jdbcCatalogName;
     this.jdbcSchemaName = jdbcSchemaName;
-    this.jdbcTableName = tableName;
+    this.jdbcTableName = Objects.requireNonNull(jdbcTableName);
     this.jdbcTableType = Objects.requireNonNull(jdbcTableType);
   }
 
@@ -142,16 +143,18 @@ SqlString generateSql() {
     return writer.toSqlString();
   }
 
-  SqlIdentifier tableName() {
-    final List<String> strings = new ArrayList<>();
+  /** Returns the table name, qualified with catalog and schema name if
+   * applicable, as a parse tree node ({@link SqlIdentifier}). */
+  public SqlIdentifier tableName() {
+    final List<String> names = new ArrayList<>(3);
     if (jdbcSchema.catalog != null) {
-      strings.add(jdbcSchema.catalog);
+      names.add(jdbcSchema.catalog);
     }
     if (jdbcSchema.schema != null) {
-      strings.add(jdbcSchema.schema);
+      names.add(jdbcSchema.schema);
     }
-    strings.add(jdbcTableName);
-    return new SqlIdentifier(strings, SqlParserPos.ZERO);
+    names.add(jdbcTableName);
+    return new SqlIdentifier(names, SqlParserPos.ZERO);
   }
 
   public RelNode toRel(RelOptTable.ToRelContext context,

core/src/main/java/org/apache/calcite/materialize/Lattice.java

@@ -47,6 +47,7 @@
 import org.apache.calcite.sql.SqlUtil;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.statistic.MapSqlStatisticProvider;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Litmus;
 import org.apache.calcite.util.Pair;

core/src/main/java/org/apache/calcite/materialize/MapSqlStatisticProvider.java

@@ -16,16 +16,48 @@
  */
 package org.apache.calcite.materialize;
 
+import org.apache.calcite.plan.RelOptTable;
+
 import java.util.List;
 
 /**
- * Estimates row counts for tables and columns.
+ * Estimates row counts for tables and columns, and whether combinations of
+ * columns form primary/unique and foreign keys.
  *
  * <p>Unlike {@link LatticeStatisticProvider}, works on raw tables and columns
  * and does not need a {@link Lattice}.
+ *
+ * <p>It uses {@link org.apache.calcite.plan.RelOptTable} because that contains
+ * enough information to generate and execute SQL, while not being tied to a
+ * lattice.
+ *
+ * <p>The main implementation,
+ * {@link org.apache.calcite.statistic.QuerySqlStatisticProvider}, executes
+ * queries on a populated database. Implementations that use database statistics
+ * (from {@code ANALYZE TABLE}, etc.) and catalog information (e.g. primary and
+ * foreign key constraints) would also be possible.
  */
 public interface SqlStatisticProvider {
-  double tableCardinality(List<String> qualifiedTableName);
+  /** Returns an estimate of the number of rows in {@code table}. */
+  double tableCardinality(RelOptTable table);
+
+  /** Returns whether a join is a foreign key; that is, whether every row in
+   * the referencing table is matched by at least one row in the referenced
+   * table.
+   *
+   * <p>For example, {@code isForeignKey(EMP, [DEPTNO], DEPT, [DEPTNO])}
+   * returns true.
+   *
+   * <p>To change "at least one" to "exactly one", you also need to call
+   * {@link #isKey}. */
+  boolean isForeignKey(RelOptTable fromTable, List<Integer> fromColumns,
+      RelOptTable toTable, List<Integer> toColumns);
+
+  /** Returns whether a collection of columns is a unique (or primary) key.
+   *
+   * <p>For example, {@code isKey(EMP, [DEPTNO]} returns true;
+   * <p>For example, {@code isKey(DEPT, [DEPTNO]} returns false. */
+  boolean isKey(RelOptTable table, List<Integer> columns);
 }
 
 // End SqlStatisticProvider.java