Aggregate index support in Cascades

docs/ReleaseNotes.md

@@ -30,6 +30,7 @@ The Guava dependency version has been updated to 31.1. Projects may need to chec
 * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
+* **Feature** Support planning aggregate indexes in Cascades. [(Issue #1885)](https://github.com/FoundationDB/fdb-record-layer/issues/1885)
 * **Breaking change** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Breaking change** Change 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Breaking change** Change 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -38,7 +38,9 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -690,9 +692,12 @@ public static Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMapFrom
 
     @Nonnull
     private static Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMap(@Nonnull final Stream<RecordType> recordTypeStream) {
+        // todo: should be removed https://github.com/FoundationDB/fdb-record-layer/issues/1884
         return recordTypeStream
+                .sorted(Comparator.comparing(RecordType::getName))
                 .flatMap(recordType -> recordType.getDescriptor().getFields().stream())
                 .collect(Collectors.groupingBy(Descriptors.FieldDescriptor::getName,
+                        LinkedHashMap::new,
                         Collectors.reducing(null,
                                 (fieldDescriptor, fieldDescriptor2) -> {
                                     Verify.verify(fieldDescriptor != null || fieldDescriptor2 != null);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Index.java

@@ -259,6 +259,12 @@ public String getName() {
         return name;
     }
 
+    /**
+     * Returns the type of the index.
+     *
+     * @return the type of the index.
+     * @see IndexTypes
+     */
     @Nonnull
     public String getType() {
         return type;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java

@@ -973,7 +973,7 @@ protected Object getBindable() {
     }
 
     /**
-     * Adjust Match Task. Attempts to improve an existing partial partial match on a (group, expression) pair
+     * Adjust Match Task. Attempts to improve an existing partial match on a (group, expression) pair
      * to a better one by enqueuing rules defined on {@link PartialMatch}.
      *
      * Simplified enqueue/execute overview:

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java

@@ -32,10 +32,10 @@
 import com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionExpansionVisitor.VisitorState;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
+import com.apple.foundationdb.record.query.plan.cascades.predicates.ValueComparisonRangePredicate.Placeholder;
 import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
-import com.apple.foundationdb.record.query.plan.cascades.predicates.ValueComparisonRangePredicate.Placeholder;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java

@@ -160,7 +160,7 @@ default Map<CorrelationIdentifier, ComparisonRange> computeBoundParameterPrefixM
      * Compute a list of {@link MatchedOrderingPart}s which forms a bridge to relate {@link KeyExpression}s and
      * {@link QueryPredicate}s.
      * @param matchInfo a pre-existing match info structure
-     * @param sortParameterIds the query should be ordered by
+     * @param sortParameterIds the parameter IDs which the query should be ordered by
      * @param isReverse reversed-ness of the order
      * @return a list of bound key parts that express the order of the outgoing data stream and their respective mappings
      *         between query and match candidate
@@ -207,7 +207,7 @@ default RelationalExpression toEquivalentExpression(@Nonnull final PartialMatch
     /**
      * Creates a logical expression that represents a scan over the materialized candidate data. This method is expected
      * to be implemented by specific implementations of {@link MatchCandidate}.
-     * @param partialMatch the {@link PartialMatch} that matched th query and the candidate
+     * @param partialMatch the {@link PartialMatch} that matched the query and the candidate
      * @param planContext the plan context for the query
      * @param comparisonRanges a {@link List} of {@link ComparisonRange}s to be applied
      * @return a new {@link RelationalExpression}
@@ -237,6 +237,20 @@ default SetMultimap<ExpressionRef<? extends RelationalExpression>, RelationalExp
         return refToExpressionMap;
     }
 
+    @Nonnull
+    List<RecordType> getQueriedRecordTypes();
+
+    int getColumnSize();
+
+    boolean isUnique();
+
+    @Nonnull
+    default Set<String> getQueriedRecordTypeNames() {
+        return getQueriedRecordTypes().stream()
+                .map(RecordType::getName)
+                .collect(ImmutableSet.toImmutableSet());
+    }
+
     @Nonnull
     static Iterable<MatchCandidate> fromIndexDefinition(@Nonnull final RecordMetaData metaData,
                                                         @Nonnull final Index index,
@@ -290,6 +304,18 @@ static Iterable<MatchCandidate> fromIndexDefinition(@Nonnull final RecordMetaDat
                     .ifPresent(resultBuilder::add);
         }
 
+        if (AggregateIndexExpansionVisitor.isAggregateIndex(type, index.getRootExpression())) {
+            expandIndexMatchCandidate(index,
+                    availableRecordTypeNames,
+                    availableRecordTypes,
+                    queriedRecordTypeNames,
+                    queriedRecordTypes,
+                    isReverse,
+                    null,
+                    new AggregateIndexExpansionVisitor(index, queriedRecordTypes))
+                    .ifPresent(resultBuilder::add);
+        }
+
         return resultBuilder.build();
     }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchInfo.java

@@ -217,6 +217,7 @@ public Optional<Boolean> deriveReverseScanOrder() {
         }
     }
 
+    @Nonnull
     public MatchInfo withOrderingInfo(@Nonnull final List<MatchedOrderingPart> matchedOrderingParts) {
         return new MatchInfo(parameterBindingMap,
                 quantifierToPartialMatchMap,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PartialMatch.java

@@ -261,7 +261,7 @@ public boolean compensationCanBeDeferred() {
     }
 
     @Nonnull
-    public static Collection<MatchInfo> matchesFromMap(@Nonnull IdentityBiMap<Quantifier, PartialMatch> partialMatchMap) {
+    public static Collection<MatchInfo> matchesFromMap(@Nonnull final IdentityBiMap<Quantifier, PartialMatch> partialMatchMap) {
         return partialMatchMap.values()
                 .stream()
                 .map(IdentityBiMap::unwrap)

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PrimaryScanMatchCandidate.java

@@ -157,6 +157,16 @@ public boolean createsDuplicates() {
         return false;
     }
 
+    @Override
+    public int getColumnSize() {
+        return primaryKey.getColumnSize(); // not sure this is correct.
+    }
+
+    @Override
+    public boolean isUnique() {
+        return true;
+    }
+
     @Nonnull
     @Override
     public RelationalExpression toEquivalentExpression(@Nonnull PartialMatch partialMatch,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java

@@ -44,7 +44,9 @@
 import java.util.List;
 
 /**
- * Visitor that translates a {@link KeyExpression} into a {@link Value}.
+ * Visitor that translates a {@link KeyExpression} into a {@link Value}, keeping a state of the currently processed
+ * input type while it is processing the {@link KeyExpression}. The main caveat of this visitor is that it is meant to only
+ * expand key expressions that are provably scalar.
  */
 @SuppressWarnings("java:S5993")
 public class ScalarTranslationVisitor implements KeyExpressionVisitor<ScalarTranslationVisitor.ScalarVisitorState, Value> {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java

@@ -20,7 +20,6 @@
 
 package com.apple.foundationdb.record.query.plan.cascades;
 
-import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 
 import javax.annotation.Nonnull;
@@ -31,9 +30,6 @@
  */
 public interface ScanWithFetchMatchCandidate extends WithPrimaryKeyMatchCandidate {
 
-    @Nonnull
-    Index getIndex();
-
     @Nonnull
     Optional<Value> pushValueThroughFetch(@Nonnull Value value,
                                           @Nonnull CorrelationIdentifier sourceAlias,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java

@@ -136,10 +136,14 @@ public ValueIndexScanMatchCandidate(@Nonnull Index index,
         this.primaryKeyValuesSupplier = Suppliers.memoize(() -> MatchCandidate.computePrimaryKeyValuesMaybe(primaryKey, baseType));
     }
 
-    @Nonnull
     @Override
-    public Index getIndex() {
-        return index;
+    public int getColumnSize() {
+        return index.getColumnSize();
+    }
+
+    @Override
+    public boolean isUnique() {
+        return index.isUnique();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java

@@ -158,10 +158,14 @@ public WindowedIndexScanMatchCandidate(@Nonnull Index index,
         this.primaryKeyValuesSupplier = Suppliers.memoize(() -> MatchCandidate.computePrimaryKeyValuesMaybe(primaryKey, baseType));
     }
 
-    @Nonnull
     @Override
-    public Index getIndex() {
-        return index;
+    public int getColumnSize() {
+        return index.getColumnSize();
+    }
+
+    @Override
+    public boolean isUnique() {
+        return index.isUnique();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WithPrimaryKeyMatchCandidate.java

@@ -20,14 +20,11 @@
 
 package com.apple.foundationdb.record.query.plan.cascades;
 
-import com.apple.foundationdb.record.metadata.RecordType;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
-import com.google.common.collect.ImmutableSet;
 
 import javax.annotation.Nonnull;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 
 /**
  * Interface to represent a candidate that uses a primary key to identify a record.
@@ -36,16 +33,6 @@ public interface WithPrimaryKeyMatchCandidate extends MatchCandidate {
     @Nonnull
     Optional<List<Value>> getPrimaryKeyValuesMaybe();
 
-    @Nonnull
-    List<RecordType> getQueriedRecordTypes();
-
-    @Nonnull
-    default Set<String> getQueriedRecordTypeNames() {
-        return getQueriedRecordTypes().stream()
-                .map(RecordType::getName)
-                .collect(ImmutableSet.toImmutableSet());
-    }
-
     @Nonnull
     static Optional<List<Value>> commonPrimaryKeyValuesMaybe(@Nonnull Iterable<? extends MatchCandidate> matchCandidates) {
         List<Value> common = null;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/NodeInfo.java

@@ -64,6 +64,13 @@ public class NodeInfo {
             NodeIcon.DATA_ACCESS_OPERATOR,
             "Covering Index Scan",
             "A covering index scan operator uses a secondary index to quickly find records in a given search range without looking up the base records.");
+
+    public static final NodeInfo AGGREGATE_INDEX_SCAN_OPERATOR = new NodeInfo(
+            "AggregateIndexScanOperator",
+            NodeIcon.DATA_ACCESS_OPERATOR,
+            "Aggregate Index Scan",
+            "An aggregate index scan operator uses a secondary index to quickly find records in a given search range without looking up the base records.");
+
     public static final NodeInfo COVERING_SPATIAL_INDEX_SCAN_OPERATOR = new NodeInfo(
             "CoveringSpatialIndexScanOperator",
             NodeIcon.DATA_ACCESS_OPERATOR,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/GroupByExpression.java

@@ -23,8 +23,14 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
 import com.apple.foundationdb.record.query.plan.cascades.Column;
+import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
+import com.apple.foundationdb.record.query.plan.cascades.Compensation;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
+import com.apple.foundationdb.record.query.plan.cascades.IdentityBiMap;
 import com.apple.foundationdb.record.query.plan.cascades.OrderingPart;
+import com.apple.foundationdb.record.query.plan.cascades.MatchInfo;
+import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
+import com.apple.foundationdb.record.query.plan.cascades.PredicateMap;
 import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
 import com.apple.foundationdb.record.query.plan.cascades.RequestedOrdering;
 import com.apple.foundationdb.record.query.plan.cascades.TranslationMap;
@@ -46,6 +52,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -249,6 +256,42 @@ public Value getRuntimeValue() {
         return computeRuntimeResultSupplier.get();
     }
 
+    @Override
+    public Compensation compensate(@Nonnull final PartialMatch partialMatch,
+                                   @Nonnull final Map<CorrelationIdentifier, ComparisonRange> boundParameterPrefixMap) {
+        // subsumedBy() is based on equality, thus we return empty here as
+        // if there is a match, it's exact
+        return Compensation.noCompensation();
+    }
+
+    @Nonnull
+    @Override
+    public Iterable<MatchInfo> subsumedBy(@Nonnull final RelationalExpression candidateExpression,
+                                          @Nonnull final AliasMap aliasMap,
+                                          @Nonnull final IdentityBiMap<Quantifier, PartialMatch> partialMatchMap) {
+
+        // the candidate must be a GROUP-BY expression.
+        if (candidateExpression.getClass() != this.getClass()) {
+            return ImmutableList.of();
+        }
+
+        final var otherGroupByExpression = (GroupByExpression)candidateExpression;
+
+        // the grouping values are encoded directly in the underlying SELECT-WHERE, reaching this point means that the
+        // grouping values had exact match so we don't need to check them.
+
+
+        // check that aggregate value is the same.
+        final var otherAggregateValue = otherGroupByExpression.getAggregateValue();
+        if (aggregateValue.subsumedBy(otherAggregateValue, aliasMap)) {
+            // placeholder for information needed for later compensation.
+            return MatchInfo.tryMerge(partialMatchMap, ImmutableMap.of(), PredicateMap.empty(), Optional.empty())
+                    .map(ImmutableList::of)
+                    .orElse(ImmutableList.of());
+        }
+        return ImmutableList.of();
+    }
+
     @Nonnull
     private Value computeResultValue() {
         final var aggregateColumn = Column.of(Type.Record.Field.of(getAggregateValue().getResultType(), Optional.of(getAggregateValueAlias().getId())), getAggregateValue());

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java

@@ -217,7 +217,7 @@ public PartiallyOrderedSet<CorrelationIdentifier> getCorrelationOrder() {
     private PartiallyOrderedSet<CorrelationIdentifier> computeCorrelationOrder() {
         return RelationalExpressionWithChildren.ChildrenAsSet.super.getCorrelationOrder();
     }
-    
+
     @Nonnull
     @Override
     @SuppressWarnings("PMD.CompareObjectsWithEquals")
@@ -360,16 +360,16 @@ public Iterable<MatchInfo> subsumedBy(@Nonnull final RelationalExpression candid
         // FROM R r, S s
         // WHERE r.a < 5 AND s.b = 10 AND r.x = s.y
         //
-        // The predicate 'r.x = r.y' can be used as a predicate for matching an index on R(x, a) or for
+        // The predicate 'r.x = s.y' can be used as a predicate for matching an index on R(x, a) or for
         // matching an index on S(b, y). In fact the predicate needs to be shared in some way such that the planner
         // can later on make the right decision based on cost, etc.
         //
         // The way this is implemented is to create two matches one where the predicate is repossessed to the match
         // at hand. When we match R(x, a) we repossess r.x = r.y to be subsumed by r.x = ? on
-        // the candidate side. Vica versa, when we match S(b, y) we repossess s.y = r.x to be subsumed by s.y = ? on the
+        // the candidate side. Vice versa, when we match S(b, y) we repossess s.y = r.x to be subsumed by s.y = ? on the
         // candidate side.
         //
-        // Using this approach we create a problem that these two matches cannot coexist in a way that they cannot
+        // Using this approach we create a problem that these two matches can coexist in a way that they cannot
         // be realized, that is planned together at all as both matches provide the other's placeholder value. In fact,
         // we have forced the match to (if it were to be planned) become the inner of a join. It would be beneficial,
         // however, to also create a version of the match that does not consider the join predicate at all.

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java

@@ -36,6 +36,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpressionWithPredicates;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan;
 import com.google.common.collect.ImmutableList;
 
@@ -234,4 +235,19 @@ public static BindingMatcher<ExplodeExpression> explodeExpression() {
     public static BindingMatcher<GroupByExpression> groupByExpression(@Nonnull final CollectionMatcher<? extends Quantifier> downstream) {
         return ofTypeOwning(GroupByExpression.class, downstream);
     }
+
+    @Nonnull
+    public static BindingMatcher<GroupByExpression> groupByExpression(@Nonnull final BindingMatcher<? extends RecordConstructorValue> downstreamAggregation,
+                                                                      @Nonnull final CollectionMatcher<? extends Quantifier> downstreamQuantifiers) {
+        return typedWithDownstream(GroupByExpression.class,
+                Extractor.identity(),
+                AllOfMatcher.matchingAllOf(GroupByExpression.class,
+                        ImmutableList.of(
+                                typedWithDownstream(GroupByExpression.class,
+                                        Extractor.of(GroupByExpression::getAggregateValue, name -> "aggregation(" + name + ")"),
+                                        downstreamAggregation),
+                                typedWithDownstream(GroupByExpression.class,
+                                        Extractor.of(RelationalExpression::getQuantifiers, name -> "quantifiers(" + name + ")"),
+                                        downstreamQuantifiers))));
+    }
 }