From 6f1e07305d47656f3f9c0b085d329439a6d1a0d1 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Fri, 26 Sep 2025 14:16:55 +0100 Subject: [PATCH 1/9] refactor MatchCandidate to allow for state to be passed via a single object instead of multiple parameters --- .../query/plan/cascades/MatchCandidate.java | 261 ++++++++++-------- 1 file changed, 145 insertions(+), 116 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java index 4990e88a93..bcaca15055 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.query.plan.cascades; +import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.KeyValueLogMessage; @@ -53,6 +54,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -274,61 +276,132 @@ default Set getQueriedRecordTypeNames() { .collect(ImmutableSet.toImmutableSet()); } + /** + * Class encapsulating the information necessary to create a {@link MatchCandidate} + * for an {@link Index}. This exists as a convenience class that allows certain + * information to be computed once and referenced during match candidate creation. + */ + @API(API.Status.INTERNAL) + class IndexExpansionInfo { + @Nonnull + private final RecordMetaData metaData; + @Nonnull + private final Index index; + private final boolean reverse; + @Nullable + private final KeyExpression commonPrimaryKeyForTypes; + @Nonnull + private final Collection indexedRecordTypes; + @Nonnull + private final Set indexedRecordTypeNames; + + private IndexExpansionInfo(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse, + @Nonnull Collection indexedRecordTypes, + @Nonnull Set indexedRecordTypeNames, + @Nullable KeyExpression commonPrimaryKeyForTypes) { + this.metaData = metaData; + this.index = index; + this.reverse = reverse; + this.indexedRecordTypes = indexedRecordTypes; + this.indexedRecordTypeNames = indexedRecordTypeNames; + this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; + } + + @Nonnull + public RecordMetaData getMetaData() { + return metaData; + } + + @Nonnull + public Index getIndex() { + return index; + } + + @Nonnull + public String getIndexName() { + return index.getName(); + } + + @Nonnull + public String getIndexType() { + return index.getType(); + } + + public boolean isReverse() { + return reverse; + } + + @Nonnull + public Collection getIndexedRecordTypes() { + return indexedRecordTypes; + } + + @Nonnull + public Set getIndexedRecordTypeNames() { + return indexedRecordTypeNames; + } + + @Nullable + public KeyExpression getCommonPrimaryKeyForTypes() { + return commonPrimaryKeyForTypes; + } + + @Nonnull + public Set getAvailableRecordTypeNames() { + return metaData.getRecordTypes().keySet(); + } + + /** + * Create an {@link IndexExpansionInfo} for a given index. + * This wraps the given parameters into a single object, as well + * as enriching the given parameters with pre-calculated items that + * can then be used during index expansion. + * + * @param metaData the meta-data that is the source of the index + * @param index the index that we are expanding + * @param reverse whether the query requires this scan be in reverse + * @return an object encapsulating information about the index + */ + @Nonnull + public static IndexExpansionInfo forIndex(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse) { + @Nonnull + final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); + @Nonnull + final Set indexedRecordTypeNames = indexedRecordTypes.stream() + .map(RecordType::getName) + .collect(ImmutableSet.toImmutableSet()); + @Nullable + final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); + + return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); + } + + } + @Nonnull static Iterable fromIndexDefinition(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean isReverse) { - final var resultBuilder = ImmutableList.builder(); - final var queriedRecordTypes = metaData.recordTypesForIndex(index); - final var commonPrimaryKeyForIndex = RecordMetaData.commonPrimaryKey(queriedRecordTypes); + final IndexExpansionInfo info = IndexExpansionInfo.forIndex(metaData, index, isReverse); + final ImmutableList.Builder resultBuilder = ImmutableList.builder(); - final var queriedRecordTypeNames = - queriedRecordTypes - .stream() - .map(RecordType::getName) - .collect(ImmutableSet.toImmutableSet()); - - final var recordTypeMap = metaData.getRecordTypes(); - final var availableRecordTypeNames = recordTypeMap.keySet(); - final var availableRecordTypes = recordTypeMap.values(); - - final var indexType = index.getType(); - - switch (indexType) { + switch (info.getIndexType()) { case IndexTypes.VALUE: case IndexTypes.VERSION: - expandValueIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - commonPrimaryKeyForIndex - ).ifPresent(resultBuilder::add); + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); break; case IndexTypes.RANK: // For rank() we need to create at least two candidates. One for BY_RANK scans and one for BY_VALUE scans. - expandValueIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - commonPrimaryKeyForIndex - ).ifPresent(resultBuilder::add); - - expandIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - commonPrimaryKeyForIndex, - new WindowedIndexExpansionVisitor(index, queriedRecordTypes) - ).ifPresent(resultBuilder::add); + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + + expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new WindowedIndexExpansionVisitor(index, info.getIndexedRecordTypes())) + .ifPresent(resultBuilder::add); break; case IndexTypes.MIN_EVER_TUPLE: // fallthrough case IndexTypes.MAX_EVER_TUPLE: // fallthrough @@ -338,36 +411,17 @@ static Iterable fromIndexDefinition(@Nonnull final RecordMetaDat case IndexTypes.SUM: // fallthrough case IndexTypes.COUNT: // fallthrough case IndexTypes.COUNT_NOT_NULL: - expandAggregateIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse - ).ifPresent(resultBuilder::add); + expandAggregateIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); break; case IndexTypes.PERMUTED_MAX: // fallthrough case IndexTypes.PERMUTED_MIN: // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use // the aggregate index expansion for BY_GROUP scans - expandValueIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - commonPrimaryKeyForIndex - ).ifPresent(resultBuilder::add); - expandAggregateIndexMatchCandidate( - index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse - ).ifPresent(resultBuilder::add); + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + expandAggregateIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); break; default: break; @@ -375,63 +429,33 @@ static Iterable fromIndexDefinition(@Nonnull final RecordMetaDat return resultBuilder.build(); } - private static Optional expandValueIndexMatchCandidate(@Nonnull final Index index, - @Nonnull final Set availableRecordTypeNames, - @Nonnull final Collection availableRecordTypes, - @Nonnull final Set queriedRecordTypeNames, - @Nonnull final Collection queriedRecordTypes, - final boolean isReverse, - @Nullable final KeyExpression commonPrimaryKeyForIndex) { - return expandIndexMatchCandidate(index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - commonPrimaryKeyForIndex, - new ValueIndexExpansionVisitor(index, queriedRecordTypes) - ); + private static Optional expandValueIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { + return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); } - private static Optional expandAggregateIndexMatchCandidate(@Nonnull final Index index, - @Nonnull final Set availableRecordTypeNames, - @Nonnull final Collection availableRecordTypes, - @Nonnull final Set queriedRecordTypeNames, - @Nonnull final Collection queriedRecordTypes, - final boolean isReverse) { - final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(index.getType()) - ? new BitmapAggregateIndexExpansionVisitor(index, queriedRecordTypes) - : new AggregateIndexExpansionVisitor(index, queriedRecordTypes); - return expandIndexMatchCandidate(index, - availableRecordTypeNames, - availableRecordTypes, - queriedRecordTypeNames, - queriedRecordTypes, - isReverse, - null, - aggregateIndexExpansionVisitor - ); + private static Optional expandAggregateIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { + final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(info.getIndexType()) + ? new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()) + : new AggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); + // Override the common primary key here. We always want it to be null because the primary key is not + // included in the expanded aggregate index + return expandIndexMatchCandidate(info, null, aggregateIndexExpansionVisitor); } @Nonnull - private static Optional expandIndexMatchCandidate(@Nonnull final Index index, - @Nonnull final Set availableRecordTypeNames, - @Nonnull final Collection availableRecordTypes, - @Nonnull final Set queriedRecordTypeNames, - @Nonnull final Collection queriedRecordTypes, - final boolean isReverse, - @Nullable final KeyExpression commonPrimaryKeyForIndex, + private static Optional expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info, + @Nullable KeyExpression commonPrimaryKey, @Nonnull final ExpansionVisitor expansionVisitor) { - final var baseRef = createBaseRef(availableRecordTypeNames, availableRecordTypes, queriedRecordTypeNames, queriedRecordTypes, new IndexAccessHint(index.getName())); + final var baseRef = createBaseRef(info, new IndexAccessHint(info.getIndexName())); try { - return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKeyForIndex, isReverse)); + return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKey, info.isReverse())); } catch (final UnsupportedOperationException uOE) { // just log and return empty if (LOGGER.isDebugEnabled()) { final String message = KeyValueLogMessage.of("unsupported index", "reason", uOE.getMessage(), - "indexName", index.getName()); + "indexName", info.getIndexName()); LOGGER.debug(message, uOE); } } @@ -450,7 +474,7 @@ static Optional fromPrimaryDefinition(@Nonnull final RecordMetaD .filter(recordType -> queriedRecordTypeNames.contains(recordType.getName())) .collect(ImmutableList.toImmutableList()); - final var baseRef = createBaseRef(metaData.getRecordTypes().keySet(), availableRecordTypes, queriedRecordTypeNames, queriedRecordTypes, new PrimaryAccessHint()); + final var baseRef = createBaseRef(metaData.getRecordTypes().keySet(), queriedRecordTypeNames, queriedRecordTypes, new PrimaryAccessHint()); final var expansionVisitor = new PrimaryAccessExpansionVisitor(availableRecordTypes, queriedRecordTypes); return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), primaryKey, isReverse)); } @@ -458,9 +482,14 @@ static Optional fromPrimaryDefinition(@Nonnull final RecordMetaD return Optional.empty(); } + @Nonnull + static Reference createBaseRef(@Nonnull IndexExpansionInfo info, + @Nonnull AccessHint accessHint) { + return createBaseRef(info.getAvailableRecordTypeNames(), info.getIndexedRecordTypeNames(), info.getIndexedRecordTypes(), accessHint); + } + @Nonnull static Reference createBaseRef(@Nonnull final Set availableRecordTypeNames, - @Nonnull final Collection availableRecordTypes, @Nonnull final Set queriedRecordTypeNames, @Nonnull final Collection queriedRecordTypes, @Nonnull AccessHint accessHint) { From 0abc66fd4880b001d253c7841fdf24084d665a2d Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Fri, 26 Sep 2025 14:57:11 +0100 Subject: [PATCH 2/9] move match candidate expansion methods around --- .../foundationdb/IndexMaintainerFactory.java | 44 +++ .../IndexMaintainerFactoryRegistry.java | 72 +++++ .../IndexMaintainerRegistryImpl.java | 17 +- .../IndexMatchCandidateRegistry.java | 53 ++++ .../query/plan/cascades/MatchCandidate.java | 243 --------------- .../cascades/MatchCandidateExpansion.java | 278 ++++++++++++++++++ .../plan/cascades/MetaDataPlanContext.java | 8 +- 7 files changed, 454 insertions(+), 261 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistry.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java index f331e18040..138c180d4c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java @@ -21,10 +21,13 @@ package com.apple.foundationdb.record.provider.foundationdb; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexValidator; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import javax.annotation.Nonnull; +import java.util.Collections; /** * A factory for {@link IndexMaintainer}. @@ -68,4 +71,45 @@ public interface IndexMaintainerFactory { */ @Nonnull IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state); + + /** + * Create {@link MatchCandidate}s to use for the given {@link Index}. + * Implementors can assume that the index is one of this factory's + * {@linkplain #getIndexTypes() supported types}. Note that this is + * structured as a method on the maintainer factory so that indexes + * defined outside the core repository can define the structure + * used by the planner to match them. However, use cases should be + * aware that the planner + * + *

+ * By default, this returns an empty collection. This means that the + * index will not be matchable by the {@link com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner}. + * Index types that want to be used in plans should implement this + * method. Utility methods in {@link MatchCandidate} can be used to + * create, for example, canonical expansions of indexes that behave like + * indexes defined in the core repository (e.g., {@link com.apple.foundationdb.record.metadata.IndexTypes#VALUE} + * indexes). + *

+ * + *

+ * Indexes may return more than one {@link MatchCandidate}. This can be + * useful for index types that have more than one way of being scanned. + * For example, the {@link com.apple.foundationdb.record.metadata.IndexTypes#RANK} + * index can be scanned {@link com.apple.foundationdb.record.IndexScanType#BY_VALUE}, + * at which point it behaves just like a {@link com.apple.foundationdb.record.metadata.IndexTypes#VALUE} + * index, or it can be scanned {@link com.apple.foundationdb.record.IndexScanType#BY_RANK}, + * at which point it behaves very differently. The different scan types are reflected + * in different {@link MatchCandidate} which encode different information about + * the index and how it should be scanned. + *

+ * + * @param metaData the meta-data in which the index is defined + * @param index the index to expend + * @param reverse whether the index is to be scanned in reverse + * @return a collection of {@link MatchCandidate}s to match the index against + */ + @Nonnull + default Iterable createMatchCandidates(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean reverse) { + return Collections.emptyList(); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistry.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistry.java new file mode 100644 index 0000000000..ab9db58e81 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistry.java @@ -0,0 +1,72 @@ +/* + * IndexMaintainerFactoryRegistry.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexValidator; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; + +import javax.annotation.Nonnull; + +/** + * Registry of {@link IndexMaintainerFactory} objects. Each index type has an associated + * implementation of {@link IndexMaintainerFactory} that can generate {@link IndexMaintainer}s + * and {@link IndexValidator}s for indexes of that type. This registry stores the mapping of + * the index type to one of those factory classes. + */ +@API(API.Status.UNSTABLE) +public interface IndexMaintainerFactoryRegistry extends IndexMaintainerRegistry, IndexMatchCandidateRegistry { + /** + * Get the {@link IndexMaintainerFactory} for the given index. + * This should look at the index's {@linkplain Index#getType() type} + * in order to identify the correct one. + * + * @param index the index to retrieve the maintainer factory for + * @return an {@link IndexMaintainerFactory} + * @throws com.apple.foundationdb.record.metadata.MetaDataException if the + * {@link IndexMaintainerFactory} cannot be retrieved (e.g., if the index type + * is unknown) + */ + @Nonnull + IndexMaintainerFactory getIndexMaintainerFactory(@Nonnull Index index); + + @Nonnull + @Override + default IndexValidator getIndexValidator(@Nonnull Index index) { + final IndexMaintainerFactory factory = getIndexMaintainerFactory(index); + return factory.getIndexValidator(index); + } + + @Nonnull + @Override + default IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { + final IndexMaintainerFactory factory = getIndexMaintainerFactory(state.index); + return factory.getIndexMaintainer(state); + } + + @Override + default Iterable createMatchCandidates(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean reverse) { + final IndexMaintainerFactory factory = getIndexMaintainerFactory(index); + return factory.createMatchCandidates(metaData, index, reverse); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java index fdafef2bd5..a471554aee 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; -import com.apple.foundationdb.record.metadata.IndexValidator; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.util.ServiceLoaderProvider; import org.slf4j.Logger; @@ -38,7 +37,7 @@ * A singleton {@link IndexMaintainerRegistry} that finds {@link IndexMaintainerFactory} classes in the classpath. */ @API(API.Status.INTERNAL) -public class IndexMaintainerRegistryImpl implements IndexMaintainerRegistry { +public class IndexMaintainerRegistryImpl implements IndexMaintainerFactoryRegistry { @Nonnull private static final Logger LOGGER = LoggerFactory.getLogger(IndexMaintainerRegistryImpl.class); @Nonnull @@ -75,21 +74,11 @@ protected IndexMaintainerRegistryImpl() { @Nonnull @Override - public IndexValidator getIndexValidator(@Nonnull Index index) { + public IndexMaintainerFactory getIndexMaintainerFactory(@Nonnull final Index index) { final IndexMaintainerFactory factory = registry.get(index.getType()); if (factory == null) { throw new MetaDataException("Unknown index type for " + index); } - return factory.getIndexValidator(index); - } - - @Nonnull - @Override - public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { - final IndexMaintainerFactory factory = registry.get(state.index.getType()); - if (factory == null) { - throw new MetaDataException("Unknown index type for " + state.index); - } - return factory.getIndexMaintainer(state); + return factory; } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java new file mode 100644 index 0000000000..23bc588f51 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java @@ -0,0 +1,53 @@ +/* + * IndexMatchCandidateRegistry.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; + +import javax.annotation.Nonnull; + +/** + * A registry for mapping indexes to {@link MatchCandidate}s that can be used during + * planning by the {@link com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner}. + * This is structured as a registry so that new index types can define the match candidate + * without needing to be defined in the core Record Layer repository. + * + * @see IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean) + * @see IndexMaintainerFactoryRegistry for a sub-interface that delegates to the {@link IndexMaintainerFactory} + * @see IndexMaintainerRegistryImpl for the default implementation + */ +public interface IndexMatchCandidateRegistry { + + /** + * Create match candidates for the given {@link Index}. + * + * @param metaData the meta-data in which the index sits + * @param index the index to expand into {@link MatchCandidate}s + * @param reverse whether the query calls for the candidate to be reversed + * @return a collection of {@link MatchCandidate}s representing this index + * @see IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean) + */ + Iterable createMatchCandidates(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse); +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java index bcaca15055..d1e5f418de 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidate.java @@ -20,19 +20,12 @@ package com.apple.foundationdb.record.query.plan.cascades; -import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.EvaluationContext; -import com.apple.foundationdb.record.RecordMetaData; -import com.apple.foundationdb.record.logging.KeyValueLogMessage; -import com.apple.foundationdb.record.metadata.Index; -import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.ScanComparisons; import com.apple.foundationdb.record.query.plan.cascades.OrderingPart.MatchedOrderingPart; -import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; -import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -48,13 +41,9 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimaps; import com.google.common.collect.SetMultimap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -72,9 +61,6 @@ */ public interface MatchCandidate { - @Nonnull - Logger LOGGER = LoggerFactory.getLogger(MatchCandidate.class); - /** * Returns the name of the match candidate. If this candidate represents an index, it will be the name of the index. * @@ -276,235 +262,6 @@ default Set getQueriedRecordTypeNames() { .collect(ImmutableSet.toImmutableSet()); } - /** - * Class encapsulating the information necessary to create a {@link MatchCandidate} - * for an {@link Index}. This exists as a convenience class that allows certain - * information to be computed once and referenced during match candidate creation. - */ - @API(API.Status.INTERNAL) - class IndexExpansionInfo { - @Nonnull - private final RecordMetaData metaData; - @Nonnull - private final Index index; - private final boolean reverse; - @Nullable - private final KeyExpression commonPrimaryKeyForTypes; - @Nonnull - private final Collection indexedRecordTypes; - @Nonnull - private final Set indexedRecordTypeNames; - - private IndexExpansionInfo(@Nonnull RecordMetaData metaData, - @Nonnull Index index, - boolean reverse, - @Nonnull Collection indexedRecordTypes, - @Nonnull Set indexedRecordTypeNames, - @Nullable KeyExpression commonPrimaryKeyForTypes) { - this.metaData = metaData; - this.index = index; - this.reverse = reverse; - this.indexedRecordTypes = indexedRecordTypes; - this.indexedRecordTypeNames = indexedRecordTypeNames; - this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; - } - - @Nonnull - public RecordMetaData getMetaData() { - return metaData; - } - - @Nonnull - public Index getIndex() { - return index; - } - - @Nonnull - public String getIndexName() { - return index.getName(); - } - - @Nonnull - public String getIndexType() { - return index.getType(); - } - - public boolean isReverse() { - return reverse; - } - - @Nonnull - public Collection getIndexedRecordTypes() { - return indexedRecordTypes; - } - - @Nonnull - public Set getIndexedRecordTypeNames() { - return indexedRecordTypeNames; - } - - @Nullable - public KeyExpression getCommonPrimaryKeyForTypes() { - return commonPrimaryKeyForTypes; - } - - @Nonnull - public Set getAvailableRecordTypeNames() { - return metaData.getRecordTypes().keySet(); - } - - /** - * Create an {@link IndexExpansionInfo} for a given index. - * This wraps the given parameters into a single object, as well - * as enriching the given parameters with pre-calculated items that - * can then be used during index expansion. - * - * @param metaData the meta-data that is the source of the index - * @param index the index that we are expanding - * @param reverse whether the query requires this scan be in reverse - * @return an object encapsulating information about the index - */ - @Nonnull - public static IndexExpansionInfo forIndex(@Nonnull RecordMetaData metaData, - @Nonnull Index index, - boolean reverse) { - @Nonnull - final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); - @Nonnull - final Set indexedRecordTypeNames = indexedRecordTypes.stream() - .map(RecordType::getName) - .collect(ImmutableSet.toImmutableSet()); - @Nullable - final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); - - return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); - } - - } - - @Nonnull - static Iterable fromIndexDefinition(@Nonnull final RecordMetaData metaData, - @Nonnull final Index index, - final boolean isReverse) { - final IndexExpansionInfo info = IndexExpansionInfo.forIndex(metaData, index, isReverse); - final ImmutableList.Builder resultBuilder = ImmutableList.builder(); - - switch (info.getIndexType()) { - case IndexTypes.VALUE: - case IndexTypes.VERSION: - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.RANK: - // For rank() we need to create at least two candidates. One for BY_RANK scans and one for BY_VALUE scans. - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - - expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new WindowedIndexExpansionVisitor(index, info.getIndexedRecordTypes())) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.MIN_EVER_TUPLE: // fallthrough - case IndexTypes.MAX_EVER_TUPLE: // fallthrough - case IndexTypes.MAX_EVER_LONG: // fallthrough - case IndexTypes.MIN_EVER_LONG: // fallthrough - case IndexTypes.BITMAP_VALUE: // fallthrough - case IndexTypes.SUM: // fallthrough - case IndexTypes.COUNT: // fallthrough - case IndexTypes.COUNT_NOT_NULL: - expandAggregateIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.PERMUTED_MAX: // fallthrough - case IndexTypes.PERMUTED_MIN: - // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use - // the aggregate index expansion for BY_GROUP scans - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - expandAggregateIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - default: - break; - } - return resultBuilder.build(); - } - - private static Optional expandValueIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { - return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); - } - - private static Optional expandAggregateIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { - final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(info.getIndexType()) - ? new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()) - : new AggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); - // Override the common primary key here. We always want it to be null because the primary key is not - // included in the expanded aggregate index - return expandIndexMatchCandidate(info, null, aggregateIndexExpansionVisitor); - } - - @Nonnull - private static Optional expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info, - @Nullable KeyExpression commonPrimaryKey, - @Nonnull final ExpansionVisitor expansionVisitor) { - final var baseRef = createBaseRef(info, new IndexAccessHint(info.getIndexName())); - try { - return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKey, info.isReverse())); - } catch (final UnsupportedOperationException uOE) { - // just log and return empty - if (LOGGER.isDebugEnabled()) { - final String message = - KeyValueLogMessage.of("unsupported index", - "reason", uOE.getMessage(), - "indexName", info.getIndexName()); - LOGGER.debug(message, uOE); - } - } - return Optional.empty(); - } - - @Nonnull - static Optional fromPrimaryDefinition(@Nonnull final RecordMetaData metaData, - @Nonnull final Set queriedRecordTypeNames, - @Nullable KeyExpression primaryKey, - final boolean isReverse) { - if (primaryKey != null) { - final var availableRecordTypes = metaData.getRecordTypes().values(); - final var queriedRecordTypes = - availableRecordTypes.stream() - .filter(recordType -> queriedRecordTypeNames.contains(recordType.getName())) - .collect(ImmutableList.toImmutableList()); - - final var baseRef = createBaseRef(metaData.getRecordTypes().keySet(), queriedRecordTypeNames, queriedRecordTypes, new PrimaryAccessHint()); - final var expansionVisitor = new PrimaryAccessExpansionVisitor(availableRecordTypes, queriedRecordTypes); - return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), primaryKey, isReverse)); - } - - return Optional.empty(); - } - - @Nonnull - static Reference createBaseRef(@Nonnull IndexExpansionInfo info, - @Nonnull AccessHint accessHint) { - return createBaseRef(info.getAvailableRecordTypeNames(), info.getIndexedRecordTypeNames(), info.getIndexedRecordTypes(), accessHint); - } - - @Nonnull - static Reference createBaseRef(@Nonnull final Set availableRecordTypeNames, - @Nonnull final Set queriedRecordTypeNames, - @Nonnull final Collection queriedRecordTypes, - @Nonnull AccessHint accessHint) { - final var quantifier = - Quantifier.forEach( - Reference.initialOf( - new FullUnorderedScanExpression(availableRecordTypeNames, - new Type.AnyRecord(false), - new AccessHints(accessHint)))); - return Reference.initialOf( - new LogicalTypeFilterExpression(queriedRecordTypeNames, - quantifier, - Type.Record.fromFieldDescriptorsMap(RecordMetaData.getFieldDescriptorMapFromTypes(queriedRecordTypes)))); - } - @Nonnull static Optional> computePrimaryKeyValuesMaybe(@Nullable KeyExpression primaryKey, @Nonnull Type flowedType) { if (primaryKey == null) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java new file mode 100644 index 0000000000..776bf608db --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java @@ -0,0 +1,278 @@ +/* + * MatchCandidateExpansion.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.logging.KeyValueLogMessage; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexTypes; +import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +public class MatchCandidateExpansion { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchCandidateExpansion.class); + + private MatchCandidateExpansion() { + } + + @Nonnull + public static Iterable fromIndexDefinition(@Nonnull final RecordMetaData metaData, + @Nonnull final Index index, + final boolean isReverse) { + final IndexExpansionInfo info = IndexExpansionInfo.forIndex(metaData, index, isReverse); + final ImmutableList.Builder resultBuilder = ImmutableList.builder(); + + switch (info.getIndexType()) { + case IndexTypes.VALUE: + case IndexTypes.VERSION: + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + break; + case IndexTypes.RANK: + // For rank() we need to create at least two candidates. One for BY_RANK scans and one for BY_VALUE scans. + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + + expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new WindowedIndexExpansionVisitor(index, info.getIndexedRecordTypes())) + .ifPresent(resultBuilder::add); + break; + case IndexTypes.MIN_EVER_TUPLE: // fallthrough + case IndexTypes.MAX_EVER_TUPLE: // fallthrough + case IndexTypes.MAX_EVER_LONG: // fallthrough + case IndexTypes.MIN_EVER_LONG: // fallthrough + case IndexTypes.BITMAP_VALUE: // fallthrough + case IndexTypes.SUM: // fallthrough + case IndexTypes.COUNT: // fallthrough + case IndexTypes.COUNT_NOT_NULL: + expandAggregateIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + break; + case IndexTypes.PERMUTED_MAX: // fallthrough + case IndexTypes.PERMUTED_MIN: + // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use + // the aggregate index expansion for BY_GROUP scans + expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + expandAggregateIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + break; + default: + break; + } + return resultBuilder.build(); + } + + public static Optional expandValueIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { + return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); + } + + public static Optional expandAggregateIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { + final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(info.getIndexType()) + ? new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()) + : new AggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); + // Override the common primary key here. We always want it to be null because the primary key is not + // included in the expanded aggregate index + return expandIndexMatchCandidate(info, null, aggregateIndexExpansionVisitor); + } + + @Nonnull + private static Optional expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info, + @Nullable KeyExpression commonPrimaryKey, + @Nonnull final ExpansionVisitor expansionVisitor) { + final var baseRef = createBaseRef(info, new IndexAccessHint(info.getIndexName())); + try { + return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKey, info.isReverse())); + } catch (final UnsupportedOperationException uOE) { + // just log and return empty + if (LOGGER.isDebugEnabled()) { + final String message = + KeyValueLogMessage.of("unsupported index", + "reason", uOE.getMessage(), + "indexName", info.getIndexName()); + LOGGER.debug(message, uOE); + } + } + return Optional.empty(); + } + + @Nonnull + public static Optional fromPrimaryDefinition(@Nonnull final RecordMetaData metaData, + @Nonnull final Set queriedRecordTypeNames, + @Nullable KeyExpression primaryKey, + final boolean isReverse) { + if (primaryKey != null) { + final var availableRecordTypes = metaData.getRecordTypes().values(); + final var queriedRecordTypes = + availableRecordTypes.stream() + .filter(recordType -> queriedRecordTypeNames.contains(recordType.getName())) + .collect(ImmutableList.toImmutableList()); + + final var baseRef = createBaseRef(metaData.getRecordTypes().keySet(), queriedRecordTypeNames, queriedRecordTypes, new PrimaryAccessHint()); + final var expansionVisitor = new PrimaryAccessExpansionVisitor(availableRecordTypes, queriedRecordTypes); + return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), primaryKey, isReverse)); + } + + return Optional.empty(); + } + + @Nonnull + private static Reference createBaseRef(@Nonnull IndexExpansionInfo info, + @Nonnull AccessHint accessHint) { + return createBaseRef(info.getAvailableRecordTypeNames(), info.getIndexedRecordTypeNames(), info.getIndexedRecordTypes(), accessHint); + } + + @Nonnull + private static Reference createBaseRef(@Nonnull final Set availableRecordTypeNames, + @Nonnull final Set queriedRecordTypeNames, + @Nonnull final Collection queriedRecordTypes, + @Nonnull AccessHint accessHint) { + final var quantifier = + Quantifier.forEach( + Reference.initialOf( + new FullUnorderedScanExpression(availableRecordTypeNames, + new Type.AnyRecord(false), + new AccessHints(accessHint)))); + return Reference.initialOf( + new LogicalTypeFilterExpression(queriedRecordTypeNames, + quantifier, + Type.Record.fromFieldDescriptorsMap(RecordMetaData.getFieldDescriptorMapFromTypes(queriedRecordTypes)))); + } + + /** + * Class encapsulating the information necessary to create a {@link MatchCandidate} + * for an {@link Index}. This exists as a convenience class that allows certain + * information to be computed once and referenced during match candidate creation. + */ + @API(API.Status.INTERNAL) + public static class IndexExpansionInfo { + @Nonnull + private final RecordMetaData metaData; + @Nonnull + private final Index index; + private final boolean reverse; + @Nullable + private final KeyExpression commonPrimaryKeyForTypes; + @Nonnull + private final Collection indexedRecordTypes; + @Nonnull + private final Set indexedRecordTypeNames; + + private IndexExpansionInfo(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse, + @Nonnull Collection indexedRecordTypes, + @Nonnull Set indexedRecordTypeNames, + @Nullable KeyExpression commonPrimaryKeyForTypes) { + this.metaData = metaData; + this.index = index; + this.reverse = reverse; + this.indexedRecordTypes = indexedRecordTypes; + this.indexedRecordTypeNames = indexedRecordTypeNames; + this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; + } + + @Nonnull + public RecordMetaData getMetaData() { + return metaData; + } + + @Nonnull + public Index getIndex() { + return index; + } + + @Nonnull + public String getIndexName() { + return index.getName(); + } + + @Nonnull + public String getIndexType() { + return index.getType(); + } + + public boolean isReverse() { + return reverse; + } + + @Nonnull + public Collection getIndexedRecordTypes() { + return indexedRecordTypes; + } + + @Nonnull + public Set getIndexedRecordTypeNames() { + return indexedRecordTypeNames; + } + + @Nullable + public KeyExpression getCommonPrimaryKeyForTypes() { + return commonPrimaryKeyForTypes; + } + + @Nonnull + public Set getAvailableRecordTypeNames() { + return metaData.getRecordTypes().keySet(); + } + + /** + * Create an {@link IndexExpansionInfo} for a given index. + * This wraps the given parameters into a single object, as well + * as enriching the given parameters with pre-calculated items that + * can then be used during index expansion. + * + * @param metaData the meta-data that is the source of the index + * @param index the index that we are expanding + * @param reverse whether the query requires this scan be in reverse + * @return an object encapsulating information about the index + */ + @Nonnull + public static IndexExpansionInfo forIndex(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse) { + @Nonnull + final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); + @Nonnull + final Set indexedRecordTypeNames = indexedRecordTypes.stream() + .map(RecordType::getName) + .collect(ImmutableSet.toImmutableSet()); + @Nullable + final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); + + return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index 6cbd34e6d6..7e4d7b7420 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -155,11 +155,11 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio final ImmutableSet.Builder matchCandidatesBuilder = ImmutableSet.builder(); for (Index index : indexList) { final Iterable candidatesForIndex = - MatchCandidate.fromIndexDefinition(metaData, index, isSortReverse); + MatchCandidateExpansion.fromIndexDefinition(metaData, index, isSortReverse); matchCandidatesBuilder.addAll(candidatesForIndex); } - MatchCandidate.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse) + MatchCandidateExpansion.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse) .ifPresent(matchCandidatesBuilder::add); return new MetaDataPlanContext(plannerConfiguration, matchCandidatesBuilder.build()); @@ -199,12 +199,12 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf final ImmutableSet.Builder matchCandidatesBuilder = ImmutableSet.builder(); for (final var index : indexList) { final Iterable candidatesForIndex = - MatchCandidate.fromIndexDefinition(metaData, index, false); + MatchCandidateExpansion.fromIndexDefinition(metaData, index, false); matchCandidatesBuilder.addAll(candidatesForIndex); } for (final var recordType : queriedRecordTypes) { - MatchCandidate.fromPrimaryDefinition(metaData, + MatchCandidateExpansion.fromPrimaryDefinition(metaData, ImmutableSet.of(recordType.getName()), recordType.getPrimaryKey(), false) From 1d15cea2a98299583e91dd48f8e4cae2a194782d Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Fri, 26 Sep 2025 19:20:42 +0100 Subject: [PATCH 3/9] use method on IndexMaintainerFactory to generate match candidates instead of a switch statement in MatchCandidate --- .../metadata/MetaDataEvolutionValidator.java | 41 +++---- .../provider/foundationdb/FDBRecordStore.java | 13 +- .../foundationdb/FDBRecordStoreBase.java | 6 +- .../foundationdb/FDBTypedRecordStore.java | 11 +- .../IndexMaintainerRegistryImpl.java | 2 +- .../AtomicMutationIndexMaintainerFactory.java | 8 ++ .../BitmapValueIndexMaintainerFactory.java | 9 ++ .../PermutedMinMaxIndexMaintainerFactory.java | 19 +++ .../indexes/RankIndexMaintainerFactory.java | 19 +++ .../indexes/ValueIndexMaintainerFactory.java | 9 ++ .../VersionIndexMaintainerFactory.java | 9 ++ .../query/plan/cascades/CascadesPlanner.java | 14 ++- .../cascades/MatchCandidateExpansion.java | 112 +++++++----------- .../plan/cascades/MetaDataPlanContext.java | 11 +- .../cascades/ValueIndexExpansionVisitor.java | 2 - .../FDBRecordStoreConcurrentTestBase.java | 2 +- .../query/QueryPlanFullySortedTest.java | 3 +- .../benchmark/BenchmarkRecordStore.java | 2 +- .../record/lucene/FDBLuceneQueryTest.java | 2 +- .../record/lucene/LuceneIndexTestUtils.java | 10 +- .../lucene/LuceneQueryIntegrationTest.java | 2 +- .../TestingIndexMaintainerRegistry.java | 20 +--- .../lucene/highlight/LuceneScaleTest.java | 2 +- .../AbstractEmbeddedStatement.java | 4 +- .../recordlayer/query/PlanGenerator.java | 23 ++-- .../relational/api/ddl/DdlTestUtil.java | 7 +- .../memory/InMemoryRelationalStatement.java | 3 +- .../recordlayer/PlanGenerationStackTest.java | 6 +- .../metadata/SchemaTemplateSerDeTests.java | 3 +- .../query/cache/ConstraintValidityTests.java | 2 +- .../query/cache/RelationalPlanCacheTests.java | 2 +- .../structuredsql/SqlVisitorTests.java | 2 +- 32 files changed, 218 insertions(+), 162 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java index a4e96d2b98..5201c26b51 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.util.pair.NonnullPair; @@ -90,7 +91,7 @@ public class MetaDataEvolutionValidator { private static final MetaDataEvolutionValidator DEFAULT_INSTANCE = new MetaDataEvolutionValidator(); @Nonnull - private final IndexMaintainerRegistry indexMaintainerRegistry; + private final IndexValidatorRegistry indexValidatorRegistry; private final boolean allowNoVersionChange; private final boolean allowNoSinceVersion; private final boolean allowIndexRebuilds; @@ -100,7 +101,7 @@ public class MetaDataEvolutionValidator { private final boolean disallowTypeRenames; private MetaDataEvolutionValidator() { - this.indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); + this.indexValidatorRegistry = IndexMaintainerRegistryImpl.instance(); this.allowNoVersionChange = false; this.allowNoSinceVersion = false; this.allowIndexRebuilds = false; @@ -111,7 +112,7 @@ private MetaDataEvolutionValidator() { } private MetaDataEvolutionValidator(@Nonnull Builder builder) { - this.indexMaintainerRegistry = builder.indexMaintainerRegistry; + this.indexValidatorRegistry = builder.indexValidatorRegistry; this.allowNoVersionChange = builder.allowNoVersionChange; this.allowNoSinceVersion = builder.allowNoSinceVersion; this.allowIndexRebuilds = builder.allowIndexRebuilds; @@ -668,7 +669,7 @@ private void validateIndex(@Nonnull RecordMetaData oldMetaData, @Nonnull Index o // If there have been any changes to the index options, ask the index validator for that type // to validate the changed options. if (!oldIndex.getOptions().equals(newIndex.getOptions())) { - IndexValidator validatorForIndex = indexMaintainerRegistry.getIndexValidator(newIndex); + IndexValidator validatorForIndex = indexValidatorRegistry.getIndexValidator(newIndex); validatorForIndex.validateChangedOptions(oldIndex); } } @@ -679,11 +680,11 @@ private void validateIndex(@Nonnull RecordMetaData oldMetaData, @Nonnull Index o * the meta-data. By default, this uses the default {@link IndexMaintainerRegistryImpl} instance. * * @return the index maintainer registry used to validate indexes - * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerRegistry) + * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerFactoryRegistry) */ @Nonnull - public IndexMaintainerRegistry getIndexMaintainerRegistry() { - return indexMaintainerRegistry; + public IndexValidatorRegistry getIndexValidatorRegistry() { + return indexValidatorRegistry; } /** @@ -823,7 +824,7 @@ public static MetaDataEvolutionValidator getDefaultInstance() { */ public static class Builder { @Nonnull - private IndexMaintainerRegistry indexMaintainerRegistry; + private IndexValidatorRegistry indexValidatorRegistry; private boolean allowNoVersionChange; private boolean allowNoSinceVersion; private boolean allowIndexRebuilds; @@ -833,7 +834,7 @@ public static class Builder { private boolean disallowTypeRenames; private Builder(@Nonnull MetaDataEvolutionValidator validator) { - this.indexMaintainerRegistry = validator.indexMaintainerRegistry; + this.indexValidatorRegistry = validator.indexValidatorRegistry; this.allowNoVersionChange = validator.allowNoVersionChange; this.allowNoSinceVersion = validator.allowNoSinceVersion; this.allowIndexRebuilds = validator.allowIndexRebuilds; @@ -844,29 +845,29 @@ private Builder(@Nonnull MetaDataEvolutionValidator validator) { } /** - * Set the registry of index maintainers used to validate indexes. + * Set the registry of index validators used to validate indexes. * - * @param indexMaintainerRegistry the index maintainer registry used to validate indexes + * @param indexValidatorRegistry the index validator registry used to validate indexes * @return this builder - * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerRegistry) - * @see MetaDataEvolutionValidator#getIndexMaintainerRegistry() + * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerFactoryRegistry) + * @see MetaDataEvolutionValidator#getIndexValidatorRegistry() */ @Nonnull - public Builder setIndexMaintainerRegistry(@Nonnull IndexMaintainerRegistry indexMaintainerRegistry) { - this.indexMaintainerRegistry = indexMaintainerRegistry; + public Builder setIndexValidatorRegistry(@Nonnull IndexMaintainerRegistry indexValidatorRegistry) { + this.indexValidatorRegistry = indexValidatorRegistry; return this; } /** - * Get the registry of index maintainers used to validate indexes. + * Get the registry of index validators used to validate indexes. * * @return the index maintainer registry used to validate indexes - * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerRegistry) - * @see MetaDataEvolutionValidator#getIndexMaintainerRegistry() + * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerFactoryRegistry) + * @see MetaDataEvolutionValidator#getIndexValidatorRegistry() */ @Nonnull - public IndexMaintainerRegistry getIndexMaintainerRegistry() { - return indexMaintainerRegistry; + public IndexValidatorRegistry getIndexValidatorRegistry() { + return indexValidatorRegistry; } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java index 9734e9bcb5..ff0211c535 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java @@ -282,7 +282,7 @@ public class FDBRecordStore extends FDBStoreBase implements FDBRecordStoreBase serializer; @Nonnull - protected final IndexMaintainerRegistry indexMaintainerRegistry; + protected final IndexMaintainerFactoryRegistry indexMaintainerRegistry; @Nonnull protected final IndexMaintenanceFilter indexMaintenanceFilter; @@ -322,7 +322,7 @@ protected FDBRecordStore(@Nonnull FDBRecordContext context, @Nonnull FormatVersion formatVersion, @Nonnull RecordMetaDataProvider metaDataProvider, @Nonnull RecordSerializer serializer, - @Nonnull IndexMaintainerRegistry indexMaintainerRegistry, + @Nonnull IndexMaintainerFactoryRegistry indexMaintainerRegistry, @Nonnull IndexMaintenanceFilter indexMaintenanceFilter, @Nonnull PipelineSizer pipelineSizer, @Nullable FDBRecordStoreStateCache storeStateCache, @@ -463,8 +463,9 @@ public RecordSerializer getSerializer() { return serializer; } + @Override @Nonnull - public IndexMaintainerRegistry getIndexMaintainerRegistry() { + public IndexMaintainerFactoryRegistry getIndexMaintainerRegistry() { return indexMaintainerRegistry; } @@ -5326,7 +5327,7 @@ public static class Builder implements BaseBuilder { private FDBRecordStoreBase.UserVersionChecker userVersionChecker; @Nonnull - private IndexMaintainerRegistry indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); + private IndexMaintainerFactoryRegistry indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); @Nonnull private IndexMaintenanceFilter indexMaintenanceFilter = IndexMaintenanceFilter.NORMAL; @@ -5523,13 +5524,13 @@ public Builder setUserVersionChecker(@Nullable UserVersionChecker userVersionChe @Override @Nonnull - public IndexMaintainerRegistry getIndexMaintainerRegistry() { + public IndexMaintainerFactoryRegistry getIndexMaintainerRegistry() { return indexMaintainerRegistry; } @Override @Nonnull - public Builder setIndexMaintainerRegistry(@Nonnull IndexMaintainerRegistry indexMaintainerRegistry) { + public Builder setIndexMaintainerRegistry(@Nonnull IndexMaintainerFactoryRegistry indexMaintainerRegistry) { this.indexMaintainerRegistry = indexMaintainerRegistry; return this; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java index 234a7b82c3..d3d58a898b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java @@ -186,6 +186,8 @@ default FDBStoreTimer getTimer() { @Nonnull RecordSerializer getSerializer(); + @Nonnull + IndexMaintainerFactoryRegistry getIndexMaintainerRegistry(); /** * Returns the index maintainer for a given index. @@ -2328,7 +2330,7 @@ default BaseBuilder setFormatVersion(FormatVersion formatVersion) { * @return the index registry to use */ @Nonnull - IndexMaintainerRegistry getIndexMaintainerRegistry(); + IndexMaintainerFactoryRegistry getIndexMaintainerRegistry(); /** * Set the registry of index maintainers to be used by the record store. @@ -2338,7 +2340,7 @@ default BaseBuilder setFormatVersion(FormatVersion formatVersion) { * @see RecordMetaDataBuilder#setIndexMaintainerRegistry */ @Nonnull - BaseBuilder setIndexMaintainerRegistry(@Nonnull IndexMaintainerRegistry indexMaintainerRegistry); + BaseBuilder setIndexMaintainerRegistry(@Nonnull IndexMaintainerFactoryRegistry indexMaintainerRegistry); /** * Get the {@link IndexMaintenanceFilter index filter} to be used by the record store. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStore.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStore.java index ffb3173d60..c02ec623a2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStore.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStore.java @@ -125,6 +125,13 @@ public RecordSerializer getSerializer() { return typedSerializer; } + + @Nonnull + @Override + public IndexMaintainerFactoryRegistry getIndexMaintainerRegistry() { + return untypedStore.getIndexMaintainerRegistry(); + } + @Nonnull @Override public IndexMaintainer getIndexMaintainer(@Nonnull final Index index) { @@ -497,13 +504,13 @@ public Builder setUserVersionChecker(@Nullable UserVersionChecker userVersion @Nonnull @Override - public IndexMaintainerRegistry getIndexMaintainerRegistry() { + public IndexMaintainerFactoryRegistry getIndexMaintainerRegistry() { return untypedStoreBuilder.getIndexMaintainerRegistry(); } @Nonnull @Override - public Builder setIndexMaintainerRegistry(@Nonnull IndexMaintainerRegistry indexMaintainerRegistry) { + public Builder setIndexMaintainerRegistry(@Nonnull IndexMaintainerFactoryRegistry indexMaintainerRegistry) { untypedStoreBuilder.setIndexMaintainerRegistry(indexMaintainerRegistry); return this; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java index a471554aee..020fe31242 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java @@ -47,7 +47,7 @@ public class IndexMaintainerRegistryImpl implements IndexMaintainerFactoryRegist private final Map registry; @Nonnull - public static IndexMaintainerRegistry instance() { + public static IndexMaintainerFactoryRegistry instance() { return INSTANCE; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java index 3d6ccd9f6f..4991a525d6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; @@ -33,6 +34,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; import com.google.protobuf.Descriptors; @@ -151,4 +154,9 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new AtomicMutationIndexMaintainer(state); } + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + return MatchCandidateExpansion.expandAggregateIndexMatchCandidate(metaData, index, reverse); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java index a61a60aa7b..ba2f9cdd92 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; @@ -32,6 +33,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; import com.google.protobuf.Descriptors; @@ -104,4 +107,10 @@ public void validateIndexForRecordType(@Nonnull RecordType recordType, @Nonnull public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new BitmapValueIndexMaintainer(state); } + + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + return MatchCandidateExpansion.expandAggregateIndexMatchCandidate(metaData, index, reverse); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java index cfcc0b9761..c4763a93fa 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexOptions; @@ -32,7 +33,10 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import java.util.Arrays; @@ -88,4 +92,19 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new PermutedMinMaxIndexMaintainer(state); } + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final ImmutableList.Builder resultBuilder = ImmutableList.builderWithExpectedSize(2); + + // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use + // the aggregate index expansion for BY_GROUP scans + MatchCandidateExpansion.expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + MatchCandidateExpansion.expandAggregateIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + + return resultBuilder.build(); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java index 6256a684eb..8718fa7ff0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.async.RankedSet; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexOptions; @@ -32,7 +33,11 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; +import com.apple.foundationdb.record.query.plan.cascades.WindowedIndexExpansionVisitor; import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import java.util.Arrays; @@ -102,4 +107,18 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new RankIndexMaintainer(state); } + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final ImmutableList.Builder resultBuilder = ImmutableList.builder(); + + // For rank() we need to create at two candidates. One for BY_RANK scans and one for BY_VALUE scans. + MatchCandidateExpansion.expandValueIndexMatchCandidate(info) + .ifPresent(resultBuilder::add); + MatchCandidateExpansion.expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new WindowedIndexExpansionVisitor(index, info.getIndexedRecordTypes())) + .ifPresent(resultBuilder::add); + + return resultBuilder.build(); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/ValueIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/ValueIndexMaintainerFactory.java index e857c8d943..4a6b5f593b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/ValueIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/ValueIndexMaintainerFactory.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.IndexValidator; @@ -28,6 +29,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; @@ -65,4 +68,10 @@ public void validate(@Nonnull MetaDataValidator metaDataValidator) { public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new ValueIndexMaintainer(state); } + + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + return MatchCandidateExpansion.expandValueIndexMatchCandidate(metaData, index, reverse); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexMaintainerFactory.java index 56fa6ee782..34081aec10 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/VersionIndexMaintainerFactory.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.IndexValidator; @@ -28,6 +29,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; import javax.annotation.Nonnull; @@ -67,4 +70,10 @@ public void validate(@Nonnull MetaDataValidator metaDataValidator) { public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { return new VersionIndexMaintainer(state); } + + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + return MatchCandidateExpansion.expandValueIndexMatchCandidate(metaData, index, reverse); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java index fb9f7b51a0..8490b949d2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java @@ -26,6 +26,8 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.ParameterRelationshipGraph; import com.apple.foundationdb.record.query.RecordQuery; @@ -213,6 +215,8 @@ public class CascadesPlanner implements QueryPlanner { @Nonnull private final RecordStoreState recordStoreState; @Nonnull + private final IndexMatchCandidateRegistry matchCandidateRegistry; + @Nonnull private Reference currentRoot; @Nonnull private PlanContext planContext; @@ -227,10 +231,11 @@ public class CascadesPlanner implements QueryPlanner { // max size of the task queue encountered during the planning private int maxQueueSize; - public CascadesPlanner(@Nonnull RecordMetaData metaData, @Nonnull RecordStoreState recordStoreState) { + public CascadesPlanner(@Nonnull RecordMetaData metaData, @Nonnull RecordStoreState recordStoreState, @Nonnull IndexMatchCandidateRegistry matchCandidateRegistry) { this.configuration = RecordQueryPlannerConfiguration.builder().build(); this.metaData = metaData; this.recordStoreState = recordStoreState; + this.matchCandidateRegistry = matchCandidateRegistry; // Placeholders until we get a query. this.currentRoot = Reference.empty(); this.planContext = PlanContext.emptyContext(); @@ -239,6 +244,10 @@ public CascadesPlanner(@Nonnull RecordMetaData metaData, @Nonnull RecordStoreSta this.taskStack = new ArrayDeque<>(); } + public static CascadesPlanner forStore(@Nonnull FDBRecordStoreBase recordStore) { + return new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState(), recordStore.getIndexMaintainerRegistry()); + } + @Nonnull @Override public RecordMetaData getRecordMetaData() { @@ -344,7 +353,7 @@ public RecordQueryPlan plan(@Nonnull final RecordQuery query, @Nonnull final ParameterRelationshipGraph parameterRelationshipGraph) { try { planPartial(() -> Reference.initialOf(RelationalExpression.fromRecordQuery(metaData, query)), - rootReference -> MetaDataPlanContext.forRecordQuery(configuration, metaData, recordStoreState, query), + rootReference -> MetaDataPlanContext.forRecordQuery(configuration, metaData, recordStoreState, matchCandidateRegistry, query), EvaluationContext.empty()); return resultOrFail(); } finally { @@ -364,6 +373,7 @@ public QueryPlanResult planGraph(@Nonnull final Supplier referenceSup MetaDataPlanContext.forRootReference(configuration, metaData, recordStoreState, + matchCandidateRegistry, rootReference, allowedIndexesOptional, indexQueryabilityFilter diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java index 776bf608db..a8119fcec3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java @@ -49,56 +49,26 @@ private MatchCandidateExpansion() { } @Nonnull - public static Iterable fromIndexDefinition(@Nonnull final RecordMetaData metaData, - @Nonnull final Index index, - final boolean isReverse) { - final IndexExpansionInfo info = IndexExpansionInfo.forIndex(metaData, index, isReverse); - final ImmutableList.Builder resultBuilder = ImmutableList.builder(); - - switch (info.getIndexType()) { - case IndexTypes.VALUE: - case IndexTypes.VERSION: - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.RANK: - // For rank() we need to create at least two candidates. One for BY_RANK scans and one for BY_VALUE scans. - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - - expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new WindowedIndexExpansionVisitor(index, info.getIndexedRecordTypes())) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.MIN_EVER_TUPLE: // fallthrough - case IndexTypes.MAX_EVER_TUPLE: // fallthrough - case IndexTypes.MAX_EVER_LONG: // fallthrough - case IndexTypes.MIN_EVER_LONG: // fallthrough - case IndexTypes.BITMAP_VALUE: // fallthrough - case IndexTypes.SUM: // fallthrough - case IndexTypes.COUNT: // fallthrough - case IndexTypes.COUNT_NOT_NULL: - expandAggregateIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - case IndexTypes.PERMUTED_MAX: // fallthrough - case IndexTypes.PERMUTED_MIN: - // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use - // the aggregate index expansion for BY_GROUP scans - expandValueIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - expandAggregateIndexMatchCandidate(info) - .ifPresent(resultBuilder::add); - break; - default: - break; - } - return resultBuilder.build(); + public static Iterable expandValueIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { + final IndexExpansionInfo info = createInfo(metaData, index, isReverse); + return expandValueIndexMatchCandidate(info) + .map(ImmutableList::of) + .orElse(ImmutableList.of()); } public static Optional expandValueIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); } + @Nonnull + public static Iterable expandAggregateIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { + final IndexExpansionInfo info = createInfo(metaData, index, isReverse); + return expandAggregateIndexMatchCandidate(info) + .map(ImmutableList::of) + .orElse(ImmutableList.of()); + } + + @Nonnull public static Optional expandAggregateIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(info.getIndexType()) ? new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()) @@ -109,9 +79,9 @@ public static Optional expandAggregateIndexMatchCandidate(@Nonnu } @Nonnull - private static Optional expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info, - @Nullable KeyExpression commonPrimaryKey, - @Nonnull final ExpansionVisitor expansionVisitor) { + public static Optional expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info, + @Nullable KeyExpression commonPrimaryKey, + @Nonnull final ExpansionVisitor expansionVisitor) { final var baseRef = createBaseRef(info, new IndexAccessHint(info.getIndexName())); try { return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKey, info.isReverse())); @@ -247,32 +217,32 @@ public KeyExpression getCommonPrimaryKeyForTypes() { public Set getAvailableRecordTypeNames() { return metaData.getRecordTypes().keySet(); } + } - /** - * Create an {@link IndexExpansionInfo} for a given index. - * This wraps the given parameters into a single object, as well - * as enriching the given parameters with pre-calculated items that - * can then be used during index expansion. - * - * @param metaData the meta-data that is the source of the index - * @param index the index that we are expanding - * @param reverse whether the query requires this scan be in reverse - * @return an object encapsulating information about the index - */ + /** + * Create an {@link IndexExpansionInfo} for a given index. + * This wraps the given parameters into a single object, as well + * as enriching the given parameters with pre-calculated items that + * can then be used during index expansion. + * + * @param metaData the meta-data that is the source of the index + * @param index the index that we are expanding + * @param reverse whether the query requires this scan be in reverse + * @return an object encapsulating information about the index + */ + @Nonnull + public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse) { @Nonnull - public static IndexExpansionInfo forIndex(@Nonnull RecordMetaData metaData, - @Nonnull Index index, - boolean reverse) { - @Nonnull - final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); - @Nonnull - final Set indexedRecordTypeNames = indexedRecordTypes.stream() - .map(RecordType::getName) - .collect(ImmutableSet.toImmutableSet()); - @Nullable - final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); + final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); + @Nonnull + final Set indexedRecordTypeNames = indexedRecordTypes.stream() + .map(RecordType::getName) + .collect(ImmutableSet.toImmutableSet()); + @Nullable + final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); - return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); - } + return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index 7e4d7b7420..1450ec1814 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; @@ -104,6 +105,7 @@ private static List readableOf(@Nonnull RecordStoreState recordStoreState public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull RecordMetaData metaData, @Nonnull RecordStoreState recordStoreState, + @Nonnull IndexMatchCandidateRegistry matchCandidateRegistry, @Nonnull RecordQuery query) { final Optional> queriedRecordTypeNamesOptional = query.getRecordTypes().isEmpty() ? Optional.empty() : Optional.of(query.getRecordTypes()); final Optional> allowedIndexesOptional = query.hasAllowedIndexes() ? Optional.of(Objects.requireNonNull(query.getAllowedIndexes())) : Optional.empty(); @@ -154,9 +156,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio final ImmutableSet.Builder matchCandidatesBuilder = ImmutableSet.builder(); for (Index index : indexList) { - final Iterable candidatesForIndex = - MatchCandidateExpansion.fromIndexDefinition(metaData, index, isSortReverse); - matchCandidatesBuilder.addAll(candidatesForIndex); + matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, isSortReverse)); } MatchCandidateExpansion.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse) @@ -168,6 +168,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull final RecordMetaData metaData, @Nonnull final RecordStoreState recordStoreState, + @Nonnull final IndexMatchCandidateRegistry matchCandidateRegistry, @Nonnull final Reference rootReference, @Nonnull final Optional> allowedIndexesOptional, @Nonnull final IndexQueryabilityFilter indexQueryabilityFilter) { @@ -198,9 +199,7 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf final ImmutableSet.Builder matchCandidatesBuilder = ImmutableSet.builder(); for (final var index : indexList) { - final Iterable candidatesForIndex = - MatchCandidateExpansion.fromIndexDefinition(metaData, index, false); - matchCandidatesBuilder.addAll(candidatesForIndex); + matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, false)); } for (final var recordType : queriedRecordTypes) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java index 48361388c5..9ee28ff507 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java @@ -32,7 +32,6 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder; import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges; import com.apple.foundationdb.record.query.plan.cascades.values.Value; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -75,7 +74,6 @@ public class ValueIndexExpansionVisitor extends KeyExpressionExpansionVisitor im private final List queriedRecordTypes; public ValueIndexExpansionVisitor(@Nonnull Index index, @Nonnull Collection queriedRecordTypes) { - Preconditions.checkArgument(SUPPORTED_INDEX_TYPES.contains(index.getType())); this.index = index; this.queriedRecordTypes = ImmutableList.copyOf(queriedRecordTypes); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java index 06d037edf0..eaac02610f 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java @@ -101,7 +101,7 @@ protected FDBRecordStore createOrOpenRecordStore(@Nonnull FDBRecordContext conte public QueryPlanner setupPlanner(@Nonnull FDBRecordStore recordStore, @Nullable PlannableIndexTypes indexTypes) { final QueryPlanner planner; if (useCascadesPlanner) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState()); + planner = CascadesPlanner.forStore(recordStore); if (Debugger.getDebugger() == null) { Debugger.setDebugger(DebuggerWithSymbolTables.withSanityChecks()); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java index e51a10d216..821448b8cc 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.metadata.IndexOptions; import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.Key; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.plan.QueryPlanner; @@ -58,7 +59,7 @@ protected void setup(@Nullable RecordMetaDataHook hook) { } metaData = builder.getRecordMetaData(); planner = isUseCascadesPlanner() ? - new CascadesPlanner(metaData, new RecordStoreState(null, null)) : + new CascadesPlanner(metaData, new RecordStoreState(null, null), IndexMaintainerRegistryImpl.instance()) : new RecordQueryPlanner(metaData, new RecordStoreState(null, null)); } diff --git a/fdb-record-layer-jmh/src/jmh/java/com/apple/foundationdb/record/benchmark/BenchmarkRecordStore.java b/fdb-record-layer-jmh/src/jmh/java/com/apple/foundationdb/record/benchmark/BenchmarkRecordStore.java index b201deb4c3..9f43bb3924 100644 --- a/fdb-record-layer-jmh/src/jmh/java/com/apple/foundationdb/record/benchmark/BenchmarkRecordStore.java +++ b/fdb-record-layer-jmh/src/jmh/java/com/apple/foundationdb/record/benchmark/BenchmarkRecordStore.java @@ -168,7 +168,7 @@ public QueryPlanner planner(@Nonnull BenchmarkTimer timer, boolean cascades) { final RecordMetaData recordMetaData = recordStoreBuilder.getMetaDataProvider().getRecordMetaData(); final RecordStoreState recordStoreState = new RecordStoreState(null, null); if (cascades) { - return new CascadesPlanner(recordMetaData, recordStoreState); + return new CascadesPlanner(recordMetaData, recordStoreState, recordStoreBuilder.getIndexMaintainerRegistry()); } else { return new RecordQueryPlanner(recordMetaData, recordStoreState, timer.getTimer()); } diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java index 16753095f1..65e0a07422 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java @@ -213,7 +213,7 @@ public static void setup() { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState()); + planner = CascadesPlanner.forStore(recordStore); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java index d1287c485a..415c39aebd 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java @@ -37,7 +37,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase; import com.apple.foundationdb.record.provider.foundationdb.FormatVersion; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistry; import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer; import com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; @@ -566,7 +566,7 @@ public static Pair rebuildIndexMetaData(final FDBR final String document, final Index index, boolean useCascadesPlanner, - @Nullable IndexMaintainerRegistry indexMaintainerRegistry) { + @Nullable IndexMaintainerFactoryRegistry indexMaintainerRegistry) { FDBRecordStore store = openRecordStore(context, path, metaDataBuilder -> { @@ -588,7 +588,7 @@ static FDBRecordStore openRecordStore(FDBRecordContext context, static FDBRecordStore openRecordStore(FDBRecordContext context, @Nonnull KeySpacePath path, FDBRecordStoreTestBase.RecordMetaDataHook hook, - @Nullable IndexMaintainerRegistry indexMaintainerRegistry) { + @Nullable IndexMaintainerFactoryRegistry indexMaintainerRegistry) { RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsTextProto.getDescriptor()); metaDataBuilder.getRecordType(COMPLEX_DOC).setPrimaryKey(concatenateFields("group", "doc_id")); hook.apply(metaDataBuilder); @@ -601,7 +601,7 @@ static FDBRecordStore openRecordStore(FDBRecordContext context, private static FDBRecordStore.Builder getStoreBuilder(@Nonnull FDBRecordContext context, @Nonnull KeySpacePath path, @Nonnull RecordMetaData metaData, - @Nullable IndexMaintainerRegistry indexMaintainerRegistry) { + @Nullable IndexMaintainerFactoryRegistry indexMaintainerRegistry) { final FDBRecordStore.Builder builder = FDBRecordStore.newBuilder() .setFormatVersion(FormatVersion.getMaximumSupportedVersion()) // set to max to test newest features (unsafe for real deployments) .setKeySpacePath(path) @@ -617,7 +617,7 @@ static QueryPlanner setupPlanner(@Nonnull FDBRecordStore recordStore, @Nullable PlannableIndexTypes indexTypes, boolean useRewritePlanner) { QueryPlanner planner; if (useRewritePlanner) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState()); + planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState(), recordStore.getIndexMaintainerRegistry()); if (Debugger.getDebugger() == null) { Debugger.setDebugger(DebuggerWithSymbolTables.withSanityChecks()); } diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java index 8e8bb03056..cc2260d70d 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java @@ -78,7 +78,7 @@ public class LuceneQueryIntegrationTest extends FDBRecordStoreQueryTestBase { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState()); + planner = CascadesPlanner.forStore(recordStore); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/TestingIndexMaintainerRegistry.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/TestingIndexMaintainerRegistry.java index d260ccff50..fd9c45479e 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/TestingIndexMaintainerRegistry.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/TestingIndexMaintainerRegistry.java @@ -23,12 +23,10 @@ import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; -import com.apple.foundationdb.record.metadata.IndexValidator; import com.apple.foundationdb.record.metadata.MetaDataException; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; import com.apple.foundationdb.record.util.ServiceLoaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +39,7 @@ * A testing-oriented version of {@link IndexMaintainerRegistry} that allows overriding a registry. * This can be used in place of the production registry */ -public class TestingIndexMaintainerRegistry implements IndexMaintainerRegistry { +public class TestingIndexMaintainerRegistry implements IndexMaintainerFactoryRegistry { @Nonnull private static final Logger LOGGER = LoggerFactory.getLogger(TestingIndexMaintainerRegistry.class); @@ -54,22 +52,12 @@ public TestingIndexMaintainerRegistry() { @Nonnull @Override - public IndexValidator getIndexValidator(@Nonnull Index index) { + public IndexMaintainerFactory getIndexMaintainerFactory(@Nonnull final Index index) { final IndexMaintainerFactory factory = registry.get(index.getType()); if (factory == null) { throw new MetaDataException("Unknown index type for " + index); } - return factory.getIndexValidator(index); - } - - @Nonnull - @Override - public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { - final IndexMaintainerFactory factory = registry.get(state.index.getType()); - if (factory == null) { - throw new MetaDataException("Unknown index type for " + state.index); - } - return factory.getIndexMaintainer(state); + return factory; } /** diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java index d8a8f99557..60321e7165 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java @@ -254,7 +254,7 @@ protected void clear() { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState()); + planner = CascadesPlanner.forStore(recordStore); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/AbstractEmbeddedStatement.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/AbstractEmbeddedStatement.java index 46465497ba..140a86496f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/AbstractEmbeddedStatement.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/AbstractEmbeddedStatement.java @@ -84,8 +84,8 @@ public boolean executeInternal(String sql) throws SQLException, RelationalExcept final var store = schema.loadStore().unwrap(FDBRecordStoreBase.class); final var planGenerator = PlanGenerator.create(planCacheMaybe, createPlanContext(store, options), - store.getRecordMetaData(), - store.getRecordStoreState(), this.options); + store, + this.options); final Plan plan = planGenerator.getPlan(sql); final var executionContext = Plan.ExecutionContext.of(conn.getTransaction(), planGenerator.getOptions(), conn, metricCollector); if (plan instanceof QueryPlan) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 8ecb427f95..4edebd3e72 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -27,6 +27,8 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.metadata.MetaDataException; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.SemanticException; @@ -387,15 +389,6 @@ private static boolean shouldNotCache(@Nonnull final Set @Nonnull final PlanContext planContext, @Nonnull final RecordMetaData metaData, @Nonnull final RecordStoreState recordStoreState, + @Nonnull final IndexMatchCandidateRegistry matchCandidateRegistry, @Nonnull final Options options) throws RelationalException { - final var planner = createPlanner(metaData, recordStoreState, planContext); + final var planner = new CascadesPlanner(metaData, recordStoreState, matchCandidateRegistry); + planner.setConfiguration(planContext.getRecordQueryPlannerConfiguration()); return new PlanGenerator(cache, planContext, planner, options); } + + @Nonnull + public static PlanGenerator create(@Nonnull final Optional cache, + @Nonnull final PlanContext planContext, + @Nonnull FDBRecordStoreBase store, + @Nonnull final Options options) throws RelationalException { + return create(cache, planContext, store.getRecordMetaData(), store.getRecordStoreState(), store.getIndexMaintainerRegistry(), options); + } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java index 2305644cb1..d6cd9bf8fb 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordStoreState; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalConnection; @@ -85,7 +86,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); } } @@ -108,7 +109,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); } } @@ -122,7 +123,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java index 0d1c96b501..b5693037fb 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.memory; import com.apple.foundationdb.record.RecordStoreState; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.Row; @@ -115,7 +116,7 @@ public T clock(@Nonnull RelationalMetric.RelationalEvent event, Supplier .withUserVersion(0) .build(); - final var planGenerator = PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), Options.NONE); + final var planGenerator = PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerRegistryImpl.instance(), Options.NONE); final Plan plan = planGenerator.getPlan(sql); if (plan instanceof QueryPlan) { throw new SQLFeatureNotSupportedException("Cannot execute queries in the InMemory Relational version, it's only good for Direct Access API"); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/PlanGenerationStackTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/PlanGenerationStackTest.java index 6d7987870a..6314eb76bb 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/PlanGenerationStackTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/PlanGenerationStackTest.java @@ -185,10 +185,10 @@ void queryTestHarness(int ignored, @Nonnull String query, @Nullable String error .withMetricsCollector(embeddedConnection.getMetricCollector()) .build(); if (error == null) { - PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store.getRecordMetaData(), store.getRecordStoreState(), Options.NONE); + PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store, Options.NONE); final Plan generatedPlan1 = planGenerator.getPlan(query); final var queryHash1 = ((QueryPlan.PhysicalQueryPlan) generatedPlan1).getRecordQueryPlan().semanticHashCode(); - planGenerator = PlanGenerator.create(Optional.empty(), planContext, store.getRecordMetaData(), store.getRecordStoreState(), Options.NONE); + planGenerator = PlanGenerator.create(Optional.empty(), planContext, store, Options.NONE); final Plan generatedPlan2 = planGenerator.getPlan(query); final var queryHash2 = ((QueryPlan.PhysicalQueryPlan) generatedPlan2).getRecordQueryPlan().semanticHashCode(); embeddedConnection.rollback(); @@ -196,7 +196,7 @@ void queryTestHarness(int ignored, @Nonnull String query, @Nullable String error Assertions.assertEquals(queryHash1, queryHash2); } else { try { - PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store.getRecordMetaData(), store.getRecordStoreState(), Options.NONE); + PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store, Options.NONE); planGenerator.getPlan(query); Assertions.fail("expected an exception to be thrown"); } catch (RelationalException e) { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java index e6397ecbd5..b4a2da32f2 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java @@ -25,6 +25,7 @@ import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.query.plan.cascades.RawSqlFunction; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.Options; @@ -610,7 +611,7 @@ public T clock(@Nonnull RelationalMetric.RelationalEvent event, .withPlannerConfiguration(PlannerConfiguration.ofAllAvailableIndexes()) .withUserVersion(0) .build(); - return PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), Options.NONE); + return PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerRegistryImpl.instance(), Options.NONE); } } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/ConstraintValidityTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/ConstraintValidityTests.java index 9875c5b522..b773aff8e5 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/ConstraintValidityTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/ConstraintValidityTests.java @@ -119,7 +119,7 @@ private PlanGenerator getPlanGenerator(@Nonnull final RelationalPlanCache cache) .withMetricsCollector(embeddedConnection.getMetricCollector()) .withUserVersion(44) .build(); - return PlanGenerator.create(Optional.of(cache), planContext, store.getRecordMetaData(), storeState, Options.builder().build()); + return PlanGenerator.create(Optional.of(cache), planContext, store.getRecordMetaData(), storeState, store.getIndexMaintainerRegistry(), Options.builder().build()); } private void planQuery(@Nonnull final RelationalPlanCache cache, diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java index 338a8ee709..61e65fc712 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java @@ -371,7 +371,7 @@ private PlanGenerator getPlanGenerator(@Nonnull final RelationalPlanCache cache, .withPlannerConfiguration(PlannerConfiguration.of(Optional.of(readableIndexes), options)) .withUserVersion(userVersion) .build(); - return PlanGenerator.create(Optional.of(cache), planContext, store.getRecordMetaData(), storeState, options); + return PlanGenerator.create(Optional.of(cache), planContext, store.getRecordMetaData(), storeState, store.getIndexMaintainerRegistry(), options); } @Nonnull diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/structuredsql/SqlVisitorTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/structuredsql/SqlVisitorTests.java index 3c41f19431..028131a31c 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/structuredsql/SqlVisitorTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/structuredsql/SqlVisitorTests.java @@ -176,7 +176,7 @@ private void isValidStatement(@Nonnull final RelationalConnection connection, @N .withSchemaTemplate(embeddedConnection.getSchemaTemplate()) .withMetricsCollector(embeddedConnection.getMetricCollector()) .build(); - final PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store.getRecordMetaData(), store.getRecordStoreState(), Options.NONE); + final PlanGenerator planGenerator = PlanGenerator.create(Optional.empty(), planContext, store, Options.NONE); Assertions.assertDoesNotThrow(() -> planGenerator.getPlan(query)); } } From 422a5f2d8817cafe4cf3469734359c00a73864e7 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 29 Sep 2025 15:00:52 +0100 Subject: [PATCH 4/9] add test that shows we can match a value-like index defined outside of the core --- .../IndexMatchCandidateRegistry.java | 2 +- .../AtomicMutationIndexMaintainerFactory.java | 8 +- .../BitmapValueIndexMaintainerFactory.java | 7 +- .../AggregateIndexExpansionVisitor.java | 16 +- .../cascades/MatchCandidateExpansion.java | 45 ++- .../cascades/ValueIndexExpansionVisitor.java | 12 +- .../OutsideValueLikeIndexMaintainer.java | 92 +++++ .../OutsideValueLikeIndexQueryTest.java | 318 ++++++++++++++++++ 8 files changed, 474 insertions(+), 26 deletions(-) create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexMaintainer.java create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java index 23bc588f51..419d85a5e6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java @@ -33,7 +33,7 @@ * without needing to be defined in the core Record Layer repository. * * @see IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean) - * @see IndexMaintainerFactoryRegistry for a sub-interface that delegates to the {@link IndexMaintainerFactory} + * @see IndexMaintainerFactoryRegistry for a sub-interface that delegates to the index maintainer factory * @see IndexMaintainerRegistryImpl for the default implementation */ public interface IndexMatchCandidateRegistry { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java index 4991a525d6..1489d1b3ff 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/AtomicMutationIndexMaintainerFactory.java @@ -34,9 +34,11 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.AggregateIndexExpansionVisitor; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; import com.google.protobuf.Descriptors; import javax.annotation.Nonnull; @@ -157,6 +159,10 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { @Nonnull @Override public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { - return MatchCandidateExpansion.expandAggregateIndexMatchCandidate(metaData, index, reverse); + if (AggregateIndexExpansionVisitor.supportsAggregateIndexType(index.getType())) { + return MatchCandidateExpansion.expandAggregateIndexMatchCandidate(metaData, index, reverse); + } else { + return ImmutableList.of(); + } } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java index ba2f9cdd92..bb023133d3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java @@ -33,6 +33,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.BitmapAggregateIndexExpansionVisitor; +import com.apple.foundationdb.record.query.plan.cascades.ExpansionVisitor; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; @@ -111,6 +113,9 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { @Nonnull @Override public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { - return MatchCandidateExpansion.expandAggregateIndexMatchCandidate(metaData, index, reverse); + final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final ExpansionVisitor expansionVisitor = new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); + return MatchCandidateExpansion.optionalToIterable( + MatchCandidateExpansion.expandIndexMatchCandidate(info, null, expansionVisitor)); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java index 5b3c71074e..86fe88a819 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.query.plan.cascades; +import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.logging.LogMessageKeys; @@ -98,8 +99,8 @@ public class AggregateIndexExpansionVisitor extends KeyExpressionExpansionVisito * @param recordTypes The indexed record types. */ public AggregateIndexExpansionVisitor(@Nonnull final Index index, @Nonnull final Collection recordTypes) { - Preconditions.checkArgument(IndexTypes.BITMAP_VALUE.equals(index.getType()) || - aggregateMap.get().containsKey(index.getType())); + Preconditions.checkArgument(supportsAggregateIndexType(index.getType()), + "Unsupported index aggregate type %s", index.getType()); Preconditions.checkArgument(index.getRootExpression() instanceof GroupingKeyExpression); this.index = index; this.groupingKeyExpression = ((GroupingKeyExpression)index.getRootExpression()); @@ -341,6 +342,17 @@ private ConstructSelectHavingResult constructSelectHaving(@Nonnull final Quantif finalPlaceholders); } + /** + * Whether the aggregate index expansion visitor knows how to expand the given index type. + * + * @param indexType the index type to try and expand + * @return whether this class can expand index of the given type + */ + @API(API.Status.INTERNAL) + public static boolean supportsAggregateIndexType(@Nonnull String indexType) { + return IndexTypes.BITMAP_VALUE.equals(indexType) || aggregateMap.get().containsKey(indexType); + } + @Nonnull public static Optional aggregateValue(@Nonnull final Index index, @Nonnull final Value argument) { return Optional.of((AggregateValue)aggregateMap.get() diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java index a8119fcec3..bcd9cff558 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.metadata.Index; -import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; @@ -42,40 +41,62 @@ import java.util.Optional; import java.util.Set; +/** + * Utility methods for expanding certain indexes into {@link MatchCandidate}s. This should be used by + * methods like {@link com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean)} + * to create the match candidate that will be used by the {@link CascadesPlanner} during query + * planning. Note that indexes may create multiple match candidates, but individual {@link ExpansionVisitor}s + * will return at most one candidate. So certain methods here return an {@link Optional}. The utility + * method {@link #optionalToIterable(Optional)} can be used to turn that {@link Optional} into s + * list containing the one candidate (if set). + */ +@API(API.Status.INTERNAL) public class MatchCandidateExpansion { + @Nonnull private static final Logger LOGGER = LoggerFactory.getLogger(MatchCandidateExpansion.class); private MatchCandidateExpansion() { } + /** + * Utility method to turn an {@link Optional} into an {@link Iterable}. If the + * optional is not empty, this will return a collection with a single element. + * If the optional is empty, it will return an empty collection. + * + * @param matchCandidateMaybe an optional that may contain a match candidate + * @return a collection containing the contents of the optional if set + */ @Nonnull - public static Iterable expandValueIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { - final IndexExpansionInfo info = createInfo(metaData, index, isReverse); - return expandValueIndexMatchCandidate(info) + public static Iterable optionalToIterable(@Nonnull Optional matchCandidateMaybe) { + return matchCandidateMaybe .map(ImmutableList::of) .orElse(ImmutableList.of()); } + @Nonnull + public static Iterable expandValueIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { + final IndexExpansionInfo info = createInfo(metaData, index, isReverse); + return optionalToIterable(expandValueIndexMatchCandidate(info)); + } + + @Nonnull public static Optional expandValueIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { - return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); + return expandIndexMatchCandidate(info, info.getCommonPrimaryKeyForTypes(), + new ValueIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); } @Nonnull public static Iterable expandAggregateIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { final IndexExpansionInfo info = createInfo(metaData, index, isReverse); - return expandAggregateIndexMatchCandidate(info) - .map(ImmutableList::of) - .orElse(ImmutableList.of()); + return optionalToIterable(expandAggregateIndexMatchCandidate(info)); } @Nonnull public static Optional expandAggregateIndexMatchCandidate(@Nonnull IndexExpansionInfo info) { - final var aggregateIndexExpansionVisitor = IndexTypes.BITMAP_VALUE.equals(info.getIndexType()) - ? new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()) - : new AggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); // Override the common primary key here. We always want it to be null because the primary key is not // included in the expanded aggregate index - return expandIndexMatchCandidate(info, null, aggregateIndexExpansionVisitor); + return expandIndexMatchCandidate(info, null, + new AggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes())); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java index 9ee28ff507..01489e5406 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java @@ -52,15 +52,9 @@ * class {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link #expand} method. */ public class ValueIndexExpansionVisitor extends KeyExpressionExpansionVisitor implements ExpansionVisitor { - @Nonnull - private static final Set SUPPORTED_INDEX_TYPES = Set.of( - IndexTypes.VALUE, - IndexTypes.VERSION, - IndexTypes.RANK, - IndexTypes.PERMUTED_MAX, - IndexTypes.PERMUTED_MIN - ); - + // We may need to rethink this as it limits the set of indexes that can support a grouping key expression + // this hard-coded list, which limits the ability of this visitor to work on types not defined + // in the core sub-project @Nonnull private static final Set GROUPED_INDEX_TYPES = Set.of( IndexTypes.RANK, diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexMaintainer.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexMaintainer.java new file mode 100644 index 0000000000..c676c227ea --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexMaintainer.java @@ -0,0 +1,92 @@ +/* + * OutsideValueLikeIndexMaintainer.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.indexes; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexValidator; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableSet; + +import javax.annotation.Nonnull; +import java.util.Set; + +/** + * Test class that is supposed to mock an index that behaves like a + * {@link com.apple.foundationdb.record.metadata.IndexTypes#VALUE} index but + * is not defined within the main code. In a "real" use case of this, there may + * be something special that the index maintainer does that differentiates it + * from the built-in index type (e.g., it may interface with an external system + * or have some kind of special storage or what have you). But if it looks to + * the query planner like a regular value index, we should nevertheless be + * to surface it to the planners. + */ +public class OutsideValueLikeIndexMaintainer extends ValueIndexMaintainer { + @Nonnull + public static final String INDEX_TYPE = "outside_value"; + + public OutsideValueLikeIndexMaintainer(final IndexMaintainerState state) { + super(state); + } + + /** + * Factory for creating these index types. + */ + @AutoService(IndexMaintainerFactory.class) + public static class Factory implements IndexMaintainerFactory { + @Nonnull + private static final Set INDEX_TYPES = ImmutableSet.of(INDEX_TYPE); + private static final IndexMaintainerFactory underlying = new ValueIndexMaintainerFactory(); + + @Nonnull + @Override + public Iterable getIndexTypes() { + return INDEX_TYPES; + } + + @Nonnull + @Override + public IndexValidator getIndexValidator(final Index index) { + // Delegate to the value index type. + return underlying.getIndexValidator(index); + } + + @Nonnull + @Override + public IndexMaintainer getIndexMaintainer(@Nonnull final IndexMaintainerState state) { + // Do not delegate here. Create the new custom index maintainer type. + // (Though as can be seen above, the methods here more-or-less all delegate + // to the value index implementation.) + return new OutsideValueLikeIndexMaintainer(state); + } + + @Nonnull + @Override + public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { + // Delegate to the value index type. + return underlying.createMatchCandidates(metaData, index, reverse); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java new file mode 100644 index 0000000000..a16713eede --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java @@ -0,0 +1,318 @@ +/* + * OutsideValueLikeIndexTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.indexes; + +import com.apple.foundationdb.record.Bindings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorIterator; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordMetaDataBuilder; +import com.apple.foundationdb.record.TestRecords1Proto; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.query.DualPlannerTest; +import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; +import com.apple.foundationdb.record.query.RecordQuery; +import com.apple.foundationdb.record.query.expressions.Comparisons; +import com.apple.foundationdb.record.query.expressions.Query; +import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; +import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.RequestedOrdering; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; +import com.apple.foundationdb.record.util.pair.NonnullPair; +import com.apple.test.Tags; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import org.junit.jupiter.api.Tag; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.apple.foundationdb.record.metadata.Key.Expressions.field; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.column; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.fieldPredicate; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.fieldValue; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.forEach; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.fullTypeScan; +import static com.apple.foundationdb.record.provider.foundationdb.query.FDBQueryGraphTestHelpers.projectColumn; +import static com.apple.foundationdb.record.query.plan.ScanComparisons.anyValueComparison; +import static com.apple.foundationdb.record.query.plan.ScanComparisons.equalities; +import static com.apple.foundationdb.record.query.plan.ScanComparisons.range; +import static com.apple.foundationdb.record.query.plan.ScanComparisons.unbounded; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ListMatcher.only; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.PrimitiveMatchers.containsAll; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.flatMapPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexName; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.recordTypes; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.scanComparisons; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.scanPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.typeFilterPlan; +import static com.apple.foundationdb.record.query.plan.cascades.properties.UsedTypesProperty.usedTypes; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Test that we can use the {@link OutsideValueLikeIndexMaintainer} to define + * indexes that are then matched by the planner(s). + */ +@Tag(Tags.RequiresFDB) +class OutsideValueLikeIndexQueryTest extends FDBRecordStoreQueryTestBase { + @Nonnull + private static final String OUTSIDE_INDEX_NAME = "outside_index"; + + @Nonnull + private static final PlannableIndexTypes WITH_OUTSIDE_INDEX_TYPES = new PlannableIndexTypes( + ImmutableSet.builder() + .addAll(PlannableIndexTypes.DEFAULT.getValueTypes()) + .add(OutsideValueLikeIndexMaintainer.INDEX_TYPE) + .build(), + PlannableIndexTypes.DEFAULT.getRankTypes(), + PlannableIndexTypes.DEFAULT.getTextTypes(), + PlannableIndexTypes.DEFAULT.getUnstoredNonPrimaryKeyTypes() + ); + + private void addOutsideNumValue2Index(@Nonnull RecordMetaDataBuilder metaDataBuilder) { + final Index index = new Index(OUTSIDE_INDEX_NAME, field("num_value_2"), OutsideValueLikeIndexMaintainer.INDEX_TYPE); + metaDataBuilder.addIndex("MySimpleRecord", index); + } + + private void openStoreWithOutsideIndex(@Nonnull FDBRecordContext context) { + openSimpleRecordStore(context, this::addOutsideNumValue2Index); + setupPlanner(WITH_OUTSIDE_INDEX_TYPES); + } + + @Nonnull + private List saveSimpleData(int count) { + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + final List results = new ArrayList<>(count); + + for (int i = 0; i < count; i++) { + final TestRecords1Proto.MySimpleRecord msg = TestRecords1Proto.MySimpleRecord.newBuilder() + .setRecNo(1_000L + i) + .setNumValue2(i % 10) + .setNumValueUnique(i) + .setStrValueIndexed(i % 2 == 0 ? "even" : "odd") + .setNumValue3Indexed(i % 7) + .build(); + recordStore.saveRecord(msg); + results.add(msg); + } + + // Write update to database and return result + commit(context); + return results; + } + } + + @Nonnull + private List saveOtherData(int count) { + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + final List results = new ArrayList<>(count); + + for (int i = 0; i < count; i++) { + final TestRecords1Proto.MyOtherRecord msg = TestRecords1Proto.MyOtherRecord.newBuilder() + .setRecNo(100_000L + i) + .setNumValue2(i % 10) + .setNumValue3Indexed(i % 7) + .build(); + recordStore.saveRecord(msg); + results.add(msg); + } + + // Write update to database and return result + commit(context); + return results; + } + } + + /** + * The old planner requires we identify the outside index as a value-like + * type. If we do that, then it will match the index like a normal + * value index. The Cascades planner actually doesn't require this (as it + * uses the index maintainer factory's match candidate method). + */ + @DualPlannerTest + void planForSort() { + final List simpleRecords = saveSimpleData(50); + saveOtherData(20); // just so that we have something to avoid during the query + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + + final RecordQuery query = RecordQuery.newBuilder() + .setRecordType("MySimpleRecord") + .setSort(field("num_value_2")) + .build(); + + final RecordQueryPlan plan = planQuery(query); + assertMatchesExactly(plan, indexPlan() + .where(indexName(OUTSIDE_INDEX_NAME)) + .and(scanComparisons(unbounded()))); + + // Validate the results. Make sure results come back in the same order and contain + // all simple records + int lastNumValue2 = Integer.MIN_VALUE; + final List queried = new ArrayList<>(); + try (RecordCursorIterator> cursor = executeQuery(plan)) { + while (cursor.hasNext()) { + TestRecords1Proto.MySimpleRecord simpleRecord = TestRecords1Proto.MySimpleRecord.newBuilder() + .mergeFrom(cursor.next().getRecord()) + .build(); + assertThat(simpleRecord.getNumValue2()) + .as("num_value_2 field should be increasing") + .isGreaterThanOrEqualTo(lastNumValue2); + lastNumValue2 = simpleRecord.getNumValue2(); + queried.add(simpleRecord); + } + } + assertThat(queried) + .hasSameElementsAs(simpleRecords); + } + } + + @DualPlannerTest + void planForFilter() { + final List simpleRecords = saveSimpleData(35); + final Set numValue2s = simpleRecords.stream() + .map(TestRecords1Proto.MySimpleRecord::getNumValue2) + .collect(Collectors.toSet()); + saveOtherData(20); // just so that we have something to avoid during the query + + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + + final RecordQuery query = RecordQuery.newBuilder() + .setRecordType("MySimpleRecord") + .setFilter(Query.field("num_value_2").equalsParameter("param")) + .setSort(field("num_value_2")) + .build(); + setupPlanner(WITH_OUTSIDE_INDEX_TYPES); + + // This should plan as a single scan over the index limited to those values that match + // the given parameter + final RecordQueryPlan plan = planQuery(query); + assertMatchesExactly(plan, indexPlan() + .where(indexName(OUTSIDE_INDEX_NAME)) + .and(scanComparisons(range("[EQUALS $param]")))); + + // Validate the query results by running against each valid parameter + for (int numValue2 : numValue2s) { + try (RecordCursorIterator> cursor = executeQuery(plan, Bindings.newBuilder().set("param", numValue2).build())) { + final List queried = new ArrayList<>(); + while (cursor.hasNext()) { + TestRecords1Proto.MySimpleRecord simpleRecord = TestRecords1Proto.MySimpleRecord.newBuilder() + .mergeFrom(cursor.next().getRecord()) + .build(); + assertThat(simpleRecord.getNumValue2()) + .isEqualTo(numValue2); + queried.add(simpleRecord); + } + assertThat(queried) + .hasSameElementsAs(simpleRecords.stream().filter(msg -> msg.getNumValue2() == numValue2).collect(Collectors.toList())); + } + } + } + } + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void useAsPartOfJoin() { + final List simpleRecords = saveSimpleData(20); + final List otherRecords = saveOtherData(20); + + // Precompute the join results, using num_value_2 as the join criterion + final Map>> expectedResults = new HashMap<>(); + for (TestRecords1Proto.MySimpleRecord simpleRecord : simpleRecords) { + for (TestRecords1Proto.MyOtherRecord otherRecord : otherRecords) { + if (simpleRecord.getNumValue2() == otherRecord.getNumValue2()) { + final Set> resultList = expectedResults.computeIfAbsent(simpleRecord.getNumValue2(), ignore -> new HashSet<>()); + resultList.add(NonnullPair.of(simpleRecord.getRecNo(), otherRecord.getRecNo())); + } + } + } + + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + + // Plan a query like: + // SELECT MySimpleRecord.rec_no AS simple_rec_no, MyOtherRecord.rec_no AS other_rec_no, MySimpleRecord.num_value_2 + // FROM MySimpleRecord, MyOtherRecord + // WHERE MySimpleRecord.num_value_2 = MyOtherRecord.num_value_2 + final RecordQueryPlan plan = planGraph(() -> { + Quantifier simpleQun = fullTypeScan(recordStore.getRecordMetaData(), "MySimpleRecord"); + Quantifier otherQun = fullTypeScan(recordStore.getRecordMetaData(), "MyOtherRecord"); + + final Quantifier select = forEach(GraphExpansion.builder() + .addResultColumn(column(simpleQun, "rec_no", "simple_rec_no")) + .addResultColumn(column(otherQun, "rec_no", "other_rec_no")) + .addResultColumn(projectColumn(simpleQun, "num_value_2")) + .addQuantifier(simpleQun) + .addQuantifier(otherQun) + .addPredicate(fieldPredicate(simpleQun, "num_value_2", new Comparisons.ValueComparison(Comparisons.Type.EQUALS, fieldValue(otherQun, "num_value_2")))) + .build().buildSelect()); + + return Reference.initialOf(new LogicalSortExpression(RequestedOrdering.preserve(), select)); + }); + + // This should produce a plan that scans and grabs every MyOtherRecord, and then it + // uses the outside value index to look up its corresponding values by num_value_2 + assertMatchesExactly(plan, flatMapPlan( + typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))) + .where(recordTypes(containsAll(ImmutableSet.of("MyOtherRecord")))), + indexPlan() + .where(indexName(OUTSIDE_INDEX_NAME)) + .and(scanComparisons(equalities(only(anyValueComparison())))) + )); + + // Validate the query results + final Map>> queriedResults = new HashMap<>(); + final TypeRepository typeRepository = TypeRepository.newBuilder().addAllTypes(usedTypes().evaluate(plan)).build(); + try (RecordCursor cursor = plan.executePlan(recordStore, EvaluationContext.forTypeRepository(typeRepository), null, ExecuteProperties.SERIAL_EXECUTE)) { + for (RecordCursorResult result = cursor.getNext(); result.hasNext(); result = cursor.getNext()) { + Message msg = result.get().getMessage(); + Descriptors.Descriptor descriptor = msg.getDescriptorForType(); + int numValue2 = (int) msg.getField(descriptor.findFieldByName("num_value_2")); + long simpleRecNo = (long) msg.getField(descriptor.findFieldByName("simple_rec_no")); + long otherRecNo = (long) msg.getField(descriptor.findFieldByName("other_rec_no")); + queriedResults.computeIfAbsent(numValue2, ignore -> new HashSet<>()) + .add(NonnullPair.of(simpleRecNo, otherRecNo)); + } + } + assertThat(queriedResults) + .isEqualTo(expectedResults); + } + } +} From 75e06d6e476282d7eab023bad5222de27ca39e28 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 29 Sep 2025 15:10:04 +0100 Subject: [PATCH 5/9] address some teamscale findings --- .../BitmapValueIndexMaintainerFactory.java | 3 +- .../PermutedMinMaxIndexMaintainerFactory.java | 3 +- .../indexes/RankIndexMaintainerFactory.java | 3 +- .../plan/cascades/IndexExpansionInfo.java | 140 ++++++++++++++++++ .../cascades/MatchCandidateExpansion.java | 119 +-------------- 5 files changed, 152 insertions(+), 116 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java index bb023133d3..4249ae80d2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/BitmapValueIndexMaintainerFactory.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; import com.apple.foundationdb.record.query.plan.cascades.BitmapAggregateIndexExpansionVisitor; import com.apple.foundationdb.record.query.plan.cascades.ExpansionVisitor; +import com.apple.foundationdb.record.query.plan.cascades.IndexExpansionInfo; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; @@ -113,7 +114,7 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { @Nonnull @Override public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { - final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, reverse); final ExpansionVisitor expansionVisitor = new BitmapAggregateIndexExpansionVisitor(info.getIndex(), info.getIndexedRecordTypes()); return MatchCandidateExpansion.optionalToIterable( MatchCandidateExpansion.expandIndexMatchCandidate(info, null, expansionVisitor)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java index c4763a93fa..66b40e9272 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/PermutedMinMaxIndexMaintainerFactory.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.IndexExpansionInfo; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.google.auto.service.AutoService; @@ -95,7 +96,7 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { @Nonnull @Override public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { - final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, reverse); final ImmutableList.Builder resultBuilder = ImmutableList.builderWithExpectedSize(2); // For permuted min and max, we use the value index expansion for BY_VALUE scans and we use diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java index 8718fa7ff0..be93ec2c1d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/RankIndexMaintainerFactory.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.query.plan.cascades.IndexExpansionInfo; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.MatchCandidateExpansion; import com.apple.foundationdb.record.query.plan.cascades.WindowedIndexExpansionVisitor; @@ -110,7 +111,7 @@ public IndexMaintainer getIndexMaintainer(@Nonnull IndexMaintainerState state) { @Nonnull @Override public Iterable createMatchCandidates(@Nonnull final RecordMetaData metaData, @Nonnull final Index index, final boolean reverse) { - final MatchCandidateExpansion.IndexExpansionInfo info = MatchCandidateExpansion.createInfo(metaData, index, reverse); + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, reverse); final ImmutableList.Builder resultBuilder = ImmutableList.builder(); // For rank() we need to create at two candidates. One for BY_RANK scans and one for BY_VALUE scans. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java new file mode 100644 index 0000000000..e5adc6b385 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java @@ -0,0 +1,140 @@ +/* + * IndexExpansionInfo.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.google.common.collect.ImmutableSet; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +/** + * Class encapsulating the information necessary to create a {@link MatchCandidate} + * for an {@link Index}. This exists as a convenience class that allows certain + * information to be computed once and referenced during match candidate creation. + */ +@API(API.Status.INTERNAL) +public final class IndexExpansionInfo { + @Nonnull + private final RecordMetaData metaData; + @Nonnull + private final Index index; + private final boolean reverse; + @Nullable + private final KeyExpression commonPrimaryKeyForTypes; + @Nonnull + private final Collection indexedRecordTypes; + @Nonnull + private final Set indexedRecordTypeNames; + + private IndexExpansionInfo(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse, + @Nonnull Collection indexedRecordTypes, + @Nonnull Set indexedRecordTypeNames, + @Nullable KeyExpression commonPrimaryKeyForTypes) { + this.metaData = metaData; + this.index = index; + this.reverse = reverse; + this.indexedRecordTypes = indexedRecordTypes; + this.indexedRecordTypeNames = indexedRecordTypeNames; + this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; + } + + @Nonnull + public RecordMetaData getMetaData() { + return metaData; + } + + @Nonnull + public Index getIndex() { + return index; + } + + @Nonnull + public String getIndexName() { + return index.getName(); + } + + @Nonnull + public String getIndexType() { + return index.getType(); + } + + public boolean isReverse() { + return reverse; + } + + @Nonnull + public Collection getIndexedRecordTypes() { + return indexedRecordTypes; + } + + @Nonnull + public Set getIndexedRecordTypeNames() { + return indexedRecordTypeNames; + } + + @Nullable + public KeyExpression getCommonPrimaryKeyForTypes() { + return commonPrimaryKeyForTypes; + } + + @Nonnull + public Set getAvailableRecordTypeNames() { + return metaData.getRecordTypes().keySet(); + } + + /** + * Create an {@link IndexExpansionInfo} for a given index. + * This wraps the given parameters into a single object, as well + * as enriching the given parameters with pre-calculated items that + * can then be used during index expansion. + * + * @param metaData the meta-data that is the source of the index + * @param index the index that we are expanding + * @param reverse whether the query requires this scan be in reverse + * @return an object encapsulating information about the index + */ + @Nonnull + public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData, + @Nonnull Index index, + boolean reverse) { + @Nonnull + final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); + @Nonnull + final Set indexedRecordTypeNames = indexedRecordTypes.stream() + .map(RecordType::getName) + .collect(ImmutableSet.toImmutableSet()); + @Nullable + final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); + + return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); + } + +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java index bcd9cff558..29947d291b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java @@ -30,14 +30,12 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collection; -import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -51,7 +49,7 @@ * list containing the one candidate (if set). */ @API(API.Status.INTERNAL) -public class MatchCandidateExpansion { +public final class MatchCandidateExpansion { @Nonnull private static final Logger LOGGER = LoggerFactory.getLogger(MatchCandidateExpansion.class); @@ -63,19 +61,19 @@ private MatchCandidateExpansion() { * optional is not empty, this will return a collection with a single element. * If the optional is empty, it will return an empty collection. * - * @param matchCandidateMaybe an optional that may contain a match candidate + * @param optional an optional that may contain a match candidate * @return a collection containing the contents of the optional if set */ @Nonnull - public static Iterable optionalToIterable(@Nonnull Optional matchCandidateMaybe) { - return matchCandidateMaybe + public static Iterable optionalToIterable(@Nonnull Optional optional) { + return optional .map(ImmutableList::of) .orElse(ImmutableList.of()); } @Nonnull public static Iterable expandValueIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { - final IndexExpansionInfo info = createInfo(metaData, index, isReverse); + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, isReverse); return optionalToIterable(expandValueIndexMatchCandidate(info)); } @@ -87,7 +85,7 @@ public static Optional expandValueIndexMatchCandidate(@Nonnull I @Nonnull public static Iterable expandAggregateIndexMatchCandidate(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean isReverse) { - final IndexExpansionInfo info = createInfo(metaData, index, isReverse); + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, isReverse); return optionalToIterable(expandAggregateIndexMatchCandidate(info)); } @@ -161,109 +159,4 @@ private static Reference createBaseRef(@Nonnull final Set availableRecor quantifier, Type.Record.fromFieldDescriptorsMap(RecordMetaData.getFieldDescriptorMapFromTypes(queriedRecordTypes)))); } - - /** - * Class encapsulating the information necessary to create a {@link MatchCandidate} - * for an {@link Index}. This exists as a convenience class that allows certain - * information to be computed once and referenced during match candidate creation. - */ - @API(API.Status.INTERNAL) - public static class IndexExpansionInfo { - @Nonnull - private final RecordMetaData metaData; - @Nonnull - private final Index index; - private final boolean reverse; - @Nullable - private final KeyExpression commonPrimaryKeyForTypes; - @Nonnull - private final Collection indexedRecordTypes; - @Nonnull - private final Set indexedRecordTypeNames; - - private IndexExpansionInfo(@Nonnull RecordMetaData metaData, - @Nonnull Index index, - boolean reverse, - @Nonnull Collection indexedRecordTypes, - @Nonnull Set indexedRecordTypeNames, - @Nullable KeyExpression commonPrimaryKeyForTypes) { - this.metaData = metaData; - this.index = index; - this.reverse = reverse; - this.indexedRecordTypes = indexedRecordTypes; - this.indexedRecordTypeNames = indexedRecordTypeNames; - this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; - } - - @Nonnull - public RecordMetaData getMetaData() { - return metaData; - } - - @Nonnull - public Index getIndex() { - return index; - } - - @Nonnull - public String getIndexName() { - return index.getName(); - } - - @Nonnull - public String getIndexType() { - return index.getType(); - } - - public boolean isReverse() { - return reverse; - } - - @Nonnull - public Collection getIndexedRecordTypes() { - return indexedRecordTypes; - } - - @Nonnull - public Set getIndexedRecordTypeNames() { - return indexedRecordTypeNames; - } - - @Nullable - public KeyExpression getCommonPrimaryKeyForTypes() { - return commonPrimaryKeyForTypes; - } - - @Nonnull - public Set getAvailableRecordTypeNames() { - return metaData.getRecordTypes().keySet(); - } - } - - /** - * Create an {@link IndexExpansionInfo} for a given index. - * This wraps the given parameters into a single object, as well - * as enriching the given parameters with pre-calculated items that - * can then be used during index expansion. - * - * @param metaData the meta-data that is the source of the index - * @param index the index that we are expanding - * @param reverse whether the query requires this scan be in reverse - * @return an object encapsulating information about the index - */ - @Nonnull - public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData, - @Nonnull Index index, - boolean reverse) { - @Nonnull - final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); - @Nonnull - final Set indexedRecordTypeNames = indexedRecordTypes.stream() - .map(RecordType::getName) - .collect(ImmutableSet.toImmutableSet()); - @Nullable - final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); - - return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, commonPrimaryKeyForTypes); - } } From dc8e8515c66e4652c5c5ec739bd7178a492186d6 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 29 Sep 2025 18:35:48 +0100 Subject: [PATCH 6/9] address some test gaps found by teamscale --- .../plan/cascades/IndexExpansionInfo.java | 5 - .../foundationdb/FDBTypedRecordStoreTest.java | 75 ++++++++-- .../NonCascadesValueIndexMaintainer.java | 69 ++++++++++ .../OutsideValueLikeIndexQueryTest.java | 130 +++++++++++------- 4 files changed, 214 insertions(+), 65 deletions(-) create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/NonCascadesValueIndexMaintainer.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java index e5adc6b385..24ccc4dd98 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java @@ -81,11 +81,6 @@ public String getIndexName() { return index.getName(); } - @Nonnull - public String getIndexType() { - return index.getType(); - } - public boolean isReverse() { return reverse; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java index b6201caf59..0eb19e2160 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java @@ -27,6 +27,8 @@ import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Query; +import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.record.test.FDBDatabaseExtension; import com.apple.foundationdb.record.test.TestKeySpace; import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension; @@ -37,9 +39,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for {@link FDBTypedRecordStore}. @@ -106,36 +113,76 @@ void writeRead() { } } - @Test - void query() { + @Nonnull + private List insertTestData() { + List inserted = new ArrayList<>(); try (FDBRecordContext context = fdb.openContext()) { openTypedRecordStore(context); - for (int i = 0; i < 100; i++) { - TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder(); - recBuilder.setRecNo(i); - recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even"); - recBuilder.setNumValueUnique(i + 1000); - recordStore.saveRecord(recBuilder.build()); + TestRecords1Proto.MySimpleRecord rec = TestRecords1Proto.MySimpleRecord.newBuilder() + .setRecNo(i) + .setStrValueIndexed((i & 1) == 1 ? "odd" : "even") + .setNumValueUnique(i + 1000) + .build(); + recordStore.saveRecord(rec); + inserted.add(rec); } context.commit(); } + return inserted; + } + + @Test + void query() { + final List data = insertTestData(); - RecordQuery query = RecordQuery.newBuilder() + final RecordQuery query = RecordQuery.newBuilder() .setRecordType("MySimpleRecord") .setFilter(Query.field("str_value_indexed").equalsValue("even")) .build(); try (FDBRecordContext context = fdb.openContext()) { openTypedRecordStore(context); - int i = 0; + final List queried = new ArrayList<>(); try (RecordCursorIterator> cursor = recordStore.executeQuery(query).asIterator()) { while (cursor.hasNext()) { TestRecords1Proto.MySimpleRecord myrec = cursor.next().getRecord(); - assertTrue((myrec.getNumValueUnique() % 2) == 0); - i++; + assertThat(myrec.getNumValueUnique() % 2) + .isZero(); + queried.add(myrec); + } + } + assertThat(queried) + .hasSize(50) + .hasSameElementsAs(data.stream().filter(rec -> "even".equals(rec.getStrValueIndexed())).collect(Collectors.toSet())); + } + } + + @Test + void queryCascades() { + final List data = insertTestData(); + + final RecordQuery query = RecordQuery.newBuilder() + .setRecordType("MySimpleRecord") + .setFilter(Query.field("str_value_indexed").equalsValue("odd")) + .build(); + + try (FDBRecordContext context = fdb.openContext()) { + openTypedRecordStore(context); + + CascadesPlanner cascadesPlanner = CascadesPlanner.forStore(recordStore); + final RecordQueryPlan plan = cascadesPlanner.plan(query); + final List queried = new ArrayList<>(); + try (RecordCursorIterator> cursor = recordStore.executeQuery(plan).asIterator()) { + while (cursor.hasNext()) { + TestRecords1Proto.MySimpleRecord myrec = cursor.next().getRecord(); + assertThat(myrec.getNumValueUnique() % 2) + .isOne(); + queried.add(myrec); } } - assertEquals(50, i); + assertThat(queried) + .hasSize(50) + .hasSameElementsAs(data.stream().filter(rec -> "odd".equals(rec.getStrValueIndexed())).collect(Collectors.toSet())); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/NonCascadesValueIndexMaintainer.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/NonCascadesValueIndexMaintainer.java new file mode 100644 index 0000000000..1280b5a285 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/NonCascadesValueIndexMaintainer.java @@ -0,0 +1,69 @@ +/* + * NonCascadesValueIndexMaintainer.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.indexes; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexValidator; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.google.auto.service.AutoService; + +import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.Set; + +/** + * Simulate a value-like index that does not implement {@link IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean)} + * and so cannot be used by the {@link com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner}. + */ +public class NonCascadesValueIndexMaintainer extends ValueIndexMaintainer { + public static final String INDEX_TYPE = "non_cascades_value"; + + public NonCascadesValueIndexMaintainer(final IndexMaintainerState state) { + super(state); + } + + @AutoService(IndexMaintainerFactory.class) + public static class Factory implements IndexMaintainerFactory { + private static final Set INDEX_TYPES = Collections.singleton(INDEX_TYPE); + private static final ValueIndexMaintainerFactory underlying = new ValueIndexMaintainerFactory(); + + @Nonnull + @Override + public Iterable getIndexTypes() { + return INDEX_TYPES; + } + + @Nonnull + @Override + public IndexValidator getIndexValidator(final Index index) { + return underlying.getIndexValidator(index); + } + + @Nonnull + @Override + public IndexMaintainer getIndexMaintainer(@Nonnull final IndexMaintainerState state) { + return new NonCascadesValueIndexMaintainer(state); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java index a16713eede..5deac35841 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorIterator; import com.apple.foundationdb.record.RecordCursorResult; @@ -74,6 +75,7 @@ import static com.apple.foundationdb.record.query.plan.ScanComparisons.unbounded; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.ListMatcher.only; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.PrimitiveMatchers.containsAll; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.filterPlan; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.flatMapPlan; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexName; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexPlan; @@ -83,6 +85,7 @@ import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.typeFilterPlan; import static com.apple.foundationdb.record.query.plan.cascades.properties.UsedTypesProperty.usedTypes; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * Test that we can use the {@link OutsideValueLikeIndexMaintainer} to define @@ -109,55 +112,50 @@ private void addOutsideNumValue2Index(@Nonnull RecordMetaDataBuilder metaDataBui metaDataBuilder.addIndex("MySimpleRecord", index); } + private void addNonCascadesNumValue2Index(@Nonnull RecordMetaDataBuilder metaDataBuilder) { + final Index index = new Index(OUTSIDE_INDEX_NAME, field("num_value_2"), NonCascadesValueIndexMaintainer.INDEX_TYPE); + metaDataBuilder.addIndex("MySimpleRecord", index); + } + private void openStoreWithOutsideIndex(@Nonnull FDBRecordContext context) { openSimpleRecordStore(context, this::addOutsideNumValue2Index); setupPlanner(WITH_OUTSIDE_INDEX_TYPES); } + private void openStoreWithNonCascadesValue2Index(@Nonnull FDBRecordContext context) { + openSimpleRecordStore(context, this::addNonCascadesNumValue2Index); + } + @Nonnull private List saveSimpleData(int count) { - try (FDBRecordContext context = openContext()) { - openStoreWithOutsideIndex(context); - final List results = new ArrayList<>(count); - - for (int i = 0; i < count; i++) { - final TestRecords1Proto.MySimpleRecord msg = TestRecords1Proto.MySimpleRecord.newBuilder() - .setRecNo(1_000L + i) - .setNumValue2(i % 10) - .setNumValueUnique(i) - .setStrValueIndexed(i % 2 == 0 ? "even" : "odd") - .setNumValue3Indexed(i % 7) - .build(); - recordStore.saveRecord(msg); - results.add(msg); - } - - // Write update to database and return result - commit(context); - return results; + final List results = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + final TestRecords1Proto.MySimpleRecord msg = TestRecords1Proto.MySimpleRecord.newBuilder() + .setRecNo(1_000L + i) + .setNumValue2(i % 10) + .setNumValueUnique(i) + .setStrValueIndexed(i % 2 == 0 ? "even" : "odd") + .setNumValue3Indexed(i % 7) + .build(); + recordStore.saveRecord(msg); + results.add(msg); } + return results; } @Nonnull private List saveOtherData(int count) { - try (FDBRecordContext context = openContext()) { - openStoreWithOutsideIndex(context); - final List results = new ArrayList<>(count); - - for (int i = 0; i < count; i++) { - final TestRecords1Proto.MyOtherRecord msg = TestRecords1Proto.MyOtherRecord.newBuilder() - .setRecNo(100_000L + i) - .setNumValue2(i % 10) - .setNumValue3Indexed(i % 7) - .build(); - recordStore.saveRecord(msg); - results.add(msg); - } - - // Write update to database and return result - commit(context); - return results; + final List results = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + final TestRecords1Proto.MyOtherRecord msg = TestRecords1Proto.MyOtherRecord.newBuilder() + .setRecNo(100_000L + i) + .setNumValue2(i % 10) + .setNumValue3Indexed(i % 7) + .build(); + recordStore.saveRecord(msg); + results.add(msg); } + return results; } /** @@ -168,11 +166,12 @@ private List saveOtherData(int count) { */ @DualPlannerTest void planForSort() { - final List simpleRecords = saveSimpleData(50); - saveOtherData(20); // just so that we have something to avoid during the query try (FDBRecordContext context = openContext()) { openStoreWithOutsideIndex(context); + final List simpleRecords = saveSimpleData(50); + saveOtherData(20); // just so that we have something to avoid during the query + final RecordQuery query = RecordQuery.newBuilder() .setRecordType("MySimpleRecord") .setSort(field("num_value_2")) @@ -206,15 +205,15 @@ void planForSort() { @DualPlannerTest void planForFilter() { - final List simpleRecords = saveSimpleData(35); - final Set numValue2s = simpleRecords.stream() - .map(TestRecords1Proto.MySimpleRecord::getNumValue2) - .collect(Collectors.toSet()); - saveOtherData(20); // just so that we have something to avoid during the query - try (FDBRecordContext context = openContext()) { openStoreWithOutsideIndex(context); + final List simpleRecords = saveSimpleData(35); + final Set numValue2s = simpleRecords.stream() + .map(TestRecords1Proto.MySimpleRecord::getNumValue2) + .collect(Collectors.toSet()); + saveOtherData(20); // just so that we have something to avoid during the query + final RecordQuery query = RecordQuery.newBuilder() .setRecordType("MySimpleRecord") .setFilter(Query.field("num_value_2").equalsParameter("param")) @@ -250,8 +249,14 @@ void planForFilter() { @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) void useAsPartOfJoin() { - final List simpleRecords = saveSimpleData(20); - final List otherRecords = saveOtherData(20); + final List simpleRecords; + final List otherRecords; + try (FDBRecordContext context = openContext()) { + openStoreWithOutsideIndex(context); + simpleRecords = saveSimpleData(20); + otherRecords = saveOtherData(20); + commit(context); + } // Precompute the join results, using num_value_2 as the join criterion final Map>> expectedResults = new HashMap<>(); @@ -315,4 +320,37 @@ void useAsPartOfJoin() { .isEqualTo(expectedResults); } } + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void fullScanWithIndexWithoutMatchCandidates() { + try (FDBRecordContext context = openContext()) { + openStoreWithNonCascadesValue2Index(context); + + final RecordQuery query1 = RecordQuery.newBuilder() + .setRecordType("MySimpleRecord") + .setFilter(Query.field("num_value_2").equalsParameter("param")) + .build(); + final RecordQueryPlan plan = planQuery(query1); + assertMatchesExactly(plan, filterPlan( + typeFilterPlan( + scanPlan().where(scanComparisons(unbounded())) + ) + )); + } + } + + @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + void cannotSortWithIndexWithoutMatchCandidates() { + try (FDBRecordContext context = openContext()) { + openStoreWithNonCascadesValue2Index(context); + + final RecordQuery query1 = RecordQuery.newBuilder() + .setRecordType("MySimpleRecord") + .setSort(field("num_value_2"), true) + .build(); + assertThatThrownBy(() -> planQuery(query1)) + .isInstanceOf(RecordCoreException.class) + .hasMessageContaining("Cascades planner could not plan query"); + } + } } From 7719731c6cebaef53a1d0d240cfddf1bbd914fde Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Mon, 29 Sep 2025 19:05:30 +0100 Subject: [PATCH 7/9] fix assertion on plan type for new query test --- .../OutsideValueLikeIndexQueryTest.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java index 5deac35841..74afb934d5 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/OutsideValueLikeIndexQueryTest.java @@ -79,6 +79,7 @@ import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.flatMapPlan; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexName; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.indexPlan; +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.predicatesFilterPlan; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.recordTypes; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.scanComparisons; import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RecordQueryPlanMatchers.scanPlan; @@ -321,34 +322,47 @@ void useAsPartOfJoin() { } } - @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) + @DualPlannerTest void fullScanWithIndexWithoutMatchCandidates() { try (FDBRecordContext context = openContext()) { openStoreWithNonCascadesValue2Index(context); - final RecordQuery query1 = RecordQuery.newBuilder() + final RecordQuery query = RecordQuery.newBuilder() .setRecordType("MySimpleRecord") .setFilter(Query.field("num_value_2").equalsParameter("param")) .build(); - final RecordQueryPlan plan = planQuery(query1); - assertMatchesExactly(plan, filterPlan( - typeFilterPlan( - scanPlan().where(scanComparisons(unbounded())) - ) - )); + final RecordQueryPlan plan = planQuery(query); + if (useCascadesPlanner) { + assertMatchesExactly(plan, predicatesFilterPlan( + typeFilterPlan( + scanPlan().where(scanComparisons(unbounded())) + ) + )); + } else { + assertMatchesExactly(plan, filterPlan( + typeFilterPlan( + scanPlan().where(scanComparisons(unbounded())) + ) + )); + } } } + /** + * Validate that indexes that aren't set up to be matched are not selected when querying. + * Ideally, we'd be able to assert that this doesn't use the index in the old planner either, + * but see: Issue #3648. + */ @DualPlannerTest(planner = DualPlannerTest.Planner.CASCADES) void cannotSortWithIndexWithoutMatchCandidates() { try (FDBRecordContext context = openContext()) { openStoreWithNonCascadesValue2Index(context); - final RecordQuery query1 = RecordQuery.newBuilder() + final RecordQuery query = RecordQuery.newBuilder() .setRecordType("MySimpleRecord") .setSort(field("num_value_2"), true) .build(); - assertThatThrownBy(() -> planQuery(query1)) + assertThatThrownBy(() -> planQuery(query)) .isInstanceOf(RecordCoreException.class) .hasMessageContaining("Cascades planner could not plan query"); } From 2e086db7f0ca8ae5570701326ee0bc746aade443 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 30 Sep 2025 15:46:39 +0100 Subject: [PATCH 8/9] Respond to comments from @normen662: * Moved the initiliazer of `CascadesPlanner` that took an `FDBRecordStoreBase` to a method on `FDBRecordStoreBase` * Expanded a comment that trailed off --- .../provider/foundationdb/FDBRecordStoreBase.java | 13 +++++++++++++ .../foundationdb/IndexMaintainerFactory.java | 6 +++++- .../record/query/plan/cascades/CascadesPlanner.java | 5 ----- .../FDBRecordStoreConcurrentTestBase.java | 3 +-- .../foundationdb/FDBTypedRecordStoreTest.java | 2 +- .../record/lucene/FDBLuceneQueryTest.java | 3 +-- .../record/lucene/LuceneIndexTestUtils.java | 3 +-- .../record/lucene/LuceneQueryIntegrationTest.java | 3 +-- .../record/lucene/highlight/LuceneScaleTest.java | 3 +-- .../relational/recordlayer/query/PlanGenerator.java | 10 ++++++++++ 10 files changed, 34 insertions(+), 17 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java index d3d58a898b..a5a94b5814 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java @@ -65,6 +65,7 @@ import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.RecordQueryPlanner; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.plans.QueryResult; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.subspace.Subspace; @@ -2120,6 +2121,18 @@ default RecordCursor executeQuery(@Nonnull RecordQueryPlan plan, return plan.executePlan(this, evaluationContext, continuation, executeProperties); } + /** + * Create a {@link CascadesPlanner} to use to plan queries on this record store. It will be + * initialized with the context necessary to generate query plans for the store, like the + * set of available types and indexes. + * + * @return a {@link CascadesPlanner} implementation initialized with state from this record store + */ + @Nonnull + default CascadesPlanner getCascadesPlanner() { + return new CascadesPlanner(getRecordMetaData(), getRecordStoreState(), getIndexMaintainerRegistry()); + } + /** * Plan a query. * @param query the query to plan diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java index 138c180d4c..e9dc5a711e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactory.java @@ -79,7 +79,11 @@ public interface IndexMaintainerFactory { * structured as a method on the maintainer factory so that indexes * defined outside the core repository can define the structure * used by the planner to match them. However, use cases should be - * aware that the planner + * aware that the planner does not provide a stable API (yet) for + * what these match candidates should contain, and so outside + * implementors should be mindful that it is on them to keep up + * with the planner as it changes. For that reason, this should only + * be overridden by users who know what they are doing. * *

* By default, this returns an empty collection. This means that the diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java index 8490b949d2..75737c4108 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.logging.LogMessageKeys; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.ParameterRelationshipGraph; @@ -244,10 +243,6 @@ public CascadesPlanner(@Nonnull RecordMetaData metaData, @Nonnull RecordStoreSta this.taskStack = new ArrayDeque<>(); } - public static CascadesPlanner forStore(@Nonnull FDBRecordStoreBase recordStore) { - return new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState(), recordStore.getIndexMaintainerRegistry()); - } - @Nonnull @Override public RecordMetaData getRecordMetaData() { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java index eaac02610f..4ae9a2497a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreConcurrentTestBase.java @@ -29,7 +29,6 @@ import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; import com.apple.foundationdb.record.query.plan.QueryPlanner; import com.apple.foundationdb.record.query.plan.RecordQueryPlanner; -import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.debug.Debugger; import com.apple.foundationdb.record.query.plan.cascades.debug.DebuggerWithSymbolTables; import com.apple.foundationdb.record.test.FDBDatabaseExtension; @@ -101,7 +100,7 @@ protected FDBRecordStore createOrOpenRecordStore(@Nonnull FDBRecordContext conte public QueryPlanner setupPlanner(@Nonnull FDBRecordStore recordStore, @Nullable PlannableIndexTypes indexTypes) { final QueryPlanner planner; if (useCascadesPlanner) { - planner = CascadesPlanner.forStore(recordStore); + planner = recordStore.getCascadesPlanner(); if (Debugger.getDebugger() == null) { Debugger.setDebugger(DebuggerWithSymbolTables.withSanityChecks()); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java index 0eb19e2160..2ed1c7b109 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBTypedRecordStoreTest.java @@ -169,7 +169,7 @@ void queryCascades() { try (FDBRecordContext context = fdb.openContext()) { openTypedRecordStore(context); - CascadesPlanner cascadesPlanner = CascadesPlanner.forStore(recordStore); + CascadesPlanner cascadesPlanner = recordStore.getCascadesPlanner(); final RecordQueryPlan plan = cascadesPlanner.plan(query); final List queried = new ArrayList<>(); try (RecordCursorIterator> cursor = recordStore.executeQuery(plan).asIterator()) { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java index 65e0a07422..b5566bd081 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java @@ -50,7 +50,6 @@ import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; -import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.record.util.pair.Pair; import com.apple.foundationdb.tuple.Tuple; @@ -213,7 +212,7 @@ public static void setup() { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = CascadesPlanner.forStore(recordStore); + planner = recordStore.getCascadesPlanner(); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java index 415c39aebd..be0a1500fe 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestUtils.java @@ -44,7 +44,6 @@ import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; import com.apple.foundationdb.record.query.plan.QueryPlanner; import com.apple.foundationdb.record.query.plan.ScanComparisons; -import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.debug.Debugger; import com.apple.foundationdb.record.query.plan.cascades.debug.DebuggerWithSymbolTables; import com.apple.foundationdb.record.util.pair.Pair; @@ -617,7 +616,7 @@ static QueryPlanner setupPlanner(@Nonnull FDBRecordStore recordStore, @Nullable PlannableIndexTypes indexTypes, boolean useRewritePlanner) { QueryPlanner planner; if (useRewritePlanner) { - planner = new CascadesPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState(), recordStore.getIndexMaintainerRegistry()); + planner = recordStore.getCascadesPlanner(); if (Debugger.getDebugger() == null) { Debugger.setDebugger(DebuggerWithSymbolTables.withSanityChecks()); } diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java index cc2260d70d..d10d22c35c 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneQueryIntegrationTest.java @@ -34,7 +34,6 @@ import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; -import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.test.Tags; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; @@ -78,7 +77,7 @@ public class LuceneQueryIntegrationTest extends FDBRecordStoreQueryTestBase { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = CascadesPlanner.forStore(recordStore); + planner = recordStore.getCascadesPlanner(); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java index 60321e7165..0cce18a88e 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/highlight/LuceneScaleTest.java @@ -50,7 +50,6 @@ import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.PlannableIndexTypes; import com.apple.foundationdb.record.query.plan.QueryPlanner; -import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.record.test.TestKeySpace; import com.apple.foundationdb.record.util.pair.Pair; @@ -254,7 +253,7 @@ protected void clear() { @Override public void setupPlanner(@Nullable PlannableIndexTypes indexTypes) { if (isUseCascadesPlanner()) { - planner = CascadesPlanner.forStore(recordStore); + planner = recordStore.getCascadesPlanner(); } else { if (indexTypes == null) { indexTypes = new PlannableIndexTypes( diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 4edebd3e72..d5adc8953c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -412,6 +412,16 @@ public static PlanGenerator create(@Nonnull final Optional return new PlanGenerator(cache, planContext, planner, options); } + /** + * Create a new instance of the plan generator for a given store. + * + * @param cache An optional instance of the query plan cache + * @param planContext The context related for planning the query and looking it in the cache + * @param store The record store to generate the planner for + * @param options a set of planner options + * @return a new instance of the plan generator + * @throws RelationalException if creation of the plan generator fails + */ @Nonnull public static PlanGenerator create(@Nonnull final Optional cache, @Nonnull final PlanContext planContext, From 6e04ce1a44d49e416bfbfdcccd38f1b32d9d145d Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 30 Sep 2025 15:49:05 +0100 Subject: [PATCH 9/9] rename IndexMaintainerRegistryImpl to IndexMaintainerFactoryRegistryImpl --- .../apple/foundationdb/record/RecordMetaDataBuilder.java | 4 ++-- .../record/metadata/MetaDataEvolutionValidator.java | 6 +++--- .../record/provider/foundationdb/FDBRecordStore.java | 2 +- ...yImpl.java => IndexMaintainerFactoryRegistryImpl.java} | 8 ++++---- .../foundationdb/IndexMatchCandidateRegistry.java | 2 +- .../record/metadata/MetaDataValidatorTest.java | 4 ++-- .../foundationdb/query/QueryPlanFullySortedTest.java | 4 ++-- .../foundationdb/relational/api/ddl/DdlTestUtil.java | 8 ++++---- .../relational/memory/InMemoryRelationalStatement.java | 4 ++-- .../recordlayer/metadata/SchemaTemplateSerDeTests.java | 4 ++-- 10 files changed, 23 insertions(+), 23 deletions(-) rename fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/{IndexMaintainerRegistryImpl.java => IndexMaintainerFactoryRegistryImpl.java} (90%) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index c6078a8562..fca2733ff4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -40,7 +40,7 @@ import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor; import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; @@ -146,7 +146,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { formerIndexes = new ArrayList<>(); unionFields = new HashMap<>(); explicitDependencies = new TreeMap<>(); - indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); + indexMaintainerRegistry = IndexMaintainerFactoryRegistryImpl.instance(); evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance(); syntheticRecordTypes = new HashMap<>(); userDefinedFunctionMap = new HashMap<>(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java index 5201c26b51..0ce214a7dd 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidator.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -101,7 +101,7 @@ public class MetaDataEvolutionValidator { private final boolean disallowTypeRenames; private MetaDataEvolutionValidator() { - this.indexValidatorRegistry = IndexMaintainerRegistryImpl.instance(); + this.indexValidatorRegistry = IndexMaintainerFactoryRegistryImpl.instance(); this.allowNoVersionChange = false; this.allowNoSinceVersion = false; this.allowIndexRebuilds = false; @@ -677,7 +677,7 @@ private void validateIndex(@Nonnull RecordMetaData oldMetaData, @Nonnull Index o /** * Get the registry of index maintainers used to validate indexes. This registry should generally * be the same registry as the registry that will be used by any record stores that may use - * the meta-data. By default, this uses the default {@link IndexMaintainerRegistryImpl} instance. + * the meta-data. By default, this uses the default {@link IndexMaintainerFactoryRegistryImpl} instance. * * @return the index maintainer registry used to validate indexes * @see com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.BaseBuilder#setIndexMaintainerRegistry(IndexMaintainerFactoryRegistry) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java index ff0211c535..75f1fd780a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java @@ -5327,7 +5327,7 @@ public static class Builder implements BaseBuilder { private FDBRecordStoreBase.UserVersionChecker userVersionChecker; @Nonnull - private IndexMaintainerFactoryRegistry indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); + private IndexMaintainerFactoryRegistry indexMaintainerRegistry = IndexMaintainerFactoryRegistryImpl.instance(); @Nonnull private IndexMaintenanceFilter indexMaintenanceFilter = IndexMaintenanceFilter.NORMAL; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistryImpl.java similarity index 90% rename from fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java rename to fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistryImpl.java index 020fe31242..5908f953e7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerRegistryImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMaintainerFactoryRegistryImpl.java @@ -37,11 +37,11 @@ * A singleton {@link IndexMaintainerRegistry} that finds {@link IndexMaintainerFactory} classes in the classpath. */ @API(API.Status.INTERNAL) -public class IndexMaintainerRegistryImpl implements IndexMaintainerFactoryRegistry { +public class IndexMaintainerFactoryRegistryImpl implements IndexMaintainerFactoryRegistry { @Nonnull - private static final Logger LOGGER = LoggerFactory.getLogger(IndexMaintainerRegistryImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(IndexMaintainerFactoryRegistryImpl.class); @Nonnull - protected static final IndexMaintainerRegistryImpl INSTANCE = new IndexMaintainerRegistryImpl(); + protected static final IndexMaintainerFactoryRegistryImpl INSTANCE = new IndexMaintainerFactoryRegistryImpl(); @Nonnull private final Map registry; @@ -68,7 +68,7 @@ protected static Map initRegistry() { return registry; } - protected IndexMaintainerRegistryImpl() { + protected IndexMaintainerFactoryRegistryImpl() { registry = initRegistry(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java index 419d85a5e6..a93c9f21ae 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexMatchCandidateRegistry.java @@ -34,7 +34,7 @@ * * @see IndexMaintainerFactory#createMatchCandidates(RecordMetaData, Index, boolean) * @see IndexMaintainerFactoryRegistry for a sub-interface that delegates to the index maintainer factory - * @see IndexMaintainerRegistryImpl for the default implementation + * @see IndexMaintainerFactoryRegistryImpl for the default implementation */ public interface IndexMatchCandidateRegistry { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataValidatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataValidatorTest.java index 40c1bd7c9c..d6c66b75e1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataValidatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataValidatorTest.java @@ -29,7 +29,7 @@ import com.apple.foundationdb.record.TestRecordsEnumProto; import com.apple.foundationdb.record.TestRecordsNameClashProto; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.ByteString; @@ -52,7 +52,7 @@ public class MetaDataValidatorTest { private void validate(RecordMetaDataBuilder metaData) { - final MetaDataValidator validator = new MetaDataValidator(metaData, IndexMaintainerRegistryImpl.instance()); + final MetaDataValidator validator = new MetaDataValidator(metaData, IndexMaintainerFactoryRegistryImpl.instance()); validator.validate(); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java index 821448b8cc..493bb67367 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/QueryPlanFullySortedTest.java @@ -28,7 +28,7 @@ import com.apple.foundationdb.record.metadata.IndexOptions; import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.Key; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.plan.QueryPlanner; @@ -59,7 +59,7 @@ protected void setup(@Nullable RecordMetaDataHook hook) { } metaData = builder.getRecordMetaData(); planner = isUseCascadesPlanner() ? - new CascadesPlanner(metaData, new RecordStoreState(null, null), IndexMaintainerRegistryImpl.instance()) : + new CascadesPlanner(metaData, new RecordStoreState(null, null), IndexMaintainerFactoryRegistryImpl.instance()) : new RecordQueryPlanner(metaData, new RecordStoreState(null, null)); } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java index d6cd9bf8fb..31632cf034 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordStoreState; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalConnection; @@ -86,7 +86,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); } } @@ -109,7 +109,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); } } @@ -123,7 +123,7 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerRegistryImpl.instance(), Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java index b5693037fb..59deba12e2 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/memory/InMemoryRelationalStatement.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.memory; import com.apple.foundationdb.record.RecordStoreState; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.Row; @@ -116,7 +116,7 @@ public T clock(@Nonnull RelationalMetric.RelationalEvent event, Supplier .withUserVersion(0) .build(); - final var planGenerator = PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerRegistryImpl.instance(), Options.NONE); + final var planGenerator = PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); final Plan plan = planGenerator.getPlan(sql); if (plan instanceof QueryPlan) { throw new SQLFeatureNotSupportedException("Cannot execute queries in the InMemory Relational version, it's only good for Direct Access API"); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java index b4a2da32f2..deb791a572 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.query.plan.cascades.RawSqlFunction; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.Options; @@ -611,7 +611,7 @@ public T clock(@Nonnull RelationalMetric.RelationalEvent event, .withPlannerConfiguration(PlannerConfiguration.ofAllAvailableIndexes()) .withUserVersion(0) .build(); - return PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerRegistryImpl.instance(), Options.NONE); + return PlanGenerator.create(Optional.empty(), ctx, ctx.getMetaData(), new RecordStoreState(null, Map.of()), IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); } } }