From fff4cbb8cb168d4bf6d591ceb012080bf9b853f0 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 12 Nov 2025 14:12:08 -0800 Subject: [PATCH 1/3] skip generating rand index match candidate --- .../plan/cascades/MetaDataPlanContext.java | 11 +- .../MetaDataPlanContextRankIndexTest.java | 193 ++++++++++++++++++ 2 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index 1450ec1814..771a098061 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -24,12 +24,14 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -199,7 +201,14 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf final ImmutableSet.Builder matchCandidatesBuilder = ImmutableSet.builder(); for (final var index : indexList) { - matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, false)); + if (IndexTypes.RANK.equals(index.getType())) { + final IndexExpansionInfo info = IndexExpansionInfo.createInfo(metaData, index, false); + // For rank() we still want to create the match candidate for BY_VALUE scans. + MatchCandidateExpansion.expandValueIndexMatchCandidate(info) + .ifPresent(matchCandidatesBuilder::add); + } else { + matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, false)); + } } for (final var recordType : queriedRecordTypes) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java new file mode 100644 index 0000000000..6d85fe9a26 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java @@ -0,0 +1,193 @@ +/* + * MetaDataPlanContextRankIndexTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordMetaDataBuilder; +import com.apple.foundationdb.record.RecordStoreState; +import com.apple.foundationdb.record.TestRecordsRankProto; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexTypes; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; +import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; +import com.apple.foundationdb.record.query.IndexQueryabilityFilter; +import com.apple.foundationdb.record.query.RecordQuery; +import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.test.Tags; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.apple.foundationdb.record.metadata.Key.Expressions.field; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link MetaDataPlanContext} methods with rank indexes, specifically testing: + * + */ +@Tag(Tags.RequiresFDB) +class MetaDataPlanContextRankIndexTest extends FDBRecordStoreQueryTestBase { + + private RecordMetaData setupMetaDataWithRankIndex() { + RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsRankProto.getDescriptor()); + + // Set primary key for HeaderRankedRecord (required by the proto definition) + metaDataBuilder.getRecordType("HeaderRankedRecord") + .setPrimaryKey(field("header").nest(field("group"), field("id"))); + + // Add a rank index on BasicRankedRecord + metaDataBuilder.addIndex("BasicRankedRecord", + new Index("rank_by_gender", + field("score").groupBy(field("gender")), + IndexTypes.RANK)); + + // Add another rank index with different structure + metaDataBuilder.addIndex("BasicRankedRecord", + new Index("simple_rank_score", + field("score").ungrouped(), + IndexTypes.RANK)); + + return metaDataBuilder.getRecordMetaData(); + } + + @Test + void testForRootReferenceRankIndexOnlyCreatesValueScanCandidate() { + try (FDBRecordContext context = openContext()) { + final RecordMetaData metaData = setupMetaDataWithRankIndex(); + + // Create and open a record store to get the state + createOrOpenRecordStore(context, metaData); + final RecordStoreState recordStoreState = recordStore.getRecordStoreState(); + + // Create a root reference for BasicRankedRecord + final FullUnorderedScanExpression scanExpression = new FullUnorderedScanExpression( + ImmutableSet.of(metaData.getRecordType("BasicRankedRecord").getName()), + Type.Record.fromDescriptor(metaData.getRecordType("BasicRankedRecord").getDescriptor()), + new AccessHints()); + + final Reference rootReference = Reference.initialOf(scanExpression); + + // Create plan context using forRootReference + final PlanContext planContext = MetaDataPlanContext.forRootReference( + RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), + metaData, + recordStoreState, + IndexMaintainerFactoryRegistryImpl.instance(), + rootReference, + Optional.empty(), + IndexQueryabilityFilter.DEFAULT); + + // Get the match candidates + final Set matchCandidates = planContext.getMatchCandidates(); + // 3 rank index, each generates a ValueIndexScanMatchCandidate, and primary key generates a match candidate + assertEquals(4, matchCandidates.size()); + + // Filter to only ValueIndexScanMatchCandidate with our rank index names + final Set rankIndexCandidates = matchCandidates.stream() + .filter(candidate -> candidate.getName().equals("rank_by_gender") || + candidate.getName().equals("simple_rank_score") || + candidate.getName().equals("BasicRankedRecord$score")) + .collect(Collectors.toSet()); + + // Verify that we have rank index candidates + assertFalse(rankIndexCandidates.isEmpty(), "Should have rank index candidates"); + assertEquals(3, rankIndexCandidates.size(), "Should have exactly 2 rank index candidates"); + + // Verify all are ValueIndexScanMatchCandidate (for BY_VALUE scans) + for (MatchCandidate candidate : rankIndexCandidates) { + assertInstanceOf(ValueIndexScanMatchCandidate.class, candidate, "Rank index candidate should be a ValueIndexScanMatchCandidate, got: " + candidate.getClass().getName()); + } + + // Verify no WindowedIndexScanMatchCandidate (BY_RANK scans) are created for rank indexes + long windowedCandidateCount = matchCandidates.stream() + .filter(candidate -> candidate instanceof WindowedIndexScanMatchCandidate) + .count(); + + assertEquals(0, windowedCandidateCount, + "Should not have any WindowedIndexScanMatchCandidate"); + } + } + + @Test + void testForRecordQueryRankIndexCreatesWindowedScanCandidate() { + try (FDBRecordContext context = openContext()) { + final RecordMetaData metaData = setupMetaDataWithRankIndex(); + + // Create and open a record store to get the state + createOrOpenRecordStore(context, metaData); + final RecordStoreState recordStoreState = recordStore.getRecordStoreState(); + + // Create a simple RecordQuery for BasicRankedRecord + final RecordQuery query = RecordQuery.newBuilder() + .setRecordType("BasicRankedRecord") + .build(); + + // Create plan context using forRecordQuery + final PlanContext planContext = MetaDataPlanContext.forRecordQuery( + RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), + metaData, + recordStoreState, + IndexMaintainerFactoryRegistryImpl.instance(), + query); + + // Get the match candidates + final Set matchCandidates = planContext.getMatchCandidates(); + assertEquals(7, matchCandidates.size()); + + // Filter to ValueIndexScanMatchCandidate with our rank index names + final Set valueIndexCandidateNames = matchCandidates.stream() + .filter(candidate -> candidate instanceof ValueIndexScanMatchCandidate) + .map(MatchCandidate::getName) + .collect(Collectors.toSet()); + + // Verify that we have ValueIndexScanMatchCandidate for rank indexes + assertEquals(ImmutableSet.of("rank_by_gender", "simple_rank_score", "BasicRankedRecord$score"), valueIndexCandidateNames, + "Should have ValueIndexScanMatchCandidate for both rank indexes"); + + // Filter to WindowedIndexScanMatchCandidate with our rank index names + final Set windowedIndexCandidateNames = matchCandidates.stream() + .filter(candidate -> candidate instanceof WindowedIndexScanMatchCandidate) + .map(MatchCandidate::getName) + .collect(Collectors.toSet()); + + // Verify that we have WindowedIndexScanMatchCandidate for rank indexes (BY_RANK scans) + assertEquals(ImmutableSet.of("rank_by_gender", "simple_rank_score", "BasicRankedRecord$score"), windowedIndexCandidateNames, + "Should have WindowedIndexScanMatchCandidate for both rank indexes"); + } + } +} From 55ac3d5805dbc3cf95befb535537926157a43510 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 12 Nov 2025 14:14:35 -0800 Subject: [PATCH 2/3] remove unused import --- .../record/query/plan/cascades/MetaDataPlanContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index 771a098061..cb3b44cd3d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; From 2efb934592cb086748d51643ec969bd6bd41d87f Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 12 Nov 2025 22:50:02 -0800 Subject: [PATCH 3/3] style --- .../query/plan/cascades/MetaDataPlanContextRankIndexTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java index 6d85fe9a26..d55dacd01a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextRankIndexTest.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.TestRecordsRankProto; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; -import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; @@ -48,10 +47,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Tests for {@link MetaDataPlanContext} methods with rank indexes, specifically testing: + * Tests for {@link MetaDataPlanContext} methods with rank indexes. *
    *
  • {@link MetaDataPlanContext#forRootReference} - verifies rank indexes only create * match candidates for value scans (BY_VALUE) and NOT for rank scans (BY_RANK)