diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java index 9eb1a779c4..1ec8535a74 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java @@ -35,7 +35,6 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingRangeSet; -import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.Message; import org.slf4j.Logger; @@ -111,24 +110,17 @@ static IndexBuildProto.IndexBuildIndexingStamp compileIndexingTypeStamp() { @Override CompletableFuture buildIndexInternalAsync() { return getRunner().runAsync(context -> - context.getReadVersionAsync() - .thenCompose(vignore -> { - SubspaceProvider subspaceProvider = common.getRecordStoreBuilder().getSubspaceProvider(); - return subspaceProvider.getSubspaceAsync(context) - .thenCompose(subspace -> - indexScrub(subspaceProvider, subspace)); - }), + context.getReadVersionAsync().thenCompose(ignore -> indexScrub()), common.indexLogMessageKeyValues("IndexScrubbing::buildIndexInternalAsync")); } @Nonnull - private CompletableFuture indexScrub(@Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture indexScrub() { final List additionalLogMessageKeyValues = Arrays.asList(LogMessageKeys.CALLING_METHOD, "indexScrub"); return iterateAllRanges(additionalLogMessageKeyValues, - this::indexScrubRangeOnly, - subspaceProvider, subspace); + this::indexScrubRangeOnly); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingBase.java index 830edf0135..fed5f81328 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingBase.java @@ -49,7 +49,6 @@ import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan; import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner; import com.apple.foundationdb.subspace.Subspace; -import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.Message; import org.slf4j.Logger; @@ -500,7 +499,7 @@ private RecordCoreException newPartlyBuiltException(IndexBuildProto.IndexBuildIn } // Helpers for implementing modules. Some of them are public to support unit-testing. - protected CompletableFuture doneOrThrottleDelayAndMaybeLogProgress(boolean done, SubspaceProvider subspaceProvider, List additionalLogMessageKeyValues) { + protected CompletableFuture doneOrThrottleDelayAndMaybeLogProgress(boolean done, List additionalLogMessageKeyValues) { if (done) { return AsyncUtil.READY_FALSE; } @@ -514,8 +513,7 @@ protected CompletableFuture doneOrThrottleDelayAndMaybeLogProgress(bool lastProgressSnapshot = StoreTimerSnapshot.from(timer); } LOGGER.info(KeyValueLogMessage.build("Indexer: Built Range", - subspaceProvider.logKey(), subspaceProvider, - LogMessageKeys.DELAY, toWait) + LogMessageKeys.DELAY, toWait) .addKeysAndValues(additionalLogMessageKeyValues != null ? additionalLogMessageKeyValues : Collections.emptyList()) .addKeysAndValues(indexingLogMessageKeyValues()) .addKeysAndValues(common.indexLogMessageKeyValues()) @@ -907,14 +905,12 @@ private CompletableFuture updateMaintainerBuilder(@Nonnull FDBRecordStore } protected CompletableFuture iterateAllRanges(List additionalLogMessageKeyValues, - BiFunction> iterateRange, - @Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { - return iterateAllRanges(additionalLogMessageKeyValues, iterateRange, subspaceProvider, subspace, null); + BiFunction> iterateRange) { + return iterateAllRanges(additionalLogMessageKeyValues, iterateRange, null); } protected CompletableFuture iterateAllRanges(List additionalLogMessageKeyValues, BiFunction> iterateRange, - @Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace, @Nullable Function> shouldReturnQuietly) { return AsyncUtil.whileTrue(() -> @@ -923,15 +919,16 @@ protected CompletableFuture iterateAllRanges(List additionalLogMes if (ex == null) { final Set indexSet = throttle.getAndResetMergeRequiredIndexes(); if (indexSet != null && !indexSet.isEmpty()) { - return mergeIndexes(indexSet, subspaceProvider) - .thenCompose(ignore -> doneOrThrottleDelayAndMaybeLogProgress(!hasMore, subspaceProvider, additionalLogMessageKeyValues)); + return mergeIndexes(indexSet) + .thenCompose(ignore -> doneOrThrottleDelayAndMaybeLogProgress(!hasMore, additionalLogMessageKeyValues)); } - return doneOrThrottleDelayAndMaybeLogProgress(!hasMore, subspaceProvider, additionalLogMessageKeyValues); + return doneOrThrottleDelayAndMaybeLogProgress(!hasMore, additionalLogMessageKeyValues); } final RuntimeException unwrappedEx = getRunner().getDatabase().mapAsyncToSyncException(ex); if (LOGGER.isInfoEnabled()) { - LOGGER.info(KeyValueLogMessage.of("possibly non-fatal error encountered building range", - LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspace.pack())), ex); + LOGGER.info(KeyValueLogMessage.build("possibly non-fatal error encountered building range") + .addKeysAndValues(common.indexLogMessageKeyValues()) + .toString(), ex); } throw unwrappedEx; }).thenCompose(Function.identity()), @@ -939,12 +936,12 @@ protected CompletableFuture iterateAllRanges(List additionalLogMes } public CompletableFuture mergeIndexes() { - return mergeIndexes(new HashSet<>(common.getTargetIndexes()), common.getRecordStoreBuilder().subspaceProvider); + return mergeIndexes(new HashSet<>(common.getTargetIndexes())); } - private CompletableFuture mergeIndexes(Set indexSet, @Nullable SubspaceProvider subspaceProvider) { + private CompletableFuture mergeIndexes(Set indexSet) { return AsyncUtil.whenAll(indexSet.stream() - .map(index -> getIndexingMerger(index).mergeIndex(subspaceProvider) + .map(index -> getIndexingMerger(index).mergeIndex() ).collect(Collectors.toList())); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingByIndex.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingByIndex.java index d76916be57..991961f0c9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingByIndex.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingByIndex.java @@ -37,7 +37,6 @@ import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingRangeSet; -import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.Message; import com.google.protobuf.ZeroCopyByteString; @@ -114,20 +113,15 @@ CompletableFuture buildIndexInternalAsync() { validateSourceAndTargetIndexes(store); // all valid; back to the future. Note that for practical reasons, readability and idempotency will be validated later return context.getReadVersionAsync() - .thenCompose(vignore -> { - SubspaceProvider subspaceProvider = common.getRecordStoreBuilder().getSubspaceProvider(); - return subspaceProvider.getSubspaceAsync(context) - .thenCompose(subspace -> buildIndexFromIndex(subspaceProvider, subspace)); - }); + .thenCompose(ignore -> buildIndexFromIndex()); }), common.indexLogMessageKeyValues("IndexingByIndex::buildIndexInternalAsync")); } @Nonnull - private CompletableFuture buildIndexFromIndex(@Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture buildIndexFromIndex() { final List additionalLogMessageKeyValues = Arrays.asList(LogMessageKeys.CALLING_METHOD, "buildIndexFromIndex"); return iterateAllRanges(additionalLogMessageKeyValues, - (store, recordsScanned) -> buildRangeOnly(store, recordsScanned), - subspaceProvider, subspace); + this::buildRangeOnly); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingCommon.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingCommon.java index 4b9b44e3fa..957d049e56 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingCommon.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingCommon.java @@ -158,6 +158,12 @@ LogMessageKeys.TARGET_INDEX_NAME, getTargetIndexesNames(), LogMessageKeys.RECORDS_SCANNED, totalRecordsScanned.get(), LogMessageKeys.INDEXER_ID, indexerId); + SubspaceProvider subspaceProvider = getRecordStoreBuilder().getSubspaceProvider(); + if (subspaceProvider != null) { + keyValues.add(subspaceProvider.logKey()); + keyValues.add(subspaceProvider); + } + if (moreKeyValues != null && !moreKeyValues.isEmpty()) { keyValues.addAll(moreKeyValues); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMerger.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMerger.java index 043c70fe20..93a4ae3735 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMerger.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMerger.java @@ -29,8 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.annotation.ParametersAreNonnullByDefault; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -52,7 +50,6 @@ public class IndexingMerger { private int mergeSuccesses = 0; private long timeQuotaMillis = 0; private final IndexingCommon common; - private SubspaceProvider subspaceProvider = null; private int repartitionDocumentCount = 0; private int repartitionSecondChances = 0; @@ -62,13 +59,12 @@ public IndexingMerger(final Index index, IndexingCommon common, long initialMer this.mergesLimit = initialMergesCountLimit; } - private CompletableFuture openRecordStore(@Nonnull FDBRecordContext context) { + private CompletableFuture openRecordStore(FDBRecordContext context) { return common.getRecordStoreBuilder().copyBuilder().setContext(context).openAsync(); } @SuppressWarnings("squid:S3776") // cognitive complexity is high, candidate for refactoring - CompletableFuture mergeIndex(@Nullable SubspaceProvider subspaceProvider) { - this.subspaceProvider = subspaceProvider; // for logs only + CompletableFuture mergeIndex() { final AtomicInteger failureCountLimit = new AtomicInteger(1000); AtomicReference mergeControlRef = new AtomicReference<>(); final FDBStoreTimer timer = common.getRunner().getTimer(); @@ -249,6 +245,7 @@ List mergerKeysAndValues(final IndexDeferredMaintenanceControl mergeCont String mergerLogMessage(String ttl, final IndexDeferredMaintenanceControl mergeControl) { final KeyValueLogMessage msg = KeyValueLogMessage.build(ttl); msg.addKeysAndValues(mergerKeysAndValues(mergeControl)); + SubspaceProvider subspaceProvider = common.getRecordStoreBuilder().getSubspaceProvider(); if (subspaceProvider != null) { msg.addKeyAndValue(subspaceProvider.logKey(), subspaceProvider); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMultiTargetByRecords.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMultiTargetByRecords.java index 358681a887..6d37491832 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMultiTargetByRecords.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMultiTargetByRecords.java @@ -34,7 +34,6 @@ import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingRangeSet; -import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.Message; @@ -112,17 +111,12 @@ List indexingLogMessageKeyValues() { CompletableFuture buildIndexInternalAsync() { return getRunner().runAsync(context -> openRecordStore(context) .thenCompose( store -> context.getReadVersionAsync() - .thenCompose(vignore -> { - SubspaceProvider subspaceProvider = common.getRecordStoreBuilder().getSubspaceProvider(); - // validation checks, if any, will be performed here - return subspaceProvider.getSubspaceAsync(context) - .thenCompose(subspace -> buildMultiTargetIndex(subspaceProvider, subspace)); - }) - ), common.indexLogMessageKeyValues("IndexingMultiTargetByRecords::buildIndexInternalAsync")); + .thenCompose(ignore -> buildMultiTargetIndex())), + common.indexLogMessageKeyValues("IndexingMultiTargetByRecords::buildIndexInternalAsync")); } @Nonnull - private CompletableFuture buildMultiTargetIndex(@Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture buildMultiTargetIndex() { final TupleRange tupleRange = common.computeRecordsRange(); final byte[] rangeStart; final byte[] rangeEnd; @@ -157,7 +151,7 @@ private CompletableFuture buildMultiTargetIndex(@Nonnull SubspaceProvider LogMessageKeys.RANGE_END, rangeEnd); return maybePresetRangeFuture.thenCompose(ignore -> - iterateAllRanges(additionalLogMessageKeyValues, this::buildRangeOnly, subspaceProvider, subspace)); + iterateAllRanges(additionalLogMessageKeyValues, this::buildRangeOnly)); } @Nonnull @@ -218,7 +212,6 @@ private static CompletableFuture insertRanges(List range return AsyncUtil.whenAll(rangeSets.stream().map(set -> set.insertRangeAsync(start, end, true)).collect(Collectors.toList())); } - @Nullable @SuppressWarnings("unused") private CompletableFuture> getRecordIfTypeMatch(FDBRecordStore store, @Nonnull RecordCursorResult> cursorResult) { // No need to "translate" rec, so store is unused diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMutuallyByRecords.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMutuallyByRecords.java index e1dac1a4e2..e4a57bdd59 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMutuallyByRecords.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMutuallyByRecords.java @@ -33,7 +33,6 @@ import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingRangeSet; -import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; import com.google.common.annotations.VisibleForTesting; @@ -277,13 +276,10 @@ private void fragmentIterationTypePlusPlus() { CompletableFuture buildIndexInternalAsync() { return getRunner().runAsync(context -> openRecordStore(context) .thenCompose( store -> context.getReadVersionAsync() - .thenCompose(vignore -> { + .thenCompose(ignore -> { // init fragment data setFragmentationData(store); - SubspaceProvider subspaceProvider = common.getRecordStoreBuilder().getSubspaceProvider(); - // validation checks, if any, will be performed here - return subspaceProvider.getSubspaceAsync(context) - .thenCompose(subspace -> buildMultiTargetIndex(subspaceProvider, subspace)); + return buildMultiTargetIndex(); }) ), common.indexLogMessageKeyValues("IndexingMutuallyByRecords::buildIndexInternalAsync", @@ -291,7 +287,7 @@ CompletableFuture buildIndexInternalAsync() { } @Nonnull - private CompletableFuture buildMultiTargetIndex(@Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture buildMultiTargetIndex() { final TupleRange tupleRange = common.computeRecordsRange(); final byte[] rangeStart; final byte[] rangeEnd; @@ -325,13 +321,11 @@ private CompletableFuture buildMultiTargetIndex(@Nonnull SubspaceProvider return maybePresetRangeFuture.thenCompose(ignore -> iterateAllRanges(additionalLogMessageKeyValues, - (store, recordsScanned) -> buildRangeOnly(store, subspaceProvider, subspace), - subspaceProvider, subspace)); + (store, recordsScanned) -> buildRangeOnly(store))); } @Nonnull - private CompletableFuture buildRangeOnly(@Nonnull FDBRecordStore store, - @Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture buildRangeOnly(@Nonnull FDBRecordStore store) { // return false when done /* Mutual indexing: * 1. detects missing ranges @@ -348,11 +342,10 @@ private CompletableFuture buildRangeOnly(@Nonnull FDBRecordStore store, validateSameMetadataOrThrow(store); IndexingRangeSet rangeSet = IndexingRangeSet.forIndexBuild(store, common.getPrimaryIndex()); return rangeSet.listMissingRangesAsync().thenCompose(missingRanges -> - buildNextRangeOnly(sortAndSquash(missingRanges), subspaceProvider, subspace)); + buildNextRangeOnly(sortAndSquash(missingRanges))); } - private CompletableFuture buildNextRangeOnly(List missingRanges, - @Nonnull SubspaceProvider subspaceProvider, @Nonnull Subspace subspace) { + private CompletableFuture buildNextRangeOnly(List missingRanges) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(KeyValueLogMessage.of("buildNextRangeOnly", LogMessageKeys.MISSING_RANGES, missingRanges)); @@ -396,7 +389,6 @@ private CompletableFuture buildNextRangeOnly(List missingRanges, additionalLogMessageKeyValues.addAll(fragmentLogMessageKeyValues()); return iterateAllRanges(additionalLogMessageKeyValues, (store, recordsScanned) -> buildThisRangeOnly(store, recordsScanned, rangeToBuild), - subspaceProvider, subspace, anyJumperCallback(rangeToBuild) ).thenCompose(ignore -> AsyncUtil.READY_TRUE); }