Skip to content

Commit ab9354c

Browse files
committed
Move the optimisation to upload Metadata method
Signed-off-by: sjs004 <simarjeet.singh004@gmail.com>
1 parent 8a2c65f commit ab9354c

File tree

1 file changed

+30
-14
lines changed

1 file changed

+30
-14
lines changed

server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ public final class RemoteStoreRefreshListener extends ReleasableRetryableRefresh
9191
private final SegmentReplicationCheckpointPublisher checkpointPublisher;
9292
private final RemoteStoreSettings remoteStoreSettings;
9393
private final RemoteStoreUploader remoteStoreUploader;
94+
private volatile long lastUploadedPrimaryTerm = INVALID_PRIMARY_TERM; // Use constant or -1
95+
private volatile long lastUploadedGeneration = -1;
96+
private volatile long lastUploadedTranslogGeneration = -1;
9497

9598
public RemoteStoreRefreshListener(
9699
IndexShard indexShard,
@@ -430,27 +433,40 @@ private boolean isRefreshAfterCommitSafe() {
430433

431434
void uploadMetadata(Collection<String> localSegmentsPostRefresh, SegmentInfos segmentInfos, ReplicationCheckpoint replicationCheckpoint)
432435
throws IOException {
436+
433437
final long maxSeqNo = ((InternalEngine) indexShard.getEngine()).currentOngoingRefreshCheckpoint();
438+
Translog.TranslogGeneration translogGeneration = indexShard.getEngine().translogManager().getTranslogGeneration();
439+
if (translogGeneration == null) {
440+
throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store");
441+
}
442+
long translogFileGeneration = translogGeneration.translogFileGeneration;
443+
if (this.lastUploadedPrimaryTerm == replicationCheckpoint.getPrimaryTerm()
444+
&& this.lastUploadedGeneration == segmentInfos.getGeneration()
445+
&& this.lastUploadedTranslogGeneration == translogFileGeneration) {
446+
447+
logger.debug("Skipping metadata upload (deduplicated) - state is already persisted.");
448+
return;
449+
}
450+
434451
SegmentInfos segmentInfosSnapshot = segmentInfos.clone();
435452
Map<String, String> userData = segmentInfosSnapshot.getUserData();
436453
userData.put(LOCAL_CHECKPOINT_KEY, String.valueOf(maxSeqNo));
437454
userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
438455
segmentInfosSnapshot.setUserData(userData, false);
439456

440-
Translog.TranslogGeneration translogGeneration = indexShard.getEngine().translogManager().getTranslogGeneration();
441-
if (translogGeneration == null) {
442-
throw new UnsupportedOperationException("Encountered null TranslogGeneration while uploading metadata to remote segment store");
443-
} else {
444-
long translogFileGeneration = translogGeneration.translogFileGeneration;
445-
remoteDirectory.uploadMetadata(
446-
localSegmentsPostRefresh,
447-
segmentInfosSnapshot,
448-
storeDirectory,
449-
translogFileGeneration,
450-
replicationCheckpoint,
451-
indexShard.getNodeId()
452-
);
453-
}
457+
remoteDirectory.uploadMetadata(
458+
localSegmentsPostRefresh,
459+
segmentInfosSnapshot,
460+
storeDirectory,
461+
translogFileGeneration,
462+
replicationCheckpoint,
463+
indexShard.getNodeId()
464+
);
465+
466+
// Update the Listener's cache on success
467+
this.lastUploadedPrimaryTerm = replicationCheckpoint.getPrimaryTerm();
468+
this.lastUploadedGeneration = segmentInfos.getGeneration();
469+
this.lastUploadedTranslogGeneration = translogFileGeneration;
454470
}
455471

456472
boolean isLowPriorityUpload() {

0 commit comments

Comments
 (0)