Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release-1.8] Fix broken initial offset on reset on a non64 bit boundary #3028

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,9 @@ private void reset() {

final var prevCommittedOffsetsArr = committedOffsets.toLongArray();
// Calculate the word index in the long array. Long size is 64.
final var wordSize = 64;
final var relativeOffset = committed - initialOffset;
final var wordOfCommitted = (int) (relativeOffset / 64);
final var wordOfCommitted = (int) (relativeOffset / wordSize);

// Copy from wordOfCommitted to the end: [..., wordOfCommitted, ...]
final var newCommittedOffsetsArr = Arrays.copyOfRange(
Expand All @@ -305,7 +306,7 @@ private void reset() {

// Re-create committedOffset BitSet and reset initialOffset.
this.committedOffsets = BitSet.valueOf(newCommittedOffsetsArr);
this.initialOffset = committed;
this.initialOffset = committed - (relativeOffset % wordSize);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,42 @@ public void shouldContinueToWorkAfterSendingALotOfRecords() {
.containsEntry(new TopicPartition("aaa", 0), 128L * 64L + 1L);
}

@Test
public void shouldContinueToWorkAfterResetWithOutOfOrderEventsOnANon64BitBoundary() {
assertThatOffsetCommitted(List.of(new TopicPartition("aaa", 0)), offsetStrategy -> {
offsetStrategy.recordReceived(record("aaa", 0, 0));

// This loop will flag 1 bit in the BitSet of the OffsetTracker beyond a 64-bit
// word boundary.
for (int i = 0; i < (1_000_000 - 64 + 1); i++) {
offsetStrategy.successfullySentToSubscriber(record("aaa", 0, i));
}

// The rest of the BitSet until the reset threshold of 1_000_000 is set with
// alternating 1/0.
for (int i = (1_000_000 - 64 + 2); i <= 1_000_000; i += 2) {
offsetStrategy.successfullySentToSubscriber(record("aaa", 0, i));
}

// Now we need to make sure that our offset got committed.
try {
Thread.sleep(1000);
} catch (final InterruptedException e) {
throw new RuntimeException(e);
}

// Now set one of the bits beyond the 1_000_000 mark, resetting the
// OffsetTracker.
offsetStrategy.successfullySentToSubscriber(record("aaa", 0, 1_000_001));

// Now fill up the gaps in the BitSet of the OffsetTracker.
for (int i = (1_000_000 - 64 + 1); i <= 1_000_000; i += 2) {
offsetStrategy.successfullySentToSubscriber(record("aaa", 0, i));
}
})
.containsEntry(new TopicPartition("aaa", 0), 1_000_002L);
}

@Test
@SuppressWarnings("unchecked")
public void recordReceived(final Vertx vertx) {
Expand Down