Skip to content

Commit 45917e9

Browse files
committed
Fix InstallSnapshotRPC bug
If a snapshot is already installed, we must notify the leader so that it can advance follower's match index
1 parent 0c77aef commit 45917e9

File tree

2 files changed

+73
-1
lines changed

2 files changed

+73
-1
lines changed

hazelcast/src/main/java/com/hazelcast/cp/internal/raft/impl/RaftNodeImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -764,7 +764,7 @@ public boolean installSnapshot(SnapshotEntry snapshot) {
764764
return false;
765765
} else if (commitIndex == snapshot.index()) {
766766
logger.warning("Ignored snapshot: " + snapshot + " since commit index is same.");
767-
return false;
767+
return true;
768768
}
769769

770770
state.commitIndex(snapshot.index());

hazelcast/src/test/java/com/hazelcast/cp/internal/raft/impl/SnapshotTest.java

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.hazelcast.cp.internal.raft.impl.dataservice.ApplyRaftRunnable;
2323
import com.hazelcast.cp.internal.raft.impl.dataservice.RaftDataService;
2424
import com.hazelcast.cp.internal.raft.impl.dto.AppendRequest;
25+
import com.hazelcast.cp.internal.raft.impl.dto.AppendSuccessResponse;
2526
import com.hazelcast.cp.internal.raft.impl.testing.LocalRaftGroup;
2627
import com.hazelcast.test.AssertTask;
2728
import com.hazelcast.test.HazelcastSerialClassRunner;
@@ -182,6 +183,77 @@ public void run() {
182183
});
183184
}
184185

186+
@Test
187+
public void when_leaderMissesInstallSnapshotResponse_then_itAdvancesMatchIndexWithNextInstallSnapshotResponse()
188+
throws ExecutionException, InterruptedException {
189+
final int entryCount = 50;
190+
RaftAlgorithmConfig raftAlgorithmConfig = new RaftAlgorithmConfig().setCommitIndexAdvanceCountToSnapshot(entryCount)
191+
.setAppendRequestBackoffTimeoutInMillis(1000);
192+
group = newGroupWithService(3, raftAlgorithmConfig);
193+
group.start();
194+
195+
final RaftNodeImpl leader = group.waitUntilLeaderElected();
196+
197+
RaftNodeImpl[] followers = group.getNodesExcept(leader.getLocalMember());
198+
final RaftNodeImpl slowFollower = followers[1];
199+
200+
// the leader cannot send AppendEntriesRPC to the follower
201+
group.dropMessagesToMember(leader.getLocalMember(), slowFollower.getLocalMember(), AppendRequest.class);
202+
203+
// the follower cannot send append response to the leader after installing the snapshot
204+
group.dropMessagesToMember(slowFollower.getLocalMember(), leader.getLocalMember(), AppendSuccessResponse.class);
205+
206+
for (int i = 0; i < entryCount; i++) {
207+
leader.replicate(new ApplyRaftRunnable("val" + i)).get();
208+
}
209+
210+
assertTrueEventually(new AssertTask() {
211+
@Override
212+
public void run() {
213+
assertEquals(entryCount, getSnapshotEntry(leader).index());
214+
}
215+
});
216+
217+
leader.replicate(new ApplyRaftRunnable("valFinal")).get();
218+
219+
group.resetAllDropRulesFrom(leader.getLocalMember());
220+
221+
assertTrueEventually(new AssertTask() {
222+
@Override
223+
public void run() {
224+
for (RaftNodeImpl raftNode : group.getNodesExcept(slowFollower.getLocalMember())) {
225+
assertEquals(entryCount + 1, getCommitIndex(raftNode));
226+
RaftDataService service = group.getService(raftNode);
227+
assertEquals(entryCount + 1, service.size());
228+
for (int i = 0; i < entryCount; i++) {
229+
assertEquals(("val" + i), service.get(i + 1));
230+
}
231+
assertEquals("valFinal", service.get(51));
232+
}
233+
234+
assertEquals(entryCount, getCommitIndex(slowFollower));
235+
RaftDataService service = group.getService(slowFollower);
236+
assertEquals(entryCount, service.size());
237+
for (int i = 0; i < entryCount; i++) {
238+
assertEquals(("val" + i), service.get(i + 1));
239+
}
240+
}
241+
});
242+
243+
group.resetAllDropRulesFrom(slowFollower.getLocalMember());
244+
245+
final long commitIndex = getCommitIndex(leader);
246+
247+
assertTrueEventually(new AssertTask() {
248+
@Override
249+
public void run() {
250+
for (RaftNode raftNode : group.getNodesExcept(leader.getLocalMember())) {
251+
assertEquals(commitIndex, getMatchIndex(leader, raftNode.getLocalMember()));
252+
}
253+
}
254+
});
255+
}
256+
185257
@Test
186258
public void when_followerMissesTheLastEntryThatGoesIntoTheSnapshot_then_itInstallsSnapshot() throws ExecutionException, InterruptedException {
187259
final int entryCount = 50;

0 commit comments

Comments
 (0)