-
Notifications
You must be signed in to change notification settings - Fork 5.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SERVER-27123 Only update the commit point as a secondary from oplog q…
…ueries against your sync source
- Loading branch information
Showing
22 changed files
with
421 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| /* | ||
| * Tests that a node on a stale branch of history won't incorrectly mark its ops as committed even | ||
| * when hearing about a commit point with a higher optime from a new primary. | ||
| */ | ||
| (function() { | ||
| 'use strict'; | ||
|
|
||
| load("jstests/libs/check_log.js"); | ||
| load("jstests/libs/write_concern_util.js"); | ||
| load("jstests/replsets/rslib.js"); | ||
|
|
||
| var name = "readCommittedStaleHistory"; | ||
| var dbName = "wMajorityCheck"; | ||
| var collName = "stepdown"; | ||
|
|
||
| var rst = new ReplSetTest({ | ||
| name: name, | ||
| nodes: [ | ||
| {}, | ||
| {}, | ||
| {rsConfig: {priority: 0}}, | ||
| ], | ||
| nodeOptions: {enableMajorityReadConcern: ""}, | ||
| useBridge: true | ||
| }); | ||
|
|
||
| if (!startSetIfSupportsReadMajority(rst)) { | ||
| jsTest.log("skipping test since storage engine doesn't support committed reads"); | ||
| return; | ||
| } | ||
|
|
||
| var nodes = rst.nodes; | ||
| rst.initiate(); | ||
|
|
||
| /** | ||
| * Waits for the given node to be in state primary *and* have finished drain mode and thus | ||
| * be available for writes. | ||
| */ | ||
| function waitForPrimary(node) { | ||
| assert.soon(function() { | ||
| return node.adminCommand('ismaster').ismaster; | ||
| }); | ||
| } | ||
|
|
||
| function stepUp(node) { | ||
| var primary = rst.getPrimary(); | ||
| if (primary != node) { | ||
| assert.throws(function() { | ||
| primary.adminCommand({replSetStepDown: 60 * 5}); | ||
| }); | ||
| } | ||
| waitForPrimary(node); | ||
| } | ||
|
|
||
| // Asserts that the given document is not visible in the committed snapshot on the given node. | ||
| function checkDocNotCommitted(node, doc) { | ||
| var docs = | ||
| node.getDB(dbName).getCollection(collName).find(doc).readConcern('majority').toArray(); | ||
| assert.eq(0, docs.length, tojson(docs)); | ||
| } | ||
|
|
||
| jsTestLog("Make sure node 0 is primary."); | ||
| rst.getPrimary(); | ||
| stepUp(nodes[0]); | ||
| var primary = rst.getPrimary(); | ||
| var secondaries = rst.getSecondaries(); | ||
| assert.eq(nodes[0], primary); | ||
| // Wait for all data bearing nodes to get up to date. | ||
| assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( | ||
| {a: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); | ||
|
|
||
| // Stop the secondaries from replicating. | ||
| stopServerReplication(secondaries); | ||
| // Stop the primary from being able to complete stepping down. | ||
| assert.commandWorked( | ||
| nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'})); | ||
|
|
||
| jsTestLog("Do a write that won't ever reach a majority of nodes"); | ||
| assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert({a: 2})); | ||
|
|
||
| // Ensure that the write that was just done is not visible in the committed snapshot. | ||
| checkDocNotCommitted(nodes[0], {a: 2}); | ||
|
|
||
| // Prevent the primary from rolling back later on. | ||
| assert.commandWorked( | ||
| nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); | ||
|
|
||
| jsTest.log("Disconnect primary from all secondaries"); | ||
| nodes[0].disconnect(nodes[1]); | ||
| nodes[0].disconnect(nodes[2]); | ||
|
|
||
| jsTest.log("Wait for a new primary to be elected"); | ||
| // Allow the secondaries to replicate again. | ||
| restartServerReplication(secondaries); | ||
|
|
||
| waitForPrimary(nodes[1]); | ||
|
|
||
| jsTest.log("Do a write to the new primary"); | ||
| assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( | ||
| {a: 3}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); | ||
|
|
||
| jsTest.log("Reconnect the old primary to the rest of the nodes"); | ||
| nodes[1].reconnect(nodes[0]); | ||
| nodes[2].reconnect(nodes[0]); | ||
|
|
||
| // Sleep 10 seconds to allow some heartbeats to be processed, so we can verify that the | ||
| // heartbeats don't cause the stale primary to incorrectly advance the commit point. | ||
| sleep(10000); | ||
|
|
||
| checkDocNotCommitted(nodes[0], {a: 2}); | ||
|
|
||
| jsTest.log("Allow the old primary to finish stepping down and become secondary"); | ||
| var res = null; | ||
| try { | ||
| res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); | ||
| } catch (e) { | ||
| // Expected - once we disable the fail point the stepdown will proceed and it's racy whether | ||
| // the stepdown closes all connections before or after the configureFailPoint command | ||
| // returns | ||
| } | ||
| if (res) { | ||
| assert.commandWorked(res); | ||
| } | ||
| rst.waitForState(nodes[0], ReplSetTest.State.SECONDARY); | ||
|
|
||
| // At this point the former primary will attempt to go into rollback, but the | ||
| // 'rollbackHangBeforeStart' will prevent it from doing so. | ||
| checkDocNotCommitted(nodes[0], {a: 2}); | ||
| checkLog.contains(nodes[0], 'rollback - rollbackHangBeforeStart fail point enabled'); | ||
| checkDocNotCommitted(nodes[0], {a: 2}); | ||
|
|
||
| jsTest.log("Allow the original primary to roll back its write and catch up to the new primary"); | ||
| assert.commandWorked( | ||
| nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); | ||
|
|
||
| assert.soonNoExcept(function() { | ||
| return null == nodes[0].getDB(dbName).getCollection(collName).findOne({a: 2}); | ||
| }, "Original primary never rolled back its write"); | ||
|
|
||
| rst.awaitReplication(); | ||
|
|
||
| // Ensure that the old primary got the write that the new primary did and sees it as committed. | ||
| assert.neq( | ||
| null, | ||
| nodes[0].getDB(dbName).getCollection(collName).find({a: 3}).readConcern('majority').next()); | ||
|
|
||
| rst.stopSet(); | ||
| }()); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.