Skip to content

Commit

Permalink
SERVER-77656 Add multitenancy rollback test
Browse files Browse the repository at this point in the history
  • Loading branch information
nadeaudi authored and Evergreen Agent committed Jun 28, 2023
1 parent 6ce6da4 commit 1da8ac4
Show file tree
Hide file tree
Showing 14 changed files with 431 additions and 13 deletions.
45 changes: 39 additions & 6 deletions jstests/replsets/libs/rollback_test.js
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,34 @@ function RollbackTest(name = "RollbackTest", replSet, nodeOptions) {

// Make sure we have a replica set up and running.
replSet = (replSet === undefined) ? performStandardSetup(nodeOptions) : replSet;

// Return an helper function to set a tenantId on commands if it is required.
let addTenantIdIfNeeded = (function() {
const adminDB = replSet.getPrimary().getDB("admin");
const flagDoc = assert.commandWorked(
adminDB.adminCommand({getParameter: 1, featureFlagRequireTenantID: 1}));
const multitenancyDoc =
assert.commandWorked(adminDB.adminCommand({getParameter: 1, multitenancySupport: 1}));
const fcvDoc = assert.commandWorked(
adminDB.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}));
if (multitenancyDoc.hasOwnProperty("multitenancySupport") &&
multitenancyDoc.multitenancySupport &&
flagDoc.hasOwnProperty("featureFlagRequireTenantID") &&
flagDoc.featureFlagRequireTenantID.value &&
MongoRunner.compareBinVersions(fcvDoc.featureCompatibilityVersion.version,
flagDoc.featureFlagRequireTenantID.version) >= 0) {
const tenantId = ObjectId();

return function(cmdObj) {
return Object.assign(cmdObj, {'$tenant': tenantId});
};
} else {
return function(cmdObj) {
return cmdObj;
};
}
})();

validateAndUseSetup(replSet);

// Majority writes in the initial phase, before transitionToRollbackOperations(), should be
Expand Down Expand Up @@ -181,9 +209,12 @@ function RollbackTest(name = "RollbackTest", replSet, nodeOptions) {
// ensure the insert was replicated and written to the on-disk journal of all 3
// nodes, with the exception of ephemeral and in-memory storage engines where
// journaling isn't supported.
assert.commandWorked(curPrimary.getDB(dbName).ensureSyncSource.insert(
{thisDocument: 'is inserted to ensure any node can sync from any other'},
{writeConcern: {w: 3, j: config.writeConcernMajorityJournalDefault}}));

assert.commandWorked(curPrimary.getDB(dbName).runCommand(addTenantIdIfNeeded({
insert: "ensureSyncSource",
documents: [{thisDocument: 'is inserted to ensure any node can sync from any other'}],
writeConcern: {w: 3, j: config.writeConcernMajorityJournalDefault}
})));
}

/**
Expand Down Expand Up @@ -455,9 +486,11 @@ function RollbackTest(name = "RollbackTest", replSet, nodeOptions) {
// storage engines are an exception because journaling isn't supported.
let writeConcern = TestData.rollbackShutdowns ? {w: 1, j: true} : {w: 1};
let dbName = "EnsureThereIsAtLeastOneOperationToRollback";
assert.commandWorked(curPrimary.getDB(dbName).ensureRollback.insert(
{thisDocument: 'is inserted to ensure rollback is not skipped'},
{writeConcern: writeConcern}));
assert.commandWorked(curPrimary.getDB(dbName).runCommand(addTenantIdIfNeeded({
insert: "ensureRollback",
documents: [{thisDocument: 'is inserted to ensure rollback is not skipped'}],
writeConcern
})));

log(`Isolating the primary ${curPrimary.host} so it will step down`);
// We should have already disconnected the primary from the secondary during the first stage
Expand Down
4 changes: 2 additions & 2 deletions jstests/replsets/rslib.js
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,13 @@ reconnect = function(conn) {
try {
// Make this work with either dbs or connections.
if (typeof (conn.getDB) == "function") {
db = conn.getDB('foo');
db = conn.getDB('config');
} else {
db = conn;
}

// Run a simple command to re-establish connection.
db.bar.stats();
db.settings.stats();

// SERVER-4241: Shell connections don't re-authenticate on reconnect.
if (jsTest.options().keyFile) {
Expand Down
169 changes: 169 additions & 0 deletions jstests/serverless/multitenancy_rollback_crud_op.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
* Test of a successfull replica set rollback for basic CRUD operations in multitenancy environment
* with featureFlagRequireTenantId. This test is modeled from rollback_crud_ops_sequence.js.
*/
load('jstests/replsets/libs/rollback_test.js');

(function() {
"use strict";

const kColl = "bar";
const tenantA = ObjectId();
const tenantB = ObjectId();

const insertDocs = function(db, coll, tenant, documents) {
assert.commandWorked(db.runCommand({insert: coll, documents, '$tenant': tenant}));
};

const updateDocs = function(db, coll, tenant, updates) {
assert.commandWorked(db.runCommand({update: coll, updates, '$tenant': tenant}));
};

const deleteMany = function(db, coll, tenant, query) {
assert.commandWorked(db.runCommand({
delete: coll,
deletes: [
{q: query, limit: 0},
],
'$tenant': tenant

}));
};

const validateCounts = function(db, coll, tenant, expect) {
for (let expected of expect) {
let res = db.runCommand({count: coll, query: expected.q, '$tenant': tenant});
assert.eq(res.n, expected.n);
}
};

// Helper function for verifying contents at the end of the test.
const checkFinalResults = function(db) {
validateCounts(db, kColl, tenantA, [
{q: {q: 70}, n: 0},
{q: {q: 40}, n: 2},
{q: {a: 'foo'}, n: 3},
{q: {q: {$gt: -1}}, n: 6},
{q: {txt: 'foo'}, n: 1},
{q: {q: 4}, n: 0}
]);

validateCounts(db, kColl, tenantB, [{q: {q: 1}, n: 1}, {q: {q: 40}, n: 0}]);

let res = db.runCommand({find: kColl, filter: {q: 0}, '$tenant': tenantA});
assert.eq(res.cursor.firstBatch.length, 1);
assert.eq(res.cursor.firstBatch[0].y, 33);

res = db.runCommand({find: 'kap', '$tenant': tenantA});
assert.eq(res.cursor.firstBatch.length, 1);

res = db.runCommand({find: 'kap2', '$tenant': tenantA});
assert.eq(res.cursor.firstBatch.length, 0);
};

function setFastGetMoreEnabled(node) {
assert.commandWorked(
node.adminCommand({configureFailPoint: 'setSmallOplogGetMoreMaxTimeMS', mode: 'alwaysOn'}),
`Failed to enable setSmallOplogGetMoreMaxTimeMS failpoint.`);
}

function setUpRst() {
const replSet = new ReplSetTest({
nodes: 3,
useBridge: true,
nodeOptions: {setParameter: {multitenancySupport: true, featureFlagRequireTenantID: true}}
});
replSet.startSet();
replSet.nodes.forEach(setFastGetMoreEnabled);

let config = replSet.getReplSetConfig();
config.members[2].priority = 0;
config.settings = {chainingAllowed: false};
replSet.initiateWithHighElectionTimeout(config);
// Tiebreaker's replication is paused for most of the test, avoid falling off the oplog.
replSet.nodes.forEach((node) => {
assert.commandWorked(node.adminCommand({replSetResizeOplog: 1, minRetentionHours: 2}));
});

assert.eq(replSet.nodes.length,
3,
"Mismatch between number of data bearing nodes and test configuration.");

return replSet;
}

const replSet = setUpRst();
const rollbackTest = new RollbackTest("MultitenancyRollbackTest", replSet);

const rollbackNode = rollbackTest.getPrimary();
rollbackNode.setSecondaryOk();
const syncSource = rollbackTest.getSecondary();
syncSource.setSecondaryOk();

const rollbackNodeDB = rollbackNode.getDB("foo");
const syncSourceDB = syncSource.getDB("foo");

// Insert initial data for both nodes.
insertDocs(rollbackNodeDB, kColl, tenantA, [{q: -2}, {q: 0}, {q: 1, a: "foo"}]);
insertDocs(rollbackNodeDB, kColl, tenantB, [{q: 1}, {q: 40, a: "foo"}]);
insertDocs(rollbackNodeDB, kColl, tenantA, [
{q: 2, a: "foo", x: 1},
{q: 3, bb: 9, a: "foo"},
{q: 40, a: 1},
{q: 40, a: 2},
{q: 70, txt: 'willremove'}
]);

// Testing capped collection.
rollbackNodeDB.createCollection("kap", {'$tenant': tenantA, capped: true, size: 5000});
insertDocs(rollbackNodeDB, 'kap', tenantA, [{foo: 1}]);
// Going back to empty on capped is a special case and must be tested.
rollbackNodeDB.createCollection("kap2", {'$tenant': tenantA, capped: true, size: 5000});

rollbackTest.awaitReplication();
rollbackTest.transitionToRollbackOperations();

// These operations are only done on 'rollbackNode' and should eventually be rolled back.
insertDocs(rollbackNodeDB, kColl, tenantA, [{q: 4}]);
updateDocs(rollbackNodeDB, kColl, tenantA, [
{q: {q: 3}, u: {q: 3, rb: true}},
]);
insertDocs(rollbackNodeDB, kColl, tenantB, [{q: 1, foo: 2}]);
deleteMany(rollbackNodeDB, kColl, tenantA, {q: 40});
updateDocs(rollbackNodeDB, kColl, tenantA, [
{q: {q: 2}, u: {q: 39, rb: true}},
]);

// Rolling back a delete will involve reinserting the item(s).
deleteMany(rollbackNodeDB, kColl, tenantA, {q: 1});
updateDocs(rollbackNodeDB, kColl, tenantA, [
{q: {q: 0}, u: {$inc: {y: 1}}},
]);
insertDocs(rollbackNodeDB, 'kap', tenantA, [{foo: 2}]);
insertDocs(rollbackNodeDB, 'kap2', tenantA, [{foo: 2}]);

// Create a collection (need to roll back the whole thing).
insertDocs(rollbackNodeDB, 'newcoll', tenantA, [{a: true}]);
// Create a new empty collection (need to roll back the whole thing).
assert.commandWorked(rollbackNodeDB.createCollection("abc", {'$tenant': tenantA}));

rollbackTest.transitionToSyncSourceOperationsBeforeRollback();

// Insert new data into syncSource so that rollbackNode enters rollback when it is reconnected.
// These operations should not be rolled back.
insertDocs(syncSourceDB, kColl, tenantA, [{txt: 'foo'}]);
deleteMany(syncSourceDB, kColl, tenantA, {q: 70});
updateDocs(syncSourceDB, kColl, tenantA, [
{q: {q: 0}, u: {$inc: {y: 33}}},
]);
deleteMany(syncSourceDB, kColl, tenantB, {q: 40});

rollbackTest.transitionToSyncSourceOperationsDuringRollback();
rollbackTest.transitionToSteadyStateOperations();

rollbackTest.awaitReplication();
checkFinalResults(rollbackNodeDB);
checkFinalResults(syncSourceDB);

rollbackTest.stop();
}());
1 change: 1 addition & 0 deletions src/mongo/db/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -2510,6 +2510,7 @@ env.Library(
's/sharding_commands_d',
's/sharding_runtime_d',
'serverinit',
'serverless/multitenancy_check',
'serverless/shard_split_donor_service',
'service_context_d',
'service_liaison_mongod',
Expand Down
2 changes: 2 additions & 0 deletions src/mongo/db/mongod_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@
#include "mongo/db/s/sharding_state.h"
#include "mongo/db/s/transaction_coordinator_service.h"
#include "mongo/db/server_options.h"
#include "mongo/db/serverless/multitenancy_check.h"
#include "mongo/db/serverless/shard_split_donor_op_observer.h"
#include "mongo/db/serverless/shard_split_donor_service.h"
#include "mongo/db/service_context.h"
Expand Down Expand Up @@ -1806,6 +1807,7 @@ int mongod_main(int argc, char* argv[]) {
setUpCatalog(service);
setUpReplication(service);
setUpObservers(service);
setUpMultitenancyCheck(service, gMultitenancySupport);
service->setServiceEntryPoint(std::make_unique<ServiceEntryPointMongod>(service));

ErrorExtraInfo::invariantHaveAllParsers();
Expand Down
3 changes: 1 addition & 2 deletions src/mongo/db/repl/rollback_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -815,8 +815,7 @@ void RollbackImpl::_correctRecordStoreCounts(OperationContext* opCtx) {
newCount = countFromScan;
}

auto status =
_storageInterface->setCollectionCount(opCtx, {nss.db().toString(), uuid}, newCount);
auto status = _storageInterface->setCollectionCount(opCtx, {nss.dbName(), uuid}, newCount);
if (!status.isOK()) {
// We ignore errors here because crashing or leaving rollback would only leave
// collection counts more inaccurate.
Expand Down
9 changes: 9 additions & 0 deletions src/mongo/db/serverless/SConscript
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
Import("env")
env = env.Clone()

env.Library(target='multitenancy_check', source=[
'multitenancy_check.cpp',
], LIBDEPS=[
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/db/service_context',
])

env.Library(
target='serverless_types_idl',
source=[
Expand Down Expand Up @@ -104,6 +111,7 @@ env.Library(
env.CppUnitTest(
target='db_serverless_test',
source=[
'multitenancy_check_test.cpp',
'serverless_operation_lock_registry_test.cpp',
'shard_split_donor_op_observer_test.cpp',
'shard_split_donor_service_test.cpp',
Expand All @@ -118,6 +126,7 @@ env.CppUnitTest(
'$BUILD_DIR/mongo/db/repl/replmocks',
'$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker',
'$BUILD_DIR/mongo/dbtests/mocklib',
'multitenancy_check',
'serverless_lock',
'shard_split_donor_service',
'shard_split_utils',
Expand Down
64 changes: 64 additions & 0 deletions src/mongo/db/serverless/multitenancy_check.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/**
* Copyright (C) 2023-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the Server Side Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/

#include "mongo/db/serverless/multitenancy_check.h"

#include "mongo/db/client.h"
#include "mongo/db/service_context.h"

namespace mongo {

const ServiceContext::Decoration<std::unique_ptr<MultitenancyCheck>> MultitenancyCheck::get =
ServiceContext::declareDecoration<std::unique_ptr<MultitenancyCheck>>();

MultitenancyCheck::MultitenancyCheck(bool multitenancySupport)
: _multitenancySupport(multitenancySupport) {}

void MultitenancyCheck::checkDollarTenantField(const BSONObj& body) const {
uassert(ErrorCodes::InvalidOptions,
"Multitenancy not enabled, cannot set $tenant in command body",
_multitenancySupport || !body["$tenant"_sd]);
}

const MultitenancyCheck* MultitenancyCheck::getPtr() {
if (!hasGlobalServiceContext()) {
// globalServiceContext is not always set for unit tests
return nullptr;
}

return MultitenancyCheck::get(getGlobalServiceContext()).get();
}

void setUpMultitenancyCheck(ServiceContext* serviceContext, bool multitenancySupport) {
auto& multitenancyCheck = MultitenancyCheck::get(serviceContext);

multitenancyCheck = std::make_unique<MultitenancyCheck>(multitenancySupport);
}

} // namespace mongo
Loading

0 comments on commit 1da8ac4

Please sign in to comment.