SERVER-43274 Implement cloners using DBClient with basic functionalit…

…y and unit tests.
mongodb · Oct 30, 2019 · a5af8f0 · a5af8f0
1 parent d3bb4b1
commit a5af8f0
Show file tree

Hide file tree

Showing 34 changed files with 2,650 additions and 6,636 deletions.
diff --git a/jstests/replsets/initial_sync_drop_collection.js b/jstests/replsets/initial_sync_drop_collection.js
@@ -3,6 +3,11 @@
 (function() {
 "use strict";
 
+// TODO(SERVER-43277): This test is disabled while we work on the Resumable Initial Sync
+// project
+/* eslint-disable no-unreachable */
+return;
+
 // Skip db hash check because secondary cannot complete initial sync.
 TestData.skipCheckDBHashes = true;
 

diff --git a/jstests/replsets/initial_sync_fcv.js b/jstests/replsets/initial_sync_fcv.js
@@ -30,11 +30,14 @@ function runInitialSync(cmd, initialFCV) {
 
     jsTestLog('Testing setting fCV with ' + tojson(cmd));
 
-    const failPointOptions = tojson({mode: 'alwaysOn', data: {database: dbName}});
+    const failPointOptions = tojson({
+        mode: 'alwaysOn',
+        data: {cloner: "DatabaseCloner", stage: "listCollections", database: dbName}
+    });
     rst.restart(1, {
         startClean: true,
         setParameter: {
-            'failpoint.initialSyncHangBeforeListCollections': failPointOptions,
+            'failpoint.hangBeforeClonerStage': failPointOptions,
             'failpoint.skipClearInitialSyncState': tojson({mode: 'alwaysOn'}),
             numInitialSyncAttempts: 2
         }
@@ -44,17 +47,17 @@ function runInitialSync(cmd, initialFCV) {
     // Initial sync clones the 'admin' database first, which will set the fCV on the
     // secondary to initialFCV. We then block the secondary before issuing 'listCollections' on
     // the test database.
-    assert.commandWorked(secondary.adminCommand(
-        {waitForFailPoint: "initialSyncHangBeforeListCollections", timesEntered: 1}));
+    assert.commandWorked(
+        secondary.adminCommand({waitForFailPoint: "hangBeforeClonerStage", timesEntered: 1}));
 
     // Initial sync is stopped right before 'listCollections' on the test database. We now run
     // the test command to modify the fCV.
     assert.commandWorked(primary.adminCommand(cmd));
 
     // Let initial sync finish, making sure that it fails due to the feature compatibility
     // version change.
-    assert.commandWorked(secondary.adminCommand(
-        {configureFailPoint: 'initialSyncHangBeforeListCollections', mode: 'off'}));
+    assert.commandWorked(
+        secondary.adminCommand({configureFailPoint: 'hangBeforeClonerStage', mode: 'off'}));
     checkLog.contains(secondary, 'Applying operation on feature compatibility version document');
 
     jsTestLog('Wait for both nodes to be up-to-date');

diff --git a/jstests/replsets/initial_sync_uuid_not_found.js b/jstests/replsets/initial_sync_uuid_not_found.js
@@ -64,7 +64,11 @@ function ResyncWithFailpoint(failpointName, failpointData) {
     jsTestLog('Check consistency and shut down replica-set');
     rst.checkReplicatedDataHashes();
 }
-ResyncWithFailpoint('initialSyncHangBeforeCollectionClone', {namespace: primaryColl.getFullName()});
-ResyncWithFailpoint('initialSyncHangAfterListCollections', {database: primaryDB.getName()});
+ResyncWithFailpoint(
+    'hangBeforeClonerStage',
+    {cloner: 'CollectionCloner', stage: 'count', namespace: primaryColl.getFullName()});
+ResyncWithFailpoint(
+    'hangAfterClonerStage',
+    {cloner: 'DatabaseCloner', stage: 'listCollections', database: primaryDB.getName()});
 rst.stopSet();
 })();
diff --git a/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js b/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js
@@ -19,8 +19,9 @@ const nss = adminDbName + "." + versionCollName;
 
 // Hang initial sync before cloning the FCV document.
 let secondary = rst.add({rsConfig: {priority: 0}});
-let failPoint =
-    configureFailPoint(secondary, 'initialSyncHangBeforeCollectionClone', {namespace: nss});
+let failPoint = configureFailPoint(secondary,
+                                   'hangBeforeClonerStage',
+                                   {cloner: 'CollectionCloner', stage: 'count', namespace: nss});
 rst.reInitiate();
 failPoint.wait();
 

diff --git a/src/mongo/client/dbclient_base.cpp b/src/mongo/client/dbclient_base.cpp
@@ -324,8 +324,11 @@ long long DBClientBase::count(
     auto dbName = (nsOrUuid.uuid() ? nsOrUuid.dbname() : (*nsOrUuid.nss()).db().toString());
     BSONObj cmd = _countCmd(nsOrUuid, query, options, limit, skip);
     BSONObj res;
-    if (!runCommand(dbName, cmd, res, options))
-        uasserted(11010, string("count fails:") + res.toString());
+    if (!runCommand(dbName, cmd, res, options)) {
+        auto status = getStatusFromCommandResult(res);
+        uassertStatusOK(status.withContext("count fails:"));
+    }
+    uassert(ErrorCodes::NoSuchKey, "Missing 'n' field for count command.", res.hasField("n"));
     return res["n"].numberLong();
 }
 
@@ -865,12 +868,15 @@ list<BSONObj> DBClientBase::getIndexSpecs(const NamespaceStringOrUUID& nsOrUuid,
 
         return specs;
     }
-    int code = res["code"].numberInt();
+    Status status = getStatusFromCommandResult(res);
 
-    if (code == ErrorCodes::NamespaceNotFound) {
+    // "NamespaceNotFound" is an error for UUID but returns an empty list for NamespaceString; this
+    // matches the behavior for other commands such as 'find' and 'count'.
+    if (nsOrUuid.nss() && status.code() == ErrorCodes::NamespaceNotFound) {
         return specs;
     }
-    uasserted(18631, str::stream() << "listIndexes failed: " << res);
+    uassertStatusOK(status.withContext(str::stream() << "listIndexes failed: " << res));
+    MONGO_UNREACHABLE;
 }
 
 

diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
@@ -655,7 +655,6 @@ env.Library(
         '$BUILD_DIR/mongo/transport/transport_layer_common',
         '$BUILD_DIR/mongo/util/fail_point',
         '$BUILD_DIR/mongo/db/repl/replication_metrics',
-        'collection_cloner',
         'initial_syncer',
         'data_replicator_external_state_initial_sync',
         'repl_coordinator_interface',
@@ -924,70 +923,29 @@ env.Library(
 )
 
 env.Library(
-    target='base_cloner_test_fixture',
-    source=[
-        'base_cloner_test_fixture.cpp',
-    ],
-    LIBDEPS=[
-        'replmocks',
-        '$BUILD_DIR/mongo/db/service_context_test_fixture',
-        '$BUILD_DIR/mongo/executor/thread_pool_task_executor_test_fixture',
-        '$BUILD_DIR/mongo/util/concurrency/thread_pool',
-    ],
-)
-
-env.Library(
-    target='collection_cloner',
+    target='initial_sync_cloners',
     source=[
+        'all_database_cloner.cpp',
+        'base_cloner.cpp',
         'collection_cloner.cpp',
+        'database_cloner.cpp',
+        env.Idlc('database_cloner.idl')[0]
     ],
-    LIBDEPS=[
+    LIBDEPS = [
         'task_runner',
         '$BUILD_DIR/mongo/base',
-        '$BUILD_DIR/mongo/client/fetcher',
-        '$BUILD_DIR/mongo/client/remote_command_retry_scheduler',
-        '$BUILD_DIR/mongo/db/catalog/collection_options',
-        '$BUILD_DIR/mongo/db/catalog/document_validation',
-        '$BUILD_DIR/mongo/executor/task_executor_interface',
-        '$BUILD_DIR/mongo/rpc/command_status',
-        '$BUILD_DIR/mongo/s/query/async_results_merger',
-        '$BUILD_DIR/mongo/util/progress_meter',
-    ],
+        '$BUILD_DIR/mongo/client/clientdriver_network',
+        '$BUILD_DIR/mongo/util/concurrency/thread_pool',
+        '$BUILD_DIR/mongo/util/net/network',
+    ],        
     LIBDEPS_PRIVATE=[
         'repl_server_parameters',
         'replication_auth',
-    ],
-)
-
-env.Library(
-    target='database_cloner',
-    source=[
-        'database_cloner.cpp',
-    ],
-    LIBDEPS=[
-        'collection_cloner',
-        '$BUILD_DIR/mongo/executor/task_executor_interface',
-    ],
-    LIBDEPS_PRIVATE=[
-        'repl_server_parameters',
+        '$BUILD_DIR/mongo/idl/idl_parser',
+        '$BUILD_DIR/mongo/db/catalog/collection_options',
         '$BUILD_DIR/mongo/db/commands/list_collections_filter',
-    ],
-)
-
-env.Library(
-    target='databases_cloner',
-    source=[
-        'databases_cloner.cpp',
-        env.Idlc('databases_cloner.idl')[0],
-    ],
-    LIBDEPS=[
-        'database_cloner',
-        '$BUILD_DIR/mongo/db/service_context',
-        '$BUILD_DIR/mongo/dbtests/mocklib',
-    ],
-    LIBDEPS_PRIVATE=[
-        '$BUILD_DIR/mongo/idl/server_parameter',
-    ],
+        '$BUILD_DIR/mongo/util/progress_meter',
+    ]
 )
 
 env.Library(
@@ -1079,9 +1037,8 @@ env.Library(
         '$BUILD_DIR/mongo/client/fetcher',
         '$BUILD_DIR/mongo/db/transaction',
         '$BUILD_DIR/mongo/db/commands/server_status_core',
-        'collection_cloner',
-        'database_cloner',
-        'databases_cloner',
+        '$BUILD_DIR/mongo/client/clientdriver_network',
+        'initial_sync_cloners',
         'multiapplier',
         'oplog',
         'oplog_application_interface',
@@ -1284,6 +1241,7 @@ env.CppUnitTest(
         '$BUILD_DIR/mongo/db/service_context_test_fixture',
         '$BUILD_DIR/mongo/db/stats/counters',
         '$BUILD_DIR/mongo/db/transaction',
+        '$BUILD_DIR/mongo/dbtests/mocklib',
         '$BUILD_DIR/mongo/executor/network_interface_factory',
         '$BUILD_DIR/mongo/executor/network_interface_mock',
         '$BUILD_DIR/mongo/executor/network_interface_thread_pool',
@@ -1359,21 +1317,19 @@ env.CppUnitTest(
 env.CppUnitTest(
     target='db_repl_cloners_test',
     source=[
-        'collection_cloner_test.cpp',
+        'all_database_cloner_test.cpp',
+        'cloner_test_fixture.cpp',
         'database_cloner_test.cpp',
-        'databases_cloner_test.cpp',
-    ],
+        'collection_cloner_test.cpp'
+        ],
     LIBDEPS=[
-        '$BUILD_DIR/mongo/db/auth/authmocks',
-        '$BUILD_DIR/mongo/db/auth/authorization_manager_global',
-        '$BUILD_DIR/mongo/db/commands/list_collections_filter',
+        '$BUILD_DIR/mongo/db/auth/authmocks',  # Required for service context test fixture
+        '$BUILD_DIR/mongo/db/catalog/collection_options',
+        '$BUILD_DIR/mongo/db/service_context_d_test_fixture',
         '$BUILD_DIR/mongo/dbtests/mocklib',
-        '$BUILD_DIR/mongo/unittest/task_executor_proxy',
-        'base_cloner_test_fixture',
-        'collection_cloner',
-        'database_cloner',
-        'databases_cloner',
-    ],
+        'replmocks',
+        'initial_sync_cloners'
+        ],
 )
 
 env.Library(