feat(NODE-3191): backport versioned api

.evergreen/config.yml

@@ -1,6 +1,6 @@
 stepback: true
 command_type: system
-exec_timeout_secs: 1200
+exec_timeout_secs: 1500
 timeout:
   - command: shell.exec
     params:
@@ -79,6 +79,7 @@ functions:
           MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \
           AUTH=${AUTH} SSL=${SSL} \
           ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
+          REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
           bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     - command: expansions.update
       params:
@@ -111,8 +112,10 @@ functions:
             rm -f ./prepare_client_encryption.sh
           fi
 
-          AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" \
-          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=1 NO_EXIT=1 \
+          MONGODB_URI="${MONGODB_URI}" \
+          AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} \
+          MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
+          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
   run checks:
     - command: shell.exec
@@ -496,39 +499,6 @@ post:
 ignore:
   - '*.md'
 tasks:
-  - name: test-latest-server
-    tags:
-      - latest
-      - server
-    commands:
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-        vars:
-          VERSION: latest
-          TOPOLOGY: server
-      - func: run tests
-  - name: test-latest-replica_set
-    tags:
-      - latest
-      - replica_set
-    commands:
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-        vars:
-          VERSION: latest
-          TOPOLOGY: replica_set
-      - func: run tests
-  - name: test-latest-sharded_cluster
-    tags:
-      - latest
-      - sharded_cluster
-    commands:
-      - func: install dependencies
-      - func: bootstrap mongo-orchestration
-        vars:
-          VERSION: latest
-          TOPOLOGY: sharded_cluster
-      - func: run tests
   - name: test-latest-server-unified
     tags:
       - latest
@@ -1144,6 +1114,23 @@ tasks:
       - func: run tests
         vars:
           UNIFIED: 1
+  - name: test-latest-server-v1-api
+    tags:
+      - latest
+      - server
+      - v1-api
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: server
+          REQUIRE_API_VERSION: '1'
+      - func: run tests
+        vars:
+          MONGODB_API_VERSION: '1'
+          UNIFIED: 1
+          NO_EXIT: ''
   - name: test-atlas-connectivity
     tags:
       - atlas-connect
@@ -1562,9 +1549,6 @@ buildvariants:
     expansions:
       NODE_LTS_NAME: fermium
     tasks: &ref_0
-      - test-latest-server
-      - test-latest-replica_set
-      - test-latest-sharded_cluster
       - test-latest-server-unified
       - test-latest-replica_set-unified
       - test-latest-sharded_cluster-unified
@@ -1616,6 +1600,7 @@ buildvariants:
       - test-2.6-server-unified
       - test-2.6-replica_set-unified
       - test-2.6-sharded_cluster-unified
+      - test-latest-server-v1-api
       - test-atlas-connectivity
       - test-auth-kerberos-legacy
       - test-auth-kerberos-unified
@@ -1672,9 +1657,6 @@ buildvariants:
     expansions:
       NODE_LTS_NAME: fermium
     tasks: &ref_1
-      - test-latest-server
-      - test-latest-replica_set
-      - test-latest-sharded_cluster
       - test-latest-server-unified
       - test-latest-replica_set-unified
       - test-latest-sharded_cluster-unified
@@ -1726,6 +1708,7 @@ buildvariants:
       - test-2.6-server-unified
       - test-2.6-replica_set-unified
       - test-2.6-sharded_cluster-unified
+      - test-latest-server-v1-api
       - test-atlas-connectivity
       - test-auth-kerberos-legacy
       - test-auth-kerberos-unified
@@ -1859,9 +1842,6 @@ buildvariants:
       NODE_LTS_NAME: fermium
       CLIENT_ENCRYPTION: true
     tasks: &ref_3
-      - test-latest-server
-      - test-latest-replica_set
-      - test-latest-sharded_cluster
       - test-latest-server-unified
       - test-latest-replica_set-unified
       - test-latest-sharded_cluster-unified
@@ -1901,6 +1881,7 @@ buildvariants:
       - test-3.2-server-unified
       - test-3.2-replica_set-unified
       - test-3.2-sharded_cluster-unified
+      - test-latest-server-v1-api
       - test-atlas-connectivity
       - test-auth-kerberos-legacy
       - test-auth-kerberos-unified

.evergreen/config.yml.in

@@ -11,7 +11,7 @@ command_type: system
 # Protect ourself against rogue test case, or curl gone wild, that runs forever
 # Good rule of thumb: the averageish length a task takes, times 5
 # That roughly accounts for variable system performance for various buildvariants
-exec_timeout_secs: 1200
+exec_timeout_secs: 1500
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -96,6 +96,7 @@ functions:
           MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \
           AUTH=${AUTH} SSL=${SSL} \
           ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
+          REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
           bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
@@ -130,8 +131,10 @@ functions:
             rm -f ./prepare_client_encryption.sh
           fi
 
-          AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" \
-          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=1 NO_EXIT=1 \
+          MONGODB_URI="${MONGODB_URI}" \
+          AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} \
+          MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
+          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run checks":

.evergreen/generate_evergreen_tasks.js

@@ -5,16 +5,15 @@ const fs = require('fs');
 const yaml = require('js-yaml');
 
 const LATEST_EFFECTIVE_VERSION = '5.0';
-const MONGODB_VERSIONS = ['latest', '4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6'];
+const LEGACY_MONGODB_VERSIONS = new Set(['4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6']);
+const MONGODB_VERSIONS = ['latest'].concat(Array.from(LEGACY_MONGODB_VERSIONS));
 const AWS_AUTH_VERSIONS = ['latest', '4.4'];
 const OCSP_VERSIONS = ['latest', '4.4'];
 const TLS_VERSIONS = ['latest', '4.2']; // also test on 4.2 because 4.4+ currently skipped on windows
 const NODE_VERSIONS = ['fermium', 'erbium', 'dubnium', 'carbon', 'boron', 'argon'];
-const TOPOLOGIES = ['server', 'replica_set', 'sharded_cluster'].concat([
-  'server-unified',
-  'replica_set-unified',
-  'sharded_cluster-unified'
-]);
+const LEGACY_TOPOLOGIES = new Set(['server', 'replica_set', 'sharded_cluster']);
+const UNIFIED_TOPOLOGIES = Array.from(LEGACY_TOPOLOGIES).map(topology => `${topology}-unified`);
+
 
 const OPERATING_SYSTEMS = [
   {
@@ -74,6 +73,7 @@ const OPERATING_SYSTEMS = [
   )
 );
 
+// TODO: NODE-3060: enable skipped tests on windows
 const WINDOWS_SKIP_TAGS = new Set([
   'atlas-connect',
   'auth'
@@ -109,9 +109,38 @@ function makeTask({ mongoVersion, topology }) {
 }
 
 MONGODB_VERSIONS.forEach(mongoVersion => {
-  TOPOLOGIES.forEach(topology =>
+  Array.from(LEGACY_TOPOLOGIES).concat(UNIFIED_TOPOLOGIES).forEach(topology => {
+    if (LEGACY_TOPOLOGIES.has(topology) && !LEGACY_MONGODB_VERSIONS.has(mongoVersion)) {
+      // MongoDB 5.0+ is only supported by the Unified Topology in driver 3.7+
+      // therefore testing the legacy toplogy can be skipped
+      return;
+    }
     BASE_TASKS.push(makeTask({ mongoVersion, topology }))
-  );
+  });
+});
+
+BASE_TASKS.push({
+  name: `test-latest-server-v1-api`,
+  tags: ['latest', 'server', 'v1-api'],
+  commands: [
+    { func: 'install dependencies' },
+    {
+      func: 'bootstrap mongo-orchestration',
+      vars: {
+        VERSION: 'latest',
+        TOPOLOGY: 'server',
+        REQUIRE_API_VERSION: '1'
+      }
+    },
+    {
+      func: 'run tests',
+      vars: {
+        MONGODB_API_VERSION: '1',
+        UNIFIED: 1,
+        NO_EXIT: ''
+      }
+    }
+  ]
 });
 
 TASKS.push(

.evergreen/run-tests.sh

@@ -59,4 +59,4 @@ nvm use 12
 npm run build:unified
 nvm use "$NODE_VERSION"
 
-MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} npm run ${TEST_NPM_SCRIPT}
+MONGODB_API_VERSION=${MONGODB_API_VERSION} MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} npm run ${TEST_NPM_SCRIPT}

lib/cmap/connection.js

@@ -37,6 +37,8 @@ class Connection extends EventEmitter {
     this.port = options.port || 27017;
     this.monitorCommands =
       typeof options.monitorCommands === 'boolean' ? options.monitorCommands : false;
+    this.serverApi = options.serverApi;
+
     this.closed = false;
     this.destroyed = false;
 
@@ -170,33 +172,58 @@ class Connection extends EventEmitter {
     });
   }
 
+  applyApiVersion(options) {
+    if (this.serverApi) {
+      options.serverApi = this.serverApi;
+    }
+    return options;
+  }
+
   // Wire protocol methods
   command(ns, cmd, options, callback) {
-    wp.command(makeServerTrampoline(this), ns, cmd, options, callback);
+    if (typeof options === 'function') {
+      callback = options;
+      options = {};
+    }
+    wp.command(makeServerTrampoline(this), ns, cmd, this.applyApiVersion(options), callback);
   }
 
   query(ns, cmd, cursorState, options, callback) {
-    wp.query(makeServerTrampoline(this), ns, cmd, cursorState, options, callback);
+    wp.query(
+      makeServerTrampoline(this),
+      ns,
+      cmd,
+      cursorState,
+      this.applyApiVersion(options),
+      callback
+    );
   }
 
   getMore(ns, cursorState, batchSize, options, callback) {
-    wp.getMore(makeServerTrampoline(this), ns, cursorState, batchSize, options, callback);
+    wp.getMore(
+      makeServerTrampoline(this),
+      ns,
+      cursorState,
+      batchSize,
+      this.applyApiVersion(options),
+      callback
+    );
   }
 
   killCursors(ns, cursorState, callback) {
-    wp.killCursors(makeServerTrampoline(this), ns, cursorState, callback);
+    wp.killCursors(makeServerTrampoline(this), ns, cursorState, this.applyApiVersion({}), callback);
   }
 
   insert(ns, ops, options, callback) {
-    wp.insert(makeServerTrampoline(this), ns, ops, options, callback);
+    wp.insert(makeServerTrampoline(this), ns, ops, this.applyApiVersion(options), callback);
   }
 
   update(ns, ops, options, callback) {
-    wp.update(makeServerTrampoline(this), ns, ops, options, callback);
+    wp.update(makeServerTrampoline(this), ns, ops, this.applyApiVersion(options), callback);
   }
 
   remove(ns, ops, options, callback) {
-    wp.remove(makeServerTrampoline(this), ns, ops, options, callback);
+    wp.remove(makeServerTrampoline(this), ns, ops, this.applyApiVersion(options), callback);
   }
 }
 

lib/cmap/connection_pool.js

@@ -41,6 +41,7 @@ const VALID_POOL_OPTIONS = new Set([
   'ssl',
   'bson',
   'connectionType',
+  'serverApi',
   'monitorCommands',
   'socketTimeout',
   'credentials',

lib/collection.js

@@ -1524,7 +1524,7 @@ Collection.prototype.count = deprecate(function(query, options, callback) {
 
   return executeOperation(
     this.s.topology,
-    new EstimatedDocumentCountOperation(this, query, options),
+    new CountDocumentsOperation(this, query, options),
     callback
   );
 }, 'collection.count is deprecated, and will be removed in a future version.' +

lib/core/connection/connect.js

@@ -108,6 +108,11 @@ function performInitialHandshake(conn, options, _callback) {
         return;
       }
 
+      if ('isWritablePrimary' in response) {
+        // Provide pre-hello-style response document.
+        response.ismaster = response.isWritablePrimary;
+      }
+
       const supportedServerErr = checkSupportedServer(response, options);
       if (supportedServerErr) {
         callback(supportedServerErr);
@@ -158,11 +163,12 @@ function performInitialHandshake(conn, options, _callback) {
 
 function prepareHandshakeDocument(authContext, callback) {
   const options = authContext.options;
+  const serverApi = authContext.connection.serverApi;
   const compressors =
     options.compression && options.compression.compressors ? options.compression.compressors : [];
 
   const handshakeDoc = {
-    ismaster: true,
+    [serverApi ? 'hello' : 'ismaster']: true,
     client: options.metadata || makeClientMetadata(options),
     compression: compressors
   };

lib/core/sdam/monitor.js

@@ -203,8 +203,9 @@ function checkServer(monitor, callback) {
     const maxAwaitTimeMS = monitor.options.heartbeatFrequencyMS;
     const topologyVersion = monitor[kServer].description.topologyVersion;
     const isAwaitable = topologyVersion != null;
+    const serverApi = monitor[kConnection].serverApi;
 
-    const cmd = { ismaster: true };
+    const cmd = { [serverApi ? 'hello' : 'ismaster']: true };
     const options = { socketTimeout: connectTimeoutMS };
 
     if (isAwaitable) {
@@ -228,6 +229,11 @@ function checkServer(monitor, callback) {
       const isMaster = result.result;
       const rttPinger = monitor[kRTTPinger];
 
+      if ('isWritablePrimary' in isMaster) {
+        // Provide pre-hello-style response document.
+        isMaster.ismaster = isMaster.isWritablePrimary;
+      }
+
       const duration =
         isAwaitable && rttPinger ? rttPinger.roundTripTime : calculateDurationInMs(start);