feat(data-service): use split CRUD client/Metadata client model COMPASS-5641

.gitignore

@@ -26,3 +26,4 @@ storage
 coverage
 .ackrc
 env-vars.sh
+mongocryptd.pid

packages/compass-crud/src/stores/crud-store.js

@@ -611,7 +611,7 @@ const configureStore = (options = {}) => {
         }
       }
 
-      const session = this.dataService.startSession();
+      const session = this.dataService.startSession('CRUD');
       const abortController = new AbortController();
       const signal = abortController.signal;
 
@@ -1029,15 +1029,15 @@ const configureStore = (options = {}) => {
 
       const fetchShardingKeysOptions = {
         maxTimeMS: query.maxTimeMS,
-        session: this.dataService.startSession()
+        session: this.dataService.startSession('CRUD')
       };
 
       const countOptions = {
         skip: query.skip,
         maxTimeMS: query.maxTimeMS > COUNT_MAX_TIME_MS_CAP ?
           COUNT_MAX_TIME_MS_CAP :
           query.maxTimeMS,
-        session: this.dataService.startSession()
+        session: this.dataService.startSession('CRUD')
       };
 
       if (this.isCountHintSafe()) {
@@ -1053,7 +1053,7 @@ const configureStore = (options = {}) => {
         maxTimeMS: query.maxTimeMS,
         promoteValues: false,
         bsonRegExp: true,
-        session: this.dataService.startSession()
+        session: this.dataService.startSession('CRUD')
       };
 
       // only set limit if it's > 0, read-only views cannot handle 0 limit.

packages/compass-crud/src/utils/cancellable-queries.spec.js

@@ -76,7 +76,7 @@ describe('cancellable-queries', function() {
   beforeEach(function() {
     sinon.restore();
 
-    session = dataService.startSession();
+    session = dataService.startSession('CRUD');
     abortController = new AbortController();
     signal = abortController.signal;
   });

packages/compass-import-export/src/utils/collection-stream.js

@@ -42,7 +42,7 @@ class WritableCollectionStream extends Writable {
   }
 
   _collection() {
-    return this.dataService._collection(this.ns);
+    return this.dataService._collection(this.ns, 'CRUD');
   }
 
   _write(document, _encoding, next) {

packages/data-service/src/connect-mongo-client.spec.ts

@@ -10,16 +10,18 @@ const setupListeners = () => {
 };
 
 describe('connectMongoClient', function () {
-  let toBeClosed: { close: () => Promise<void> }[] = [];
+  const toBeClosed = new Set<{ close: () => Promise<void> }>();
+
+  beforeEach(function () {
+    toBeClosed.clear();
+  });
 
   afterEach(async function () {
     for (const mongoClientOrTunnel of toBeClosed) {
       if (mongoClientOrTunnel) {
         await mongoClientOrTunnel.close();
       }
     }
-
-    toBeClosed = [];
   });
 
   describe('local', function () {
@@ -34,32 +36,77 @@ describe('connectMongoClient', function () {
     });
 
     it('should return connection config when connected successfully', async function () {
-      const [client, tunnel, { url, options }] = await connectMongoClient(
-        {
-          connectionString: 'mongodb://localhost:27018',
+      const [metadataClient, crudClient, tunnel, { url, options }] =
+        await connectMongoClient(
+          {
+            connectionString: 'mongodb://localhost:27018',
+          },
+          setupListeners
+        );
+
+      for (const closeLater of [metadataClient, crudClient, tunnel]) {
+        toBeClosed.add(closeLater);
+      }
+
+      expect(metadataClient).to.equal(crudClient);
+      expect(url).to.equal('mongodb://localhost:27018');
+
+      expect(options).to.deep.equal({
+        monitorCommands: true,
+        useSystemCA: undefined,
+        autoEncryption: undefined,
+      });
+    });
+
+    it('should return two different clients when AutoEncryption is enabled', async function () {
+      const autoEncryption = {
+        keyVaultNamespace: 'encryption.__keyVault',
+        kmsProviders: {
+          local: { key: Buffer.alloc(96) },
         },
-        setupListeners
-      );
+        bypassAutoEncryption: true,
+      };
+      const [metadataClient, crudClient, tunnel, { url, options }] =
+        await connectMongoClient(
+          {
+            connectionString: 'mongodb://localhost:27018',
+            fleOptions: {
+              storeCredentials: false,
+              autoEncryption,
+            },
+          },
+          setupListeners
+        );
 
-      toBeClosed.push(client, tunnel);
+      for (const closeLater of [metadataClient, crudClient, tunnel]) {
+        toBeClosed.add(closeLater);
+      }
 
-      assert.strictEqual(url, 'mongodb://localhost:27018');
+      expect(metadataClient).to.not.equal(crudClient);
+      expect(metadataClient.options.autoEncryption).to.equal(undefined);
+      expect(crudClient.options.autoEncryption).to.be.an('object');
+      expect(url).to.equal('mongodb://localhost:27018');
 
-      assert.deepStrictEqual(options, {
+      expect(options).to.deep.equal({
         monitorCommands: true,
         useSystemCA: undefined,
+        autoEncryption,
       });
     });
 
     it('should not override a user-specified directConnection option', async function () {
-      const [client, tunnel, { url, options }] = await connectMongoClient(
-        {
-          connectionString: 'mongodb://localhost:27018/?directConnection=false',
-        },
-        setupListeners
-      );
+      const [metadataClient, crudClient, tunnel, { url, options }] =
+        await connectMongoClient(
+          {
+            connectionString:
+              'mongodb://localhost:27018/?directConnection=false',
+          },
+          setupListeners
+        );
 
-      toBeClosed.push(client, tunnel);
+      for (const closeLater of [metadataClient, crudClient, tunnel]) {
+        toBeClosed.add(closeLater);
+      }
 
       assert.strictEqual(
         url,
@@ -69,6 +116,7 @@ describe('connectMongoClient', function () {
       assert.deepStrictEqual(options, {
         monitorCommands: true,
         useSystemCA: undefined,
+        autoEncryption: undefined,
       });
     });
 

packages/data-service/src/connect-mongo-client.ts

@@ -5,6 +5,7 @@ import type { DevtoolsConnectOptions } from '@mongodb-js/devtools-connect';
 import type SSHTunnel from '@mongodb-js/ssh-tunnel';
 import EventEmitter from 'events';
 import { redactConnectionOptions, redactConnectionString } from './redact';
+import _ from 'lodash';
 
 import createLoggerAndTelemetry from '@mongodb-js/compass-logging';
 import type { ConnectionOptions } from './connection-options';
@@ -21,9 +22,10 @@ export default async function connectMongoClientCompass(
   setupListeners: (client: MongoClient) => void
 ): Promise<
   [
-    MongoClient,
-    SSHTunnel | undefined,
-    { url: string; options: MongoClientOptions }
+    metadataClient: MongoClient,
+    crudClient: MongoClient,
+    sshTunnel: SSHTunnel | undefined,
+    options: { url: string; options: DevtoolsConnectOptions }
   ]
 > {
   debug(
@@ -35,6 +37,7 @@ export default async function connectMongoClientCompass(
   const options: DevtoolsConnectOptions = {
     monitorCommands: true,
     useSystemCA: connectionOptions.useSystemCA,
+    autoEncryption: connectionOptions.fleOptions?.autoEncryption,
   };
 
   // If connectionOptions.sshTunnel is defined, open an ssh tunnel.
@@ -61,32 +64,54 @@ export default async function connectMongoClientCompass(
   const connectLogger = new EventEmitter();
   hookLogger(connectLogger, log.unbound, 'compass', redactConnectionString);
 
-  let mongoClient: MongoClient | undefined;
+  async function connectSingleClient(
+    overrideOptions: DevtoolsConnectOptions
+  ): Promise<MongoClient> {
+    const client = await connectMongoClient(
+      url,
+      // Deep clone because of https://jira.mongodb.org/browse/NODE-4124,
+      // the options here are being mutated.
+      _.cloneDeep({ ...options, ...overrideOptions }),
+      connectLogger,
+      CompassMongoClient
+    );
+    await client.db('admin').command({ ping: 1 });
+    return client;
+  }
+
+  let metadataClient: MongoClient | undefined;
+  let crudClient: MongoClient | undefined;
   try {
     debug('waiting for MongoClient to connect ...');
-    mongoClient = await Promise.race([
-      (async () => {
-        const mongoClient = await connectMongoClient(
-          url,
-          options,
-          connectLogger,
-          CompassMongoClient
-        );
-        await mongoClient.db('admin').command({ ping: 1 });
-        return mongoClient;
-      })(),
+    // Create one or two clients, depending on whether CSFLE
+    // is enabled. If it is, create one for interacting with
+    // server metadata (e.g. build info, instance data, etc.)
+    // and one for interacting with the actual CRUD data.
+    // If CSFLE is disabled, use a single client for both cases.
+    [metadataClient, crudClient] = await Promise.race([
+      Promise.all([
+        connectSingleClient({ autoEncryption: undefined }),
+        options.autoEncryption ? connectSingleClient({}) : undefined,
+      ]),
       waitForTunnelError(tunnel),
     ]); // waitForTunnel always throws, never resolves
 
-    return [mongoClient, tunnel, { url, options }];
+    return [
+      metadataClient,
+      crudClient ?? metadataClient,
+      tunnel,
+      { url, options },
+    ];
   } catch (err: any) {
     debug('connection error', err);
     debug('force shutting down ssh tunnel ...');
-    await Promise.all([forceCloseTunnel(tunnel), mongoClient?.close()]).catch(
-      () => {
-        /* ignore errors */
-      }
-    );
+    await Promise.all([
+      forceCloseTunnel(tunnel),
+      crudClient?.close(),
+      metadataClient?.close(),
+    ]).catch(() => {
+      /* ignore errors */
+    });
     throw err;
   }
 }

packages/data-service/src/data-service.spec.ts

@@ -1311,7 +1311,7 @@ describe('DataService', function () {
 
       describe('#startSession', function () {
         it('returns a new client session', function () {
-          const session = dataService.startSession();
+          const session = dataService.startSession('CRUD');
           expect(session.constructor.name).to.equal('ClientSession');
 
           // used by killSessions, must be a bson UUID in order to work
@@ -1322,22 +1322,22 @@ describe('DataService', function () {
 
       describe('#killSessions', function () {
         it('does not throw if kill a non existing session', async function () {
-          const session = dataService.startSession();
+          const session = dataService.startSession('CRUD');
           await dataService.killSessions(session);
         });
 
         it('kills a command with a session', async function () {
           const commandSpy = sinon.spy();
           sandbox.replace(
-            (dataService as any)._client,
+            (dataService as any)._crudClient,
             'db',
             () =>
               ({
                 command: commandSpy,
               } as any)
           );
 
-          const session = dataService.startSession();
+          const session = dataService.startSession('CRUD');
           await dataService.killSessions(session);
 
           expect(commandSpy.args[0][0]).to.deep.equal({
@@ -1353,7 +1353,9 @@ describe('DataService', function () {
       const dataService = new DataService({
         connectionString: 'mongodb://localhost:27020',
       });
-      (dataService as any)._client = createMongoClientMock(clientConfig);
+      const client = createMongoClientMock(clientConfig);
+      (dataService as any)._crudClient = client;
+      (dataService as any)._metadataClient = client;
       return dataService;
     }