feat: detect CSFLE library path and pass it to driver MONGOSH-1118

packages/arg-parser/src/arg-mapper.ts

@@ -32,6 +32,16 @@ function setAutoEncrypt<Key extends keyof AutoEncryptionOptions>(
   return setDriver(i, 'autoEncryption', autoEncryption);
 }
 
+type AutoEncryptionExtraOptions = NonNullable<AutoEncryptionOptions['extraOptions']>;
+function setAutoEncryptExtra<Key extends keyof AutoEncryptionExtraOptions>(
+  i: Readonly<ConnectionInfo>,
+  key: Key,
+  value: AutoEncryptionExtraOptions[Key]): ConnectionInfo {
+  const extraOptions = i.driverOptions.autoEncryption?.extraOptions ?? {};
+  extraOptions[key] = value;
+  return setAutoEncrypt(i, 'extraOptions', extraOptions);
+}
+
 type AWSKMSOptions = NonNullable<NonNullable<AutoEncryptionOptions['kmsProviders']>['aws']>;
 function setAWSKMS<Key extends keyof AWSKMSOptions>(
   i: Readonly<ConnectionInfo>,
@@ -95,6 +105,7 @@ const MAPPINGS: {
   awsSecretAccessKey: (i, v) => setAWSKMS(i, 'secretAccessKey', v),
   awsSessionToken: (i, v) => setAWSKMS(i, 'sessionToken', v),
   awsIamSessionToken: (i, v) => setAuthMechProp(i, 'AWS_SESSION_TOKEN', v),
+  csfleLibraryPath: (i, v) => setAutoEncryptExtra(i, 'csflePath', v),
   gssapiServiceName: (i, v) => setAuthMechProp(i, 'SERVICE_NAME', v),
   sspiRealmOverride: (i, v) => setAuthMechProp(i, 'SERVICE_REALM', v),
   sspiHostnameCanonicalization:

packages/arg-parser/src/cli-options.ts

@@ -16,6 +16,7 @@ export interface CliOptions {
   awsIamSessionToken?: string;
   awsSecretAccessKey?: string;
   awsSessionToken?: string;
+  csfleLibraryPath?: string;
   db?: string;
   eval?: string;
   gssapiServiceName?: string;

packages/build/src/compile/signable-compiler.ts

@@ -73,6 +73,10 @@ export class SignableCompiler {
       path: await findModulePath('service-provider-server', 'os-dns-native'),
       requireRegexp: /\bos_dns_native\.node$/
     };
+    const csfleLibraryVersionAddon = {
+      path: await findModulePath('cli-repl', 'mongodb-csfle-library-version'),
+      requireRegexp: /\bmongodb_csfle_library_version\.node$/
+    };
     // Warning! Until https://jira.mongodb.org/browse/MONGOSH-990,
     // packages/service-provider-server *also* has a copy of these.
     // We use the versions included in packages/cli-repl here, so these
@@ -110,7 +114,8 @@ export class SignableCompiler {
       addons: [
         fleAddon,
         osDnsAddon,
-        kerberosAddon
+        kerberosAddon,
+        csfleLibraryVersionAddon
       ].concat(winCAAddon ? [
         winCAAddon
       ] : []).concat(winConsoleProcessListAddon ? [

packages/build/src/packaging/download-csfle-library.ts

@@ -4,21 +4,20 @@ import { promises as fs, constants as fsConstants } from 'fs';
 import { downloadMongoDb, DownloadOptions } from '../download-mongodb';
 import { BuildVariant, getDistro, getArch } from '../config';
 
-export async function downloadCsfleLibrary(variant: BuildVariant): Promise<string> {
+export async function downloadCsfleLibrary(variant: BuildVariant | 'host'): Promise<string> {
   const opts: DownloadOptions = {};
-  opts.arch = getArch(variant);
-  opts.distro = lookupReleaseDistro(variant);
+  opts.arch = variant === 'host' ? undefined : getArch(variant);
+  opts.distro = variant === 'host' ? undefined : lookupReleaseDistro(variant);
   opts.enterprise = true;
   opts.csfle = true;
   console.info('mongosh: downloading latest csfle shared library for inclusion in package:', JSON.stringify(opts));
 
   let libdir = '';
   const csfleTmpTargetDir = path.resolve(__dirname, '..', '..', '..', '..', 'tmp', 'csfle-store', variant);
-  // Download mongodb for latest server version. Since the CSFLE shared
-  // library is not part of a non-rc release yet and 5.3.0 not released yet, try:
-  // 1. release server version, 2. '5.3.0' specifically, 3. any version at all
+  // Download mongodb for latest server version. Fall back to the 6.0.0-rcX
+  // version if no stable version is available.
   let error: Error | undefined;
-  for (const version of [ 'stable', '5.3.0', 'unstable' ]) {
+  for (const version of [ 'stable', '>= 6.0.0-rc0' ]) {
     try {
       libdir = await downloadMongoDb(csfleTmpTargetDir, version, opts);
       break;

packages/cli-repl/package.json

@@ -86,11 +86,13 @@
     "@types/yargs-parser": "^15.0.0",
     "chai-as-promised": "^7.1.1",
     "lodash": "^4.17.21",
-    "moment": "^2.29.1"
+    "moment": "^2.29.1",
+    "mongodb-csfle-library-dummy": "^1.0.1"
   },
   "optionalDependencies": {
     "macos-export-certificate-and-key": "^1.1.1",
     "win-export-certificate-and-key": "^1.1.1",
-    "get-console-process-list": "^1.0.4"
+    "get-console-process-list": "^1.0.4",
+    "mongodb-csfle-library-version": "^1.0.2"
   }
 }

packages/cli-repl/src/arg-parser.ts

@@ -23,6 +23,7 @@ const OPTIONS = {
     'awsSecretAccessKey',
     'awsSessionToken',
     'awsIamSessionToken',
+    'csfleLibraryPath',
     'db',
     'eval',
     'gssapiHostName',

packages/cli-repl/src/cli-repl.ts

@@ -18,8 +18,8 @@ import { buildInfo } from './build-info';
 import type { StyleDefinition } from './clr';
 import { ConfigManager, ShellHomeDirectory, ShellHomePaths } from './config-directory';
 import { CliReplErrors } from './error-codes';
+import type { CSFLELibraryPathResult } from './csfle-library-paths';
 import { MongoLogManager, MongoLogWriter, mongoLogId } from 'mongodb-log-writer';
-import { MongocryptdManager } from './mongocryptd-manager';
 import MongoshNodeRepl, { MongoshNodeReplOptions, MongoshIOProvider } from './mongosh-repl';
 import { setupLoggerAndTelemetry, ToggleableAnalytics } from '@mongosh/logging';
 import { MongoshBus, CliUserConfig, CliUserConfigValidator } from '@mongosh/types';
@@ -50,8 +50,8 @@ type AnalyticsOptions = {
 export type CliReplOptions = {
   /** The set of parsed command line flags. */
   shellCliOptions: CliOptions;
-  /** The list of executable paths for mongocryptd. */
-  mongocryptdSpawnPaths?: string[][],
+  /** A function for getting the shared library path for CSFLE. */
+  getCSFLELibraryPaths?: (bus: MongoshBus) => Promise<CSFLELibraryPathResult>;
   /** The stream to read user input from. */
   input: Readable;
   /** The stream to write shell output to. */
@@ -78,7 +78,8 @@ class CliRepl implements MongoshIOProvider {
   mongoshRepl: MongoshNodeRepl;
   bus: MongoshBus;
   cliOptions: CliOptions;
-  mongocryptdManager: MongocryptdManager;
+  getCSFLELibraryPaths?: (bus: MongoshBus) => Promise<CSFLELibraryPathResult>;
+  cachedCSFLELibraryPath?: Promise<CSFLELibraryPathResult>;
   shellHomeDirectory: ShellHomeDirectory;
   configDirectory: ConfigManager<CliUserConfigOnDisk>;
   config: CliUserConfigOnDisk;
@@ -113,6 +114,7 @@ class CliRepl implements MongoshIOProvider {
       enableTelemetry: true
     };
 
+    this.getCSFLELibraryPaths = options.getCSFLELibraryPaths;
     this.globalConfigPaths = options.globalConfigPaths ?? [];
     this.shellHomeDirectory = new ShellHomeDirectory(options.shellHomePaths);
     this.configDirectory = new ConfigManager<CliUserConfigOnDisk>(
@@ -129,11 +131,6 @@ class CliRepl implements MongoshIOProvider {
         this.bus.emit('mongosh:update-user', { userId: config.userId, anonymousId: config.telemetryAnonymousId });
       });
 
-    this.mongocryptdManager = new MongocryptdManager(
-      options.mongocryptdSpawnPaths ?? [],
-      this.shellHomeDirectory,
-      this.bus);
-
     this.logManager = new MongoLogManager({
       directory: this.shellHomeDirectory.localPath('.'),
       retentionDays: 30,
@@ -235,13 +232,26 @@ class CliRepl implements MongoshIOProvider {
     this.globalConfig = await this.loadGlobalConfigFile();
 
     if (driverOptions.autoEncryption) {
+      const origExtraOptions = driverOptions.autoEncryption.extraOptions ?? {};
+      if (origExtraOptions.csflePath) {
+        // If a CSFLE path has been specified through 'driverOptions', save it
+        // for later use.
+        this.cachedCSFLELibraryPath = Promise.resolve({
+          csflePath: origExtraOptions.csflePath
+        });
+      }
+
       const extraOptions = {
-        ...(driverOptions.autoEncryption.extraOptions ?? {}),
-        ...(await this.startMongocryptd())
+        ...origExtraOptions,
+        ...await this.getCSFLELibraryOptions()
       };
 
       driverOptions.autoEncryption = { ...driverOptions.autoEncryption, extraOptions };
     }
+    if (Object.keys(driverOptions.autoEncryption ?? {}).join(',') === 'extraOptions') {
+      // In this case, autoEncryption opts were only specified for CSFLE library specs
+      delete driverOptions.autoEncryption;
+    }
 
     const initialServiceProvider = await this.connect(driverUri, driverOptions);
     const initialized = await this.mongoshRepl.initialize(initialServiceProvider);
@@ -601,7 +611,6 @@ class CliRepl implements MongoshIOProvider {
         flushDuration = Date.now() - flushStart;
       }
     }
-    this.mongocryptdManager.close();
     // eslint-disable-next-line chai-friendly/no-unused-expressions
     this.logWriter?.info('MONGOSH', mongoLogId(1_000_000_045), 'analytics', 'Flushed outstanding data', {
       flushError,
@@ -640,16 +649,12 @@ class CliRepl implements MongoshIOProvider {
     return this.mongoshRepl.clr(text, style);
   }
 
-  /** Start a mongocryptd instance for automatic FLE. */
-  async startMongocryptd(): Promise<AutoEncryptionOptions['extraOptions']> {
-    try {
-      return await this.mongocryptdManager.start();
-    } catch (e: any) {
-      if (e?.code === 'ENOENT') {
-        throw new MongoshRuntimeError('Could not find a working mongocryptd - ensure your local installation works correctly. See the mongosh log file for additional information. Please also refer to the documentation: https://docs.mongodb.com/manual/reference/security-client-side-encryption-appendix/');
-      }
-      throw e;
+  /** Get the right CSFLE shared library loading options. */
+  async getCSFLELibraryOptions(): Promise<AutoEncryptionOptions['extraOptions']> {
+    if (!this.getCSFLELibraryPaths) {
+      throw new MongoshInternalError('This instance of mongosh is not configured for CSFLE');
     }
+    return (this.cachedCSFLELibraryPath ??= this.getCSFLELibraryPaths(this.bus));
   }
 
   /** Provide extra information for reporting internal errors */

packages/cli-repl/src/csfle-library-paths.spec.ts

@@ -0,0 +1,78 @@
+import { expect } from 'chai';
+import { SHARED_LIBRARY_SUFFIX, getCSFLELibraryPaths } from './csfle-library-paths';
+import csfleLibraryDummy from 'mongodb-csfle-library-dummy';
+import type { MongoshBus } from '@mongosh/types';
+import { useTmpdir } from '../test/repl-helpers';
+import { EventEmitter } from 'events';
+import { promises as fs } from 'fs';
+import path from 'path';
+
+describe('getCSFLELibraryPaths', () => {
+  let bus: MongoshBus;
+  let events: any[];
+  let fakeMongoshExecPath: string;
+  const tmpdir = useTmpdir();
+  const csfleFilename = `mongosh_csfle_v1.${SHARED_LIBRARY_SUFFIX}`;
+  const expectedVersion = { version: BigInt('0x0001000000000000'), versionStr: 'mongo_csfle_v1-dummy' };
+
+  beforeEach(async function() {
+    events = [];
+    bus = new EventEmitter();
+    bus.on('mongosh:csfle-load-found', (ev) => events.push(['mongosh:csfle-load-found', ev]));
+    bus.on('mongosh:csfle-load-skip', (ev) => events.push(['mongosh:csfle-load-skip', ev]));
+    fakeMongoshExecPath = path.join(tmpdir.path, 'bin', 'mongosh');
+    await fs.mkdir(path.join(tmpdir.path, 'bin'), { recursive: true });
+    await fs.mkdir(path.join(tmpdir.path, 'lib'), { recursive: true });
+    await fs.writeFile(fakeMongoshExecPath, '# dummy', { mode: 0o755 });
+  });
+
+  it('will look up a shared library located in <bindir>/../lib/', async function() {
+    const csflePath = path.join(tmpdir.path, 'lib', csfleFilename);
+    await fs.copyFile(csfleLibraryDummy, csflePath);
+    expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({
+      csflePath,
+      expectedVersion
+    });
+    expect(events).to.deep.equal([
+      [ 'mongosh:csfle-load-found', { csflePath, expectedVersion } ]
+    ]);
+  });
+
+  it('will look up a shared library located in <bindir>/', async function() {
+    const csflePath = path.join(tmpdir.path, 'bin', csfleFilename);
+    await fs.copyFile(csfleLibraryDummy, csflePath);
+    expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({
+      csflePath,
+      expectedVersion
+    });
+    expect(events[0][0]).to.equal('mongosh:csfle-load-skip');
+    expect(events[0][1].reason).to.include('ENOENT');
+    expect(events.slice(1)).to.deep.equal([
+      [ 'mongosh:csfle-load-found', { csflePath, expectedVersion } ]
+    ]);
+  });
+
+  it('will reject a shared library if it is not readable', async function() {
+    if (process.platform === 'win32') {
+      return this.skip();
+    }
+    const csflePath = path.join(tmpdir.path, 'lib', csfleFilename);
+    await fs.copyFile(csfleLibraryDummy, csflePath);
+    await fs.chmod(csflePath, 0o000);
+    expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({});
+    expect(events[0][0]).to.equal('mongosh:csfle-load-skip');
+    expect(events[0][1].reason).to.include('EACCES');
+  });
+
+  it('will reject a shared library if its permissions are world-writable', async function() {
+    if (process.platform === 'win32') {
+      return this.skip();
+    }
+    const csflePath = path.join(tmpdir.path, 'lib', csfleFilename);
+    await fs.copyFile(csfleLibraryDummy, csflePath);
+    await fs.chmod(csflePath, 0o777);
+    expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({});
+    expect(events[0][0]).to.equal('mongosh:csfle-load-skip');
+    expect(events[0][1].reason).to.include('permissions mismatch');
+  });
+});