diff --git a/packages/arg-parser/src/arg-mapper.ts b/packages/arg-parser/src/arg-mapper.ts index ff6f514b20..c96707b6b2 100644 --- a/packages/arg-parser/src/arg-mapper.ts +++ b/packages/arg-parser/src/arg-mapper.ts @@ -32,6 +32,16 @@ function setAutoEncrypt( return setDriver(i, 'autoEncryption', autoEncryption); } +type AutoEncryptionExtraOptions = NonNullable; +function setAutoEncryptExtra( + i: Readonly, + key: Key, + value: AutoEncryptionExtraOptions[Key]): ConnectionInfo { + const extraOptions = i.driverOptions.autoEncryption?.extraOptions ?? {}; + extraOptions[key] = value; + return setAutoEncrypt(i, 'extraOptions', extraOptions); +} + type AWSKMSOptions = NonNullable['aws']>; function setAWSKMS( i: Readonly, @@ -95,6 +105,7 @@ const MAPPINGS: { awsSecretAccessKey: (i, v) => setAWSKMS(i, 'secretAccessKey', v), awsSessionToken: (i, v) => setAWSKMS(i, 'sessionToken', v), awsIamSessionToken: (i, v) => setAuthMechProp(i, 'AWS_SESSION_TOKEN', v), + csfleLibraryPath: (i, v) => setAutoEncryptExtra(i, 'csflePath', v), gssapiServiceName: (i, v) => setAuthMechProp(i, 'SERVICE_NAME', v), sspiRealmOverride: (i, v) => setAuthMechProp(i, 'SERVICE_REALM', v), sspiHostnameCanonicalization: diff --git a/packages/arg-parser/src/cli-options.ts b/packages/arg-parser/src/cli-options.ts index 5aef32e7bd..3976487feb 100644 --- a/packages/arg-parser/src/cli-options.ts +++ b/packages/arg-parser/src/cli-options.ts @@ -16,6 +16,7 @@ export interface CliOptions { awsIamSessionToken?: string; awsSecretAccessKey?: string; awsSessionToken?: string; + csfleLibraryPath?: string; db?: string; eval?: string; gssapiServiceName?: string; diff --git a/packages/build/src/compile/signable-compiler.ts b/packages/build/src/compile/signable-compiler.ts index 6100feafad..35a3568657 100644 --- a/packages/build/src/compile/signable-compiler.ts +++ b/packages/build/src/compile/signable-compiler.ts @@ -73,6 +73,10 @@ export class SignableCompiler { path: await findModulePath('service-provider-server', 'os-dns-native'), requireRegexp: /\bos_dns_native\.node$/ }; + const csfleLibraryVersionAddon = { + path: await findModulePath('cli-repl', 'mongodb-csfle-library-version'), + requireRegexp: /\bmongodb_csfle_library_version\.node$/ + }; // Warning! Until https://jira.mongodb.org/browse/MONGOSH-990, // packages/service-provider-server *also* has a copy of these. // We use the versions included in packages/cli-repl here, so these @@ -110,7 +114,8 @@ export class SignableCompiler { addons: [ fleAddon, osDnsAddon, - kerberosAddon + kerberosAddon, + csfleLibraryVersionAddon ].concat(winCAAddon ? [ winCAAddon ] : []).concat(winConsoleProcessListAddon ? [ diff --git a/packages/build/src/packaging/download-csfle-library.ts b/packages/build/src/packaging/download-csfle-library.ts index 8f473a1574..b0f98f0794 100644 --- a/packages/build/src/packaging/download-csfle-library.ts +++ b/packages/build/src/packaging/download-csfle-library.ts @@ -4,21 +4,20 @@ import { promises as fs, constants as fsConstants } from 'fs'; import { downloadMongoDb, DownloadOptions } from '../download-mongodb'; import { BuildVariant, getDistro, getArch } from '../config'; -export async function downloadCsfleLibrary(variant: BuildVariant): Promise { +export async function downloadCsfleLibrary(variant: BuildVariant | 'host'): Promise { const opts: DownloadOptions = {}; - opts.arch = getArch(variant); - opts.distro = lookupReleaseDistro(variant); + opts.arch = variant === 'host' ? undefined : getArch(variant); + opts.distro = variant === 'host' ? undefined : lookupReleaseDistro(variant); opts.enterprise = true; opts.csfle = true; console.info('mongosh: downloading latest csfle shared library for inclusion in package:', JSON.stringify(opts)); let libdir = ''; const csfleTmpTargetDir = path.resolve(__dirname, '..', '..', '..', '..', 'tmp', 'csfle-store', variant); - // Download mongodb for latest server version. Since the CSFLE shared - // library is not part of a non-rc release yet and 5.3.0 not released yet, try: - // 1. release server version, 2. '5.3.0' specifically, 3. any version at all + // Download mongodb for latest server version. Fall back to the 6.0.0-rcX + // version if no stable version is available. let error: Error | undefined; - for (const version of [ 'stable', '5.3.0', 'unstable' ]) { + for (const version of [ 'stable', '>= 6.0.0-rc0' ]) { try { libdir = await downloadMongoDb(csfleTmpTargetDir, version, opts); break; diff --git a/packages/cli-repl/package-lock.json b/packages/cli-repl/package-lock.json index 71b717f2a4..be56bf70ed 100644 --- a/packages/cli-repl/package-lock.json +++ b/packages/cli-repl/package-lock.json @@ -1504,6 +1504,22 @@ "whatwg-url": "^11.0.0" } }, + "mongodb-csfle-library-dummy": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mongodb-csfle-library-dummy/-/mongodb-csfle-library-dummy-1.0.1.tgz", + "integrity": "sha512-HrNXwbXcgyO93EqCuNpeUborAmO5Vmr8ZCswxk7MdAke1dO8gmNo02NlabE47PwiCGzf3TPzmX5cf4K7TXIt3Q==", + "dev": true + }, + "mongodb-csfle-library-version": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mongodb-csfle-library-version/-/mongodb-csfle-library-version-1.0.2.tgz", + "integrity": "sha512-DzO4BDGh8nQUEjr7HcB9w1K1CZlfWQRA1Rkq1ROk8aJoaaEK2m++cyVHVUNzHGrYu7X1r5yqHlGxfPw5bSEU0w==", + "optional": true, + "requires": { + "bindings": "^1.5.0", + "node-addon-api": "^4.3.0" + } + }, "mongodb-log-writer": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/mongodb-log-writer/-/mongodb-log-writer-1.1.4.tgz", diff --git a/packages/cli-repl/package.json b/packages/cli-repl/package.json index dfc4291602..41bc612cc0 100644 --- a/packages/cli-repl/package.json +++ b/packages/cli-repl/package.json @@ -86,11 +86,13 @@ "@types/yargs-parser": "^15.0.0", "chai-as-promised": "^7.1.1", "lodash": "^4.17.21", - "moment": "^2.29.1" + "moment": "^2.29.1", + "mongodb-csfle-library-dummy": "^1.0.1" }, "optionalDependencies": { "macos-export-certificate-and-key": "^1.1.1", "win-export-certificate-and-key": "^1.1.1", - "get-console-process-list": "^1.0.4" + "get-console-process-list": "^1.0.4", + "mongodb-csfle-library-version": "^1.0.2" } } diff --git a/packages/cli-repl/src/arg-parser.ts b/packages/cli-repl/src/arg-parser.ts index eb3a9a48c9..af5791258a 100644 --- a/packages/cli-repl/src/arg-parser.ts +++ b/packages/cli-repl/src/arg-parser.ts @@ -23,6 +23,7 @@ const OPTIONS = { 'awsSecretAccessKey', 'awsSessionToken', 'awsIamSessionToken', + 'csfleLibraryPath', 'db', 'eval', 'gssapiHostName', diff --git a/packages/cli-repl/src/cli-repl.ts b/packages/cli-repl/src/cli-repl.ts index d6bf515915..5581e16437 100644 --- a/packages/cli-repl/src/cli-repl.ts +++ b/packages/cli-repl/src/cli-repl.ts @@ -18,8 +18,8 @@ import { buildInfo } from './build-info'; import type { StyleDefinition } from './clr'; import { ConfigManager, ShellHomeDirectory, ShellHomePaths } from './config-directory'; import { CliReplErrors } from './error-codes'; +import type { CSFLELibraryPathResult } from './csfle-library-paths'; import { MongoLogManager, MongoLogWriter, mongoLogId } from 'mongodb-log-writer'; -import { MongocryptdManager } from './mongocryptd-manager'; import MongoshNodeRepl, { MongoshNodeReplOptions, MongoshIOProvider } from './mongosh-repl'; import { setupLoggerAndTelemetry, ToggleableAnalytics } from '@mongosh/logging'; import { MongoshBus, CliUserConfig, CliUserConfigValidator } from '@mongosh/types'; @@ -50,8 +50,8 @@ type AnalyticsOptions = { export type CliReplOptions = { /** The set of parsed command line flags. */ shellCliOptions: CliOptions; - /** The list of executable paths for mongocryptd. */ - mongocryptdSpawnPaths?: string[][], + /** A function for getting the shared library path for CSFLE. */ + getCSFLELibraryPaths?: (bus: MongoshBus) => Promise; /** The stream to read user input from. */ input: Readable; /** The stream to write shell output to. */ @@ -78,7 +78,8 @@ class CliRepl implements MongoshIOProvider { mongoshRepl: MongoshNodeRepl; bus: MongoshBus; cliOptions: CliOptions; - mongocryptdManager: MongocryptdManager; + getCSFLELibraryPaths?: (bus: MongoshBus) => Promise; + cachedCSFLELibraryPath?: Promise; shellHomeDirectory: ShellHomeDirectory; configDirectory: ConfigManager; config: CliUserConfigOnDisk; @@ -113,6 +114,7 @@ class CliRepl implements MongoshIOProvider { enableTelemetry: true }; + this.getCSFLELibraryPaths = options.getCSFLELibraryPaths; this.globalConfigPaths = options.globalConfigPaths ?? []; this.shellHomeDirectory = new ShellHomeDirectory(options.shellHomePaths); this.configDirectory = new ConfigManager( @@ -129,11 +131,6 @@ class CliRepl implements MongoshIOProvider { this.bus.emit('mongosh:update-user', { userId: config.userId, anonymousId: config.telemetryAnonymousId }); }); - this.mongocryptdManager = new MongocryptdManager( - options.mongocryptdSpawnPaths ?? [], - this.shellHomeDirectory, - this.bus); - this.logManager = new MongoLogManager({ directory: this.shellHomeDirectory.localPath('.'), retentionDays: 30, @@ -235,13 +232,26 @@ class CliRepl implements MongoshIOProvider { this.globalConfig = await this.loadGlobalConfigFile(); if (driverOptions.autoEncryption) { + const origExtraOptions = driverOptions.autoEncryption.extraOptions ?? {}; + if (origExtraOptions.csflePath) { + // If a CSFLE path has been specified through 'driverOptions', save it + // for later use. + this.cachedCSFLELibraryPath = Promise.resolve({ + csflePath: origExtraOptions.csflePath + }); + } + const extraOptions = { - ...(driverOptions.autoEncryption.extraOptions ?? {}), - ...(await this.startMongocryptd()) + ...origExtraOptions, + ...await this.getCSFLELibraryOptions() }; driverOptions.autoEncryption = { ...driverOptions.autoEncryption, extraOptions }; } + if (Object.keys(driverOptions.autoEncryption ?? {}).join(',') === 'extraOptions') { + // In this case, autoEncryption opts were only specified for CSFLE library specs + delete driverOptions.autoEncryption; + } const initialServiceProvider = await this.connect(driverUri, driverOptions); const initialized = await this.mongoshRepl.initialize(initialServiceProvider); @@ -601,7 +611,6 @@ class CliRepl implements MongoshIOProvider { flushDuration = Date.now() - flushStart; } } - this.mongocryptdManager.close(); // eslint-disable-next-line chai-friendly/no-unused-expressions this.logWriter?.info('MONGOSH', mongoLogId(1_000_000_045), 'analytics', 'Flushed outstanding data', { flushError, @@ -640,16 +649,12 @@ class CliRepl implements MongoshIOProvider { return this.mongoshRepl.clr(text, style); } - /** Start a mongocryptd instance for automatic FLE. */ - async startMongocryptd(): Promise { - try { - return await this.mongocryptdManager.start(); - } catch (e: any) { - if (e?.code === 'ENOENT') { - throw new MongoshRuntimeError('Could not find a working mongocryptd - ensure your local installation works correctly. See the mongosh log file for additional information. Please also refer to the documentation: https://docs.mongodb.com/manual/reference/security-client-side-encryption-appendix/'); - } - throw e; + /** Get the right CSFLE shared library loading options. */ + async getCSFLELibraryOptions(): Promise { + if (!this.getCSFLELibraryPaths) { + throw new MongoshInternalError('This instance of mongosh is not configured for CSFLE'); } + return (this.cachedCSFLELibraryPath ??= this.getCSFLELibraryPaths(this.bus)); } /** Provide extra information for reporting internal errors */ diff --git a/packages/cli-repl/src/csfle-library-paths.spec.ts b/packages/cli-repl/src/csfle-library-paths.spec.ts new file mode 100644 index 0000000000..0f3c872a52 --- /dev/null +++ b/packages/cli-repl/src/csfle-library-paths.spec.ts @@ -0,0 +1,78 @@ +import { expect } from 'chai'; +import { SHARED_LIBRARY_SUFFIX, getCSFLELibraryPaths } from './csfle-library-paths'; +import csfleLibraryDummy from 'mongodb-csfle-library-dummy'; +import type { MongoshBus } from '@mongosh/types'; +import { useTmpdir } from '../test/repl-helpers'; +import { EventEmitter } from 'events'; +import { promises as fs } from 'fs'; +import path from 'path'; + +describe('getCSFLELibraryPaths', () => { + let bus: MongoshBus; + let events: any[]; + let fakeMongoshExecPath: string; + const tmpdir = useTmpdir(); + const csfleFilename = `mongosh_csfle_v1.${SHARED_LIBRARY_SUFFIX}`; + const expectedVersion = { version: BigInt('0x0001000000000000'), versionStr: 'mongo_csfle_v1-dummy' }; + + beforeEach(async function() { + events = []; + bus = new EventEmitter(); + bus.on('mongosh:csfle-load-found', (ev) => events.push(['mongosh:csfle-load-found', ev])); + bus.on('mongosh:csfle-load-skip', (ev) => events.push(['mongosh:csfle-load-skip', ev])); + fakeMongoshExecPath = path.join(tmpdir.path, 'bin', 'mongosh'); + await fs.mkdir(path.join(tmpdir.path, 'bin'), { recursive: true }); + await fs.mkdir(path.join(tmpdir.path, 'lib'), { recursive: true }); + await fs.writeFile(fakeMongoshExecPath, '# dummy', { mode: 0o755 }); + }); + + it('will look up a shared library located in /../lib/', async function() { + const csflePath = path.join(tmpdir.path, 'lib', csfleFilename); + await fs.copyFile(csfleLibraryDummy, csflePath); + expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({ + csflePath, + expectedVersion + }); + expect(events).to.deep.equal([ + [ 'mongosh:csfle-load-found', { csflePath, expectedVersion } ] + ]); + }); + + it('will look up a shared library located in /', async function() { + const csflePath = path.join(tmpdir.path, 'bin', csfleFilename); + await fs.copyFile(csfleLibraryDummy, csflePath); + expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({ + csflePath, + expectedVersion + }); + expect(events[0][0]).to.equal('mongosh:csfle-load-skip'); + expect(events[0][1].reason).to.include('ENOENT'); + expect(events.slice(1)).to.deep.equal([ + [ 'mongosh:csfle-load-found', { csflePath, expectedVersion } ] + ]); + }); + + it('will reject a shared library if it is not readable', async function() { + if (process.platform === 'win32') { + return this.skip(); + } + const csflePath = path.join(tmpdir.path, 'lib', csfleFilename); + await fs.copyFile(csfleLibraryDummy, csflePath); + await fs.chmod(csflePath, 0o000); + expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({}); + expect(events[0][0]).to.equal('mongosh:csfle-load-skip'); + expect(events[0][1].reason).to.include('EACCES'); + }); + + it('will reject a shared library if its permissions are world-writable', async function() { + if (process.platform === 'win32') { + return this.skip(); + } + const csflePath = path.join(tmpdir.path, 'lib', csfleFilename); + await fs.copyFile(csfleLibraryDummy, csflePath); + await fs.chmod(csflePath, 0o777); + expect(await getCSFLELibraryPaths(bus, fakeMongoshExecPath)).to.deep.equal({}); + expect(events[0][0]).to.equal('mongosh:csfle-load-skip'); + expect(events[0][1].reason).to.include('permissions mismatch'); + }); +}); diff --git a/packages/cli-repl/src/csfle-library-paths.ts b/packages/cli-repl/src/csfle-library-paths.ts new file mode 100644 index 0000000000..5979124f92 --- /dev/null +++ b/packages/cli-repl/src/csfle-library-paths.ts @@ -0,0 +1,99 @@ +import path from 'path'; +import { promises as fs, constants as fsConstants } from 'fs'; +import type { MongoshBus } from '@mongosh/types'; + +export const SHARED_LIBRARY_SUFFIX = + // eslint-disable-next-line no-nested-ternary + process.platform === 'win32' ? 'dll' : + process.platform === 'darwin' ? 'dylib' : 'so'; + +export interface CSFLELibraryPathResult { + csflePath?: string; + expectedVersion?: { version: bigint; versionStr: string }; +} + +/** + * Figure out the possible shared library paths for the CSFLE shared library + * that we are supposed to use. + */ +export async function getCSFLELibraryPaths( + bus: MongoshBus, + pretendProcessExecPathForTesting: string | undefined = undefined): Promise { + const execPath = pretendProcessExecPathForTesting ?? process.execPath; + + let getCSFLESharedLibraryVersion: typeof import('mongodb-csfle-library-version'); + try { + getCSFLESharedLibraryVersion = require('mongodb-csfle-library-version'); + } catch (err) { + getCSFLESharedLibraryVersion = () => ({ version: BigInt(0), versionStr: '' }); + } + + if (execPath === process.argv[1] || pretendProcessExecPathForTesting) { + const bindir = path.dirname(execPath); + const execPathStat = await fs.stat(execPath); + for await (const libraryCandidate of [ + // Location of the shared library in the deb and rpm packages + path.resolve(bindir, '..', 'lib', `mongosh_csfle_v1.${SHARED_LIBRARY_SUFFIX}`), + // Location of the shared library in the zip and tgz packages + path.resolve(bindir, `mongosh_csfle_v1.${SHARED_LIBRARY_SUFFIX}`) + ]) { + try { + const permissionsMismatch = await ensureMatchingPermissions(libraryCandidate, execPathStat); + if (permissionsMismatch) { + bus.emit('mongosh:csfle-load-skip', { + csflePath: libraryCandidate, + reason: 'permissions mismatch', + details: permissionsMismatch + }); + continue; + } + + const version = getCSFLESharedLibraryVersion(libraryCandidate); + const result = { + csflePath: libraryCandidate, + expectedVersion: version + }; + bus.emit('mongosh:csfle-load-found', result); + return result; + } catch (err: any) { + bus.emit('mongosh:csfle-load-skip', { + csflePath: libraryCandidate, + reason: err.message + }); + } + } + } else { + bus.emit('mongosh:csfle-load-skip', { + csflePath: '', + reason: 'Skipping CSFLE library searching because this is not a single-executable mongosh' + }); + } + return {}; +} + +// Check whether permissions for a file match what we expect them to be. +// Returns 'null' in case of no mismatch and information that is useful +// for debugging/logging in the mismatch case. +async function ensureMatchingPermissions(filename: string, execPathStat: { uid: number, gid: number }): Promise { + if (process.platform === 'win32') { + // On Windows systems, there are no permissions checks that + // we could reasonably do here. + return null; + } + await fs.access(filename, fsConstants.R_OK); + const stat = await fs.stat(filename); + // On UNIX systems, only load shared libraries if they are coming + // from a user we can consider trusted (current user or the one who owns + // the mongosh binary to begin with) and they are not writable by other + // users. + if (((stat.uid !== execPathStat.uid && stat.uid !== process.getuid()) || + (stat.gid !== execPathStat.gid && stat.gid !== process.getgid()) || + stat.mode & 0o002 /* world-writable */)) { + return { + libraryStat: { uid: stat.uid, gid: stat.gid, mode: stat.mode }, + mongoshStat: { uid: execPathStat.uid, gid: stat.gid }, + currentUser: { uid: process.getuid(), gid: process.getgid() } + }; + } + return null; +} diff --git a/packages/cli-repl/src/index.ts b/packages/cli-repl/src/index.ts index 59c4f96b5d..4f141eb009 100644 --- a/packages/cli-repl/src/index.ts +++ b/packages/cli-repl/src/index.ts @@ -3,7 +3,6 @@ import CliRepl from './cli-repl'; import clr from './clr'; import { getStoragePaths } from './config-directory'; import { MONGOSH_WIKI, TELEMETRY_GREETING_MESSAGE, USAGE } from './constants'; -import { getMongocryptdPaths } from './mongocryptd-manager'; import { runSmokeTests } from './smoke-tests'; import { buildInfo } from './build-info'; @@ -17,7 +16,6 @@ export { CliRepl, parseCliArgs, getStoragePaths, - getMongocryptdPaths, runSmokeTests, buildInfo }; diff --git a/packages/cli-repl/src/mongocryptd-manager.spec.ts b/packages/cli-repl/src/mongocryptd-manager.spec.ts deleted file mode 100644 index 1e12960043..0000000000 --- a/packages/cli-repl/src/mongocryptd-manager.spec.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* eslint-disable chai-friendly/no-unused-expressions */ -import Nanobus from 'nanobus'; -import { promises as fs } from 'fs'; -import path from 'path'; -import { getMongocryptdPaths, MongocryptdManager } from './mongocryptd-manager'; -import type { MongoshBus } from '@mongosh/types'; -import { ShellHomeDirectory } from './config-directory'; -import { startTestServer } from '../../../testing/integration-testing-hooks'; -import { eventually } from '../../../testing/eventually'; -import { expect } from 'chai'; - -describe('getMongocryptdPaths', () => { - it('always includes plain `mongocryptd`', async() => { - expect(await getMongocryptdPaths()).to.deep.include(['mongocryptd']); - }); -}); - -describe('MongocryptdManager', () => { - let basePath: string; - let bus: MongoshBus; - let shellHomeDirectory: ShellHomeDirectory; - let spawnPaths: string[][]; - let manager: MongocryptdManager; - let events: { event: string, data: any }[]; - const makeManager = () => { - manager = new MongocryptdManager(spawnPaths, shellHomeDirectory, bus); - return manager; - }; - - const fakeMongocryptdDir = path.resolve(__dirname, '..', 'test', 'fixtures', 'fake-mongocryptd'); - - beforeEach(() => { - const nanobus = new Nanobus(); - events = []; - nanobus.on('*', (event, data) => events.push({ event: event as string, data })); - bus = nanobus; - - spawnPaths = []; - basePath = path.resolve(__dirname, '..', '..', '..', 'tmp', 'test', `${Date.now()}`, `${Math.random()}`); - shellHomeDirectory = new ShellHomeDirectory({ - shellRoamingDataPath: basePath, - shellLocalDataPath: basePath, - shellRcPath: basePath - }); - }); - afterEach(() => { - manager?.close(); - }); - - it('does a no-op close when not initialized', () => { - expect(makeManager().close().state).to.equal(null); - }); - - for (const otherMongocryptd of ['none', 'missing', 'broken', 'weirdlog', 'broken-after']) { - for (const version of ['4.2', '4.4']) { // This refers to the log format version - for (const variant of ['withunix', 'nounix']) { - // eslint-disable-next-line no-loop-func - it(`spawns a working mongocryptd (${version}, ${variant}, other mongocryptd: ${otherMongocryptd})`, async() => { - spawnPaths = [ - [ - process.execPath, - path.resolve(fakeMongocryptdDir, `working-${version}-${variant}.js`) - ] - ]; - if (otherMongocryptd === 'missing') { - spawnPaths.unshift([ path.resolve(fakeMongocryptdDir, 'nonexistent') ]); - } - if (otherMongocryptd === 'broken') { - spawnPaths.unshift([ process.execPath, path.resolve(fakeMongocryptdDir, 'exit1') ]); - } - if (otherMongocryptd === 'weirdlog') { - spawnPaths.unshift([ process.execPath, path.resolve(fakeMongocryptdDir, 'weirdlog') ]); - } - if (otherMongocryptd === 'broken-after') { - spawnPaths.push([ process.execPath, path.resolve(fakeMongocryptdDir, 'exit1') ]); - } - expect(await makeManager().start()).to.deep.equal({ - mongocryptdURI: variant === 'nounix' ? - 'mongodb://localhost:27020' : - 'mongodb://%2Ftmp%2Fmongocryptd.sock', - mongocryptdBypassSpawn: true - }); - - const tryspawns = events.filter(({ event }) => event === 'mongosh:mongocryptd-tryspawn'); - expect(tryspawns).to.have.lengthOf( - otherMongocryptd === 'none' || otherMongocryptd === 'broken-after' ? 1 : 2); - }); - } - } - } - - it('passes relevant arguments to mongocryptd', async() => { - spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]]; - await makeManager().start(); - const pidfile = path.join(manager.path, 'mongocryptd.pid'); - expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).args).to.deep.equal([ - ...spawnPaths[0], - '--idleShutdownTimeoutSecs', '60', - '--pidfilepath', pidfile, - '--port', '0', - ...(process.platform !== 'win32' ? ['--unixSocketPrefix', path.dirname(pidfile)] : []) - ]); - }); - - it('multiple start() calls are no-ops', async() => { - spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]]; - const manager = makeManager(); - await manager.start(); - const pid1 = manager.state.proc.pid; - await manager.start(); - expect(manager.state.proc.pid).to.equal(pid1); - }); - - it('handles synchronous throws from child_process.spawn', async() => { - spawnPaths = [['']]; - try { - await makeManager().start(); - expect.fail('missed exception'); - } catch (e: any) { - expect(e.code).to.equal('ERR_INVALID_ARG_VALUE'); - } - }); - - it('throws if no spawn paths are provided at all', async() => { - spawnPaths = []; - try { - await makeManager().start(); - expect.fail('missed exception'); - } catch (e: any) { - expect(e.name).to.equal('MongoshInternalError'); - } - }); - - it('includes stderr in the log if stdout is unparseable', async() => { - spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'weirdlog.js')]]; - try { - await makeManager().start(); - expect.fail('missed exception'); - } catch (e: any) { - expect(e.name).to.equal('MongoshInternalError'); - } - const nostdoutErrors = events.filter(({ event, data }) => { - return event === 'mongosh:mongocryptd-error' && data.cause === 'nostdout'; - }); - expect(nostdoutErrors).to.deep.equal([{ - event: 'mongosh:mongocryptd-error', - data: { cause: 'nostdout', stderr: 'Diagnostic message!\n' } - }]); - }); - - it('cleans up previously created, empty directory entries', async() => { - spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'writepidfile.js')]]; - - const manager = makeManager(); - await manager.start(); - const pidfile = path.join(manager.path, 'mongocryptd.pid'); - expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).pid).to.be.a('number'); - manager.close(); - - // The file remains after close, but is gone after creating a new one: - await fs.stat(pidfile); - await makeManager().start(); - try { - await fs.stat(pidfile); - expect.fail('missed exception'); - } catch (e: any) { - expect(e.code).to.equal('ENOENT'); - } - }); - - context('with network testing', () => { - const testServer = startTestServer('shared'); - - beforeEach(async() => { - process.env.MONGOSH_TEST_PROXY_TARGET_PORT = await testServer.port(); - }); - afterEach(() => { - delete process.env.MONGOSH_TEST_PROXY_TARGET_PORT; - }); - - it('performs keepalive pings', async() => { - spawnPaths = [[process.execPath, path.resolve(fakeMongocryptdDir, 'withnetworking.js')]]; - const manager = makeManager(); - manager.idleShutdownTimeoutSecs = 1; - await manager.start(); - const pidfile = path.join(manager.path, 'mongocryptd.pid'); - await eventually(async() => { - expect(JSON.parse(await fs.readFile(pidfile, 'utf8')).connections).to.be.greaterThan(1); - }); - }); - }); -}); diff --git a/packages/cli-repl/src/mongocryptd-manager.ts b/packages/cli-repl/src/mongocryptd-manager.ts deleted file mode 100644 index a0a194265c..0000000000 --- a/packages/cli-repl/src/mongocryptd-manager.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { ChildProcess, spawn } from 'child_process'; -import { promises as fs, constants as fsConstants } from 'fs'; -import { isIP } from 'net'; -import path from 'path'; -import readline from 'readline'; -import { Readable, PassThrough } from 'stream'; -import { MongoshInternalError } from '@mongosh/errors'; -import { CliServiceProvider } from '@mongosh/service-provider-server'; -import type { MongoshBus } from '@mongosh/types'; -import { parseAnyLogEntry, LogEntry } from './log-entry'; -import { ShellHomeDirectory } from './config-directory'; - -/** - * Figure out the possible executable paths for the mongocryptd - * binary that we are supposed to use. - */ -export async function getMongocryptdPaths(): Promise { - const bindir = path.dirname(process.execPath); - const result = []; - for await (const mongocryptdCandidate of [ - // Location of mongocryptd-mongosh in the deb and rpm packages - path.resolve(bindir, '..', 'libexec', 'mongocryptd-mongosh'), - // Location of mongocryptd-mongosh in the zip and tgz packages - path.resolve(bindir, 'mongocryptd-mongosh'), - path.resolve(bindir, 'mongocryptd-mongosh.exe') - ]) { - try { - await fs.access(mongocryptdCandidate, fsConstants.X_OK); - result.push([ mongocryptdCandidate ]); - } catch { /* ignore error */ } - } - return [...result, ['mongocryptd']]; -} - -/** - * The relevant information regarding the state of a mongocryptd process. - */ -type MongocryptdState = { - /** The connection string for the current mongocryptd instance. */ - uri: string; - /** The process handle for the current mongocryptd instance. */ - proc: ChildProcess; - /** An interval to prevent the mongocryptd instance from going idle. */ - interval: NodeJS.Timeout; -}; - -/** - * A helper class to manage mongocryptd child processes that we may need to spawn. - */ -export class MongocryptdManager { - spawnPaths: string[][]; - bus: MongoshBus; - path: string; - state: MongocryptdState | null; - idleShutdownTimeoutSecs = 60; - - /** - * @param spawnPaths A list of executables to spawn - * @param shellHomeDirectory A place for storing mongosh-related files - * @param bus A message bus for sharing diagnostic events about mongocryptd lifetimes - */ - constructor(spawnPaths: string[][], shellHomeDirectory: ShellHomeDirectory, bus: MongoshBus) { - this.spawnPaths = spawnPaths; - this.path = shellHomeDirectory.localPath(`mongocryptd-${process.pid}-${(Math.random() * 100000) | 0}`); - this.bus = bus; - this.state = null; - } - - /** - * Start a mongocryptd process and return matching driver options for it. - */ - async start(): Promise<{ mongocryptdURI: string, mongocryptdBypassSpawn: true }> { - if (!this.state) { - [ this.state ] = await Promise.all([ - this._spawn(), - this._cleanupOldMongocryptdDirectories() - ]); - } - - return { - mongocryptdURI: this.state.uri, - mongocryptdBypassSpawn: true - }; - } - - /** - * Stop the managed mongocryptd process, if any. This is kept synchronous - * in order to be usable inside process.on('exit') listeners. - */ - close = (): this => { - process.removeListener('exit', this.close); - if (this.state) { - this.state.proc.kill(); - clearInterval(this.state.interval); - this.state = null; - } - return this; - }; - - /** - * Create an async iterator over the individual log lines in a mongo(crypt)d - * process's stdout, while also forwarding the log events to the bus. - * - * @param stdout Any Readable stream that follows the mongodb logv2 or logv1 formats. - * @param pid The process id, used for logging. - */ - async* createLogEntryIterator(stdout: Readable, pid: number): AsyncIterable { - for await (const line of readline.createInterface({ input: stdout })) { - if (!line.trim()) { - continue; - } - try { - const logEntry = parseAnyLogEntry(line); - this.bus.emit('mongosh:mongocryptd-log', { pid, logEntry }); - yield logEntry; - } catch (error: any) { - this.bus.emit('mongosh:mongocryptd-error', { pid, cause: 'parse', error }); - break; - } - } - } - - /** - * Create a mongocryptd child process. - * - * @param spawnPath The first arguments to pass on the command line. - */ - _spawnMongocryptdProcess(spawnPath: string[]): ChildProcess { - const [ executable, ...args ] = [ - ...spawnPath, - '--idleShutdownTimeoutSecs', String(this.idleShutdownTimeoutSecs), - '--pidfilepath', path.join(this.path, 'mongocryptd.pid'), - '--port', '0', - ...(process.platform !== 'win32' ? ['--unixSocketPrefix', this.path] : []) - ]; - const proc = spawn(executable, args, { - stdio: ['inherit', 'pipe', 'pipe'] - }); - - proc.on('exit', (code, signal) => { - const logEntry = { exit: { code, signal } }; - this.bus.emit('mongosh:mongocryptd-log', { pid: proc.pid, logEntry }); - }); - return proc; - } - - /** - * Try to spawn mongocryptd using the paths passed to the constructor, - * and parse the process's log to understand on what path/port it - * is listening on. - */ - async _spawn(): Promise { - if (this.spawnPaths.length === 0) { - throw new MongoshInternalError('No mongocryptd spawn path given'); - } - - await fs.mkdir(this.path, { recursive: true, mode: 0o700 }); - process.on('exit', this.close); - - let proc: ChildProcess | undefined = undefined; - let uri = ''; - let lastError: Error | undefined = undefined; - for (const spawnPath of this.spawnPaths) { - this.bus.emit('mongosh:mongocryptd-tryspawn', { spawnPath, path: this.path }); - try { - proc = this._spawnMongocryptdProcess(spawnPath); - } catch (error: any) { - // Spawn can fail both synchronously and asynchronously. - // We log the error either way and just try the next one. - lastError = error; - this.bus.emit('mongosh:mongocryptd-error', { cause: 'spawn', error }); - continue; - } - // eslint-disable-next-line no-loop-func - proc.on('error', (error) => { - lastError = error; - this.bus.emit('mongosh:mongocryptd-error', { cause: 'spawn', error }); - }); - let stderr = ''; - // eslint-disable-next-line chai-friendly/no-unused-expressions - proc.stderr?.setEncoding('utf8').on('data', chunk => { stderr += chunk; }); - - const { pid } = proc; - - // Get an object-mode Readable stream of parsed log events. - const logEntryStream = Readable.from(this.createLogEntryIterator(proc.stdout as Readable, pid)); - const { socket, port } = await filterLogStreamForSocketAndPort(logEntryStream); - if (!socket && port === -1) { - // This likely means that stdout ended before we could get a path/port - // from it, most likely because spawning itself failed. - proc.kill(); - this.bus.emit('mongosh:mongocryptd-error', { cause: 'nostdout', stderr }); - continue; - } - - // Keep the stream going even when not being consumed in order to get - // the log events on the bus. - logEntryStream.resume(); - - // No UNIX socket means we're on Windows, where we have to use networking. - uri = !socket ? `mongodb://localhost:${port}` : `mongodb://${encodeURIComponent(socket)}`; - break; - } - if (!proc || !uri) { - throw lastError ?? new MongoshInternalError('Could not successfully spawn mongocryptd'); - } - - const interval = setInterval(async() => { - // Use half the idle timeout of the process for regular keepalive pings. - let sp; - try { - sp = await CliServiceProvider.connect(uri, { - serverSelectionTimeoutMS: this.idleShutdownTimeoutSecs * 1000 - }, {}, this.bus); - await sp.runCommandWithCheck('admin', { isMaster: 1 }); - } catch (error: any) { - this.bus.emit('mongosh:mongocryptd-error', { cause: 'ping', error }); - } finally { - if (sp !== undefined) { - await sp.close(true); - } - } - }, this.idleShutdownTimeoutSecs * 1000 / 2); - interval.unref(); - proc.unref(); - - return { uri, proc, interval }; - } - - /** - * Run when starting a new mongocryptd process. Clean up old, unused - * directories that were created by previous operations like this. - */ - async _cleanupOldMongocryptdDirectories(): Promise { - try { - const toBeRemoved = []; - for await (const dirent of await fs.opendir(path.resolve(this.path, '..'))) { - // A directory with an empty mongocryptd.pid indicates that the - // mongocryptd process in question has terminated. - if (dirent.name.startsWith('mongocryptd-') && dirent.isDirectory()) { - let size = 0; - try { - size = (await fs.stat(path.join(dirent.name, 'mongocryptd.pid'))).size; - } catch (err: any) { - if (err?.code !== 'ENOENT') { - throw err; - } - } - if (size === 0) { - toBeRemoved.push(path.join(this.path, '..', dirent.name)); - } - } - } - for (const dir of toBeRemoved) { - if (path.resolve(dir) !== path.resolve(this.path)) { - await fs.rmdir(dir, { recursive: true }); - } - } - } catch (error: any) { - this.bus.emit('mongosh:mongocryptd-error', { cause: 'cleanup', error }); - } - } -} - -/** - * Look at a log entry to figure out whether we are listening on a - * UNIX domain socket. - * - * @param logEntry A parsed mongodb log line. - * @returns The domain socket in question, or an empty string if the log line did not match. - */ -function getSocketFromLogEntry(logEntry: LogEntry): string { - let match; - // Log message id 23015 has the format - // { t: , s: 'I', c: 'NETWORK', id: 23016, ctx: 'listener', msg: '...', attr: { address: '/tmp/q/mongocryptd.sock' } } - if (logEntry.id === 23015) { - if (!isIP(logEntry.attr.address)) { - return logEntry.attr.address; - } - } - // Or, 4.2-style: I NETWORK [listener] Listening on /tmp/mongocryptd.sock - if (logEntry.id === undefined && (match = logEntry.message.match(/^Listening on (?.+)$/i))) { - const { addr } = match.groups as any; - if (!isIP(addr)) { - return addr; - } - } - return ''; -} - -/** - * Look at a log entry to figure out whether we are listening on a - * TCP port. - * - * @param logEntry A parsed mongodb log line. - * @returns The port in question, or an -1 if the log line did not match. - */ -function getPortFromLogEntry(logEntry: LogEntry): number { - let match; - // Log message id 23016 has the format - // { t: , s: 'I', c: 'NETWORK', id: 23016, ctx: 'listener', msg: '...', attr: { port: 27020 } } - if (logEntry.id === 23016) { - return logEntry.attr.port; - } - // Or, 4.2-style: I NETWORK [listener] waiting for connections on port 27020 - if (logEntry.id === undefined && (match = logEntry.message.match(/^waiting for connections on port (?\d+)$/i))) { - return +(match.groups?.port ?? '0'); - } - return -1; -} - -/** - * Go through a stream of parsed log entry objects and return the port/path - * data once found. - * - * @input A mongodb logv2/logv1 stream. - * @returns The (UNIX domain socket and) port that the target process is listening on. - */ -async function filterLogStreamForSocketAndPort(input: Readable): Promise<{ port: number, socket: string }> { - let port = -1; - let socket = ''; - const inputDuplicate: AsyncIterable = input.pipe(new PassThrough({ objectMode: true })); - - for await (const logEntry of inputDuplicate) { - if (logEntry.component !== 'NETWORK' || logEntry.context !== 'listener') { - continue; // We are only interested in listening network events - } - socket ||= getSocketFromLogEntry(logEntry); - port = getPortFromLogEntry(logEntry); - if (port !== -1) { - break; - } - } - return { socket, port }; -} diff --git a/packages/cli-repl/src/mongosh-repl.ts b/packages/cli-repl/src/mongosh-repl.ts index ad0c978f93..470d8aebdc 100644 --- a/packages/cli-repl/src/mongosh-repl.ts +++ b/packages/cli-repl/src/mongosh-repl.ts @@ -37,7 +37,7 @@ export type MongoshIOProvider = Omit, 'validateCon getHistoryFilePath(): string; exit(code?: number): Promise; readFileUTF8(filename: string): Promise<{ contents: string, absolutePath: string }>; - startMongocryptd(): Promise; + getCSFLELibraryOptions(): Promise; bugReportErrorMessageInfo?(): string | undefined; }; @@ -855,12 +855,10 @@ class MongoshNodeRepl implements EvaluationListener { } /** - * Start a mongocryptd instance that is required for automatic FLE. - * - * @returns Information about how to connect to the started mongocryptd instance. + * Get the right CSFLE shared library loading options. */ - async startMongocryptd(): Promise { - return this.ioProvider.startMongocryptd(); + async getCSFLELibraryOptions(): Promise { + return this.ioProvider.getCSFLELibraryOptions(); } /** diff --git a/packages/cli-repl/src/run.ts b/packages/cli-repl/src/run.ts index 01eef78394..30309b95f7 100644 --- a/packages/cli-repl/src/run.ts +++ b/packages/cli-repl/src/run.ts @@ -1,5 +1,6 @@ -import { CliRepl, parseCliArgs, getMongocryptdPaths, runSmokeTests, USAGE, buildInfo } from './index'; +import { CliRepl, parseCliArgs, runSmokeTests, USAGE, buildInfo } from './index'; import { getStoragePaths, getGlobalConfigPaths } from './config-directory'; +import { getCSFLELibraryPaths } from './csfle-library-paths'; import { getTlsCertificateSelector } from './tls-certificate-selector'; import { redactURICredentials } from '@mongosh/history'; import { generateConnectionInfoFromCliArgs } from '@mongosh/arg-parser'; @@ -39,22 +40,21 @@ import stream from 'stream'; console.log(JSON.stringify(buildInfo(), null, ' ')); } else if (options.smokeTests) { const smokeTestServer = process.env.MONGOSH_SMOKE_TEST_SERVER; + const csfleLibraryOpts = options.csfleLibraryPath ? [ + `--csfleLibraryPath=${options.csfleLibraryPath}` + ] : []; if (process.execPath === process.argv[1]) { // This is the compiled binary. Use only the path to it. - await runSmokeTests(smokeTestServer, process.execPath); + await runSmokeTests(smokeTestServer, process.execPath, ...csfleLibraryOpts); } else { // This is not the compiled binary. Use node + this script. - await runSmokeTests(smokeTestServer, process.execPath, process.argv[1]); + await runSmokeTests(smokeTestServer, process.execPath, process.argv[1], ...csfleLibraryOpts); } } else { - let mongocryptdSpawnPaths = [['mongocryptd']]; if (process.execPath === process.argv[1]) { // Remove the built-in Node.js listener that prints e.g. deprecation // warnings in single-binary release mode. process.removeAllListeners('warning'); - // Look for mongocryptd in the locations where our packaging would - // have put it. - mongocryptdSpawnPaths = await getMongocryptdPaths(); } // This is for testing under coverage, see the the comment in the tests @@ -98,7 +98,7 @@ import stream from 'stream'; shellCliOptions: { ...options, }, - mongocryptdSpawnPaths, + getCSFLELibraryPaths, input: process.stdin, output: process.stdout, onExit: process.exit, diff --git a/packages/cli-repl/src/smoke-tests-fle.ts b/packages/cli-repl/src/smoke-tests-fle.ts index 6339e076b7..ebf8178088 100644 --- a/packages/cli-repl/src/smoke-tests-fle.ts +++ b/packages/cli-repl/src/smoke-tests-fle.ts @@ -1,6 +1,6 @@ /** - * Test script that verifies that automatic encryption using mongocryptd - * works when using the Mongo() object to construct the encryption key and + * Test script that verifies that automatic encryption using the CSFLE shared + * library works when using the Mongo() object to construct the encryption key and * to create an auto-encryption-aware connection. */ @@ -12,7 +12,7 @@ const assert = function(value, message) { process.exit(1); } }; -// There is no mongocryptd binary for darwin-x64 or rhel80-s390x yet. +// There is no CSFLE shared library binary for darwin-x64 or rhel80-s390x yet. if ((os.platform() === 'darwin' && os.arch() === 'arm64') || (os.platform() === 'linux' && os.arch() === 's390x' && fs.readFileSync('/etc/os-release', 'utf8').includes('VERSION_ID="8'))) { print('Test skipped') diff --git a/packages/cli-repl/src/smoke-tests.spec.ts b/packages/cli-repl/src/smoke-tests.spec.ts index 605feca537..55081fd956 100644 --- a/packages/cli-repl/src/smoke-tests.spec.ts +++ b/packages/cli-repl/src/smoke-tests.spec.ts @@ -1,17 +1,21 @@ import { runSmokeTests } from './'; import path from 'path'; -import { startTestServer, useBinaryPath } from '../../../testing/integration-testing-hooks'; +import { startTestServer, downloadCurrentCsfleSharedLibrary } from '../../../testing/integration-testing-hooks'; describe('smoke tests', () => { const testServer = startTestServer('shared'); - useBinaryPath(testServer); // Get mongocryptd in the PATH for this test + let csfleLibrary: string; + + before(async() => { + csfleLibrary = await downloadCurrentCsfleSharedLibrary(); + }); it('self-test passes', async() => { // Use ts-node to run the .ts files directly so nyc can pick them up for // coverage. await runSmokeTests( await testServer.connectionString(), - process.execPath, '-r', 'ts-node/register', path.resolve(__dirname, 'run.ts') + process.execPath, '-r', 'ts-node/register', path.resolve(__dirname, 'run.ts'), '--csfleLibraryPath', csfleLibrary ); }); }); diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index faf61534dd..ac6def9648 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -2,7 +2,12 @@ import { expect } from 'chai'; import { MongoClient } from 'mongodb'; import { TestShell } from './test-shell'; import { eventually } from '../../../testing/eventually'; -import { startTestServer, useBinaryPath, skipIfServerVersion, skipIfCommunityServer } from '../../../testing/integration-testing-hooks'; +import { + startTestServer, + skipIfServerVersion, + skipIfCommunityServer, + downloadCurrentCsfleSharedLibrary +} from '../../../testing/integration-testing-hooks'; import { makeFakeHTTPServer, fakeAWSHandlers } from '../../../testing/fake-kms'; import { once } from 'events'; import { serialize } from 'v8'; @@ -13,14 +18,15 @@ describe('FLE tests', () => { const testServer = startTestServer('shared'); skipIfServerVersion(testServer, '< 4.2'); // FLE only available on 4.2+ skipIfCommunityServer(testServer); // FLE is enterprise-only - useBinaryPath(testServer); // Get mongocryptd in the PATH for this test let kmsServer: ReturnType; let dbname: string; + let csfleLibrary: string; before(async() => { kmsServer = makeFakeHTTPServer(fakeAWSHandlers); kmsServer.listen(0); await once(kmsServer, 'listening'); + csfleLibrary = await downloadCurrentCsfleSharedLibrary(); }); after(() => { kmsServer.close(); @@ -50,6 +56,7 @@ describe('FLE tests', () => { async function makeTestShell(): Promise { return TestShell.start({ args: [ + `--csfleLibraryPath=${csfleLibrary}`, `--awsAccessKeyId=${accessKeyId}`, `--awsSecretAccessKey=${secretAccessKey}`, `--keyVaultNamespace=${dbname}.keyVault`, @@ -140,7 +147,7 @@ describe('FLE tests', () => { it('works when a schemaMap option has been passed', async() => { const shell = TestShell.start({ - args: ['--nodb'] + args: ['--nodb', `--csfleLibraryPath=${csfleLibrary}`] }); await shell.waitForPrompt(); await shell.executeLine('local = { key: BinData(0, "kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY") }'); @@ -169,7 +176,7 @@ describe('FLE tests', () => { it('performs KeyVault data key management as expected', async() => { const shell = TestShell.start({ - args: [await testServer.connectionString()] + args: [await testServer.connectionString(), `--csfleLibraryPath=${csfleLibrary}`] }); await shell.waitForPrompt(); // Wrapper for executeLine that expects single-line output diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/exit1.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/exit1.js deleted file mode 100644 index fb011534f4..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/exit1.js +++ /dev/null @@ -1,3 +0,0 @@ -/* eslint-disable */ -'use strict'; -process.exitCode = 1; diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/weirdlog.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/weirdlog.js deleted file mode 100644 index bbccfcdde9..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/weirdlog.js +++ /dev/null @@ -1,4 +0,0 @@ -/* eslint-disable */ -'use strict'; -console.error('Diagnostic message!'); -console.log('Hello world!'); // Hard-to-parse log message. diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/withnetworking.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/withnetworking.js deleted file mode 100644 index 5232089870..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/withnetworking.js +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable */ -'use strict'; -const net = require('net'); -const fs = require('fs'); -const pidfile = process.argv[process.argv.indexOf('--pidfilepath') + 1]; - -let connections = 0; -const server = net.createServer(socket => { - connections++; - fs.writeFileSync(pidfile, JSON.stringify({ pid: process.pid, connections })); - const proxied = net.connect(+process.env.MONGOSH_TEST_PROXY_TARGET_PORT); - socket.pipe(proxied).pipe(socket); -}); -server.listen(0, () => { - console.log(`{"c":"NETWORK","id":23016,"ctx":"listener","attr":{"port":${server.address().port}}}`); -}); diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-nounix.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-nounix.js deleted file mode 100644 index 21424ec029..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-nounix.js +++ /dev/null @@ -1,7 +0,0 @@ -/* eslint-disable */ -'use strict'; -console.log(` -2021-03-29T19:35:35.244+0200 I CONTROL [initandlisten] options: {} -2021-03-29T18:48:10.769+0200 I NETWORK [listener] Listening on 127.0.0.1 -2021-03-29T18:48:10.769+0200 I NETWORK [listener] waiting for connections on port 27020 -`); diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-withunix.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-withunix.js deleted file mode 100644 index 24112969b9..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.2-withunix.js +++ /dev/null @@ -1,8 +0,0 @@ -/* eslint-disable */ -'use strict'; -console.log(` -2021-03-29T19:35:35.244+0200 I CONTROL [initandlisten] options: {} -2021-03-29T18:48:10.769+0200 I NETWORK [listener] Listening on /tmp/mongocryptd.sock -2021-03-29T18:48:10.769+0200 I NETWORK [listener] Listening on 127.0.0.1 -2021-03-29T18:48:10.769+0200 I NETWORK [listener] waiting for connections on port 27020 -`); diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-nounix.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-nounix.js deleted file mode 100644 index d59595e296..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-nounix.js +++ /dev/null @@ -1,7 +0,0 @@ -/* eslint-disable */ -'use strict'; -console.log(` -{"t":{"$date":"2021-03-29T19:34:57.800+02:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{}}} -{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"127.0.0.1"}} -{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I", "c":"NETWORK", "id":23016, "ctx":"listener","msg":"Waiting for connections","attr":{"port":27020,"ssl":"off"}} -`); diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-withunix.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-withunix.js deleted file mode 100644 index 8be68c3ec7..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/working-4.4-withunix.js +++ /dev/null @@ -1,8 +0,0 @@ -/* eslint-disable */ -'use strict'; -console.log(` -{"t":{"$date":"2021-03-29T19:34:57.800+02:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{}}} -{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"/tmp/mongocryptd.sock"}} -{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"127.0.0.1"}} -{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I", "c":"NETWORK", "id":23016, "ctx":"listener","msg":"Waiting for connections","attr":{"port":27020,"ssl":"off"}} -`); diff --git a/packages/cli-repl/test/fixtures/fake-mongocryptd/writepidfile.js b/packages/cli-repl/test/fixtures/fake-mongocryptd/writepidfile.js deleted file mode 100644 index a80bbddd2e..0000000000 --- a/packages/cli-repl/test/fixtures/fake-mongocryptd/writepidfile.js +++ /dev/null @@ -1,12 +0,0 @@ -/* eslint-disable */ -'use strict'; -const fs = require('fs'); -const pidfile = process.argv[process.argv.indexOf('--pidfilepath') + 1]; - -fs.writeFileSync(pidfile, JSON.stringify({ - pid: process.pid, - args: process.argv -})); -console.log('{"t":{"$date":"2021-03-29T18:48:32.518+02:00"},"s":"I","c":"NETWORK","id":23016,"ctx":"listener","msg":"Waiting for connections","attr":{"port":27020,"ssl":"off"}}'); - -setInterval(() => {}, 1000); diff --git a/packages/logging/src/setup-logger-and-telemetry.spec.ts b/packages/logging/src/setup-logger-and-telemetry.spec.ts index be7174f98f..a7168dd28f 100644 --- a/packages/logging/src/setup-logger-and-telemetry.spec.ts +++ b/packages/logging/src/setup-logger-and-telemetry.spec.ts @@ -68,9 +68,8 @@ describe('setupLoggerAndTelemetry', () => { bus.emit('mongosh:eval-cli-script'); bus.emit('mongosh:globalconfig-load', { filename: '/etc/mongosh.conf', found: true }); - bus.emit('mongosh:mongocryptd-tryspawn', { spawnPath: ['mongocryptd'], path: 'path' }); - bus.emit('mongosh:mongocryptd-error', { cause: 'something', error: new Error('mongocryptd error!'), stderr: 'stderr', pid: 12345 }); - bus.emit('mongosh:mongocryptd-log', { pid: 12345, logEntry: {} }); + bus.emit('mongosh:csfle-load-skip', { csflePath: 'path', reason: 'reason' }); + bus.emit('mongosh:csfle-load-found', { csflePath: 'path', expectedVersion: { versionStr: 'someversion' } }); bus.emit('mongosh-snippets:loaded', { installdir: '/' }); bus.emit('mongosh-snippets:npm-lookup', { existingVersion: 'v1.2.3' }); @@ -155,12 +154,10 @@ describe('setupLoggerAndTelemetry', () => { expect(logOutput[i++].msg).to.equal('Evaluating script passed on the command line'); expect(logOutput[i].msg).to.equal('Loading global configuration file'); expect(logOutput[i++].attr.filename).to.equal('/etc/mongosh.conf'); - expect(logOutput[i].msg).to.equal('Trying to spawn mongocryptd'); - expect(logOutput[i++].attr).to.deep.equal({ spawnPath: ['mongocryptd'], path: 'path' }); - expect(logOutput[i].msg).to.equal('Error running mongocryptd'); - expect(logOutput[i++].attr).to.deep.equal({ cause: 'something', error: 'mongocryptd error!', stderr: 'stderr', pid: 12345 }); - expect(logOutput[i].msg).to.equal('mongocryptd log message'); - expect(logOutput[i++].attr).to.deep.equal({ pid: 12345, logEntry: {} }); + expect(logOutput[i].msg).to.equal('Skipping shared library candidate'); + expect(logOutput[i++].attr).to.deep.equal({ csflePath: 'path', reason: 'reason' }); + expect(logOutput[i].msg).to.equal('Accepted shared library candidate'); + expect(logOutput[i++].attr).to.deep.equal({ csflePath: 'path', expectedVersion: 'someversion' }); expect(logOutput[i].msg).to.equal('Loaded snippets'); expect(logOutput[i++].attr).to.deep.equal({ installdir: '/' }); expect(logOutput[i].msg).to.equal('Performing npm lookup'); diff --git a/packages/logging/src/setup-logger-and-telemetry.ts b/packages/logging/src/setup-logger-and-telemetry.ts index 881d3b0dc6..27d87d6dde 100644 --- a/packages/logging/src/setup-logger-and-telemetry.ts +++ b/packages/logging/src/setup-logger-and-telemetry.ts @@ -13,9 +13,8 @@ import type { StartLoadingCliScriptsEvent, StartMongoshReplEvent, GlobalConfigFileLoadEvent, - MongocryptdTrySpawnEvent, - MongocryptdLogEvent, - MongocryptdErrorEvent, + CSFLELibrarySkipEvent, + CSFLELibraryFoundEvent, SnippetsCommandEvent, SnippetsErrorEvent, SnippetsFetchIndexErrorEvent, @@ -273,21 +272,17 @@ export function setupLoggerAndTelemetry( }); }); - bus.on('mongosh:mongocryptd-tryspawn', function(ev: MongocryptdTrySpawnEvent) { - log.info('MONGOCRYPTD', mongoLogId(1_000_000_016), 'mongocryptd', 'Trying to spawn mongocryptd', ev); + bus.on('mongosh:csfle-load-skip', function(ev: CSFLELibrarySkipEvent) { + log.info('CSFLE', mongoLogId(1_000_000_050), 'csfle', 'Skipping shared library candidate', ev); }); - bus.on('mongosh:mongocryptd-error', function(ev: MongocryptdErrorEvent) { - log.warn('MONGOCRYPTD', mongoLogId(1_000_000_017), 'mongocryptd', 'Error running mongocryptd', { - ...ev, - error: ev.error?.message + bus.on('mongosh:csfle-load-found', function(ev: CSFLELibraryFoundEvent) { + log.warn('CSFLE', mongoLogId(1_000_000_051), 'csfle', 'Accepted shared library candidate', { + csflePath: ev.csflePath, + expectedVersion: ev.expectedVersion.versionStr }); }); - bus.on('mongosh:mongocryptd-log', function(ev: MongocryptdLogEvent) { - log.info('MONGOCRYPTD', mongoLogId(1_000_000_018), 'mongocryptd', 'mongocryptd log message', ev); - }); - bus.on('mongosh-snippets:loaded', function(ev: SnippetsLoadedEvent) { log.info('MONGOSH-SNIPPETS', mongoLogId(1_000_000_019), 'snippets', 'Loaded snippets', ev); }); diff --git a/packages/shell-api/src/mongo.ts b/packages/shell-api/src/mongo.ts index 8744b5afe3..c1636182dd 100644 --- a/packages/shell-api/src/mongo.ts +++ b/packages/shell-api/src/mongo.ts @@ -192,7 +192,7 @@ export default class Mongo extends ShellApiClass { } else if (driverOptions.autoEncryption) { driverOptions.autoEncryption.extraOptions = { ...driverOptions.autoEncryption.extraOptions, - ...await this._instanceState.evaluationListener?.startMongocryptd?.() + ...await this._instanceState.evaluationListener?.getCSFLELibraryOptions?.() }; } const parentProvider = this._instanceState.initialServiceProvider; diff --git a/packages/shell-api/src/shell-instance-state.ts b/packages/shell-api/src/shell-instance-state.ts index 7a2d7d2f13..588cd0876d 100644 --- a/packages/shell-api/src/shell-instance-state.ts +++ b/packages/shell-api/src/shell-instance-state.ts @@ -98,10 +98,10 @@ export interface EvaluationListener extends Partial Promise; + getCSFLELibraryOptions?: () => Promise; } /** diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 81ea094c7c..2482d2b349 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -77,16 +77,15 @@ export interface GlobalConfigFileLoadEvent { found: boolean; } -export interface MongocryptdTrySpawnEvent { - spawnPath: string[]; - path: string; +export interface CSFLELibrarySkipEvent { + csflePath: string; + reason: string; + details?: any; } -export interface MongocryptdErrorEvent { - cause: string; - error?: Error; - stderr?: string; - pid?: number; +export interface CSFLELibraryFoundEvent { + csflePath: string; + expectedVersion: { versionStr: string }; } export interface MongocryptdLogEvent { @@ -262,17 +261,13 @@ export interface MongoshBusEventsMap extends ConnectEventMap { */ 'mongosh:eval-interrupted': () => void; /** - * Signals the start of trying to spawn a `mongocryptd` process. + * Signals that a potential CSFLE library search path was skipped. */ - 'mongosh:mongocryptd-tryspawn': (ev: MongocryptdTrySpawnEvent) => void; + 'mongosh:csfle-load-skip': (ev: CSFLELibrarySkipEvent) => void; /** - * Signals an error while interfacing with a `mongocryptd` process. + * Signals that a potential CSFLE library search path was accepted. */ - 'mongosh:mongocryptd-error': (ev: MongocryptdErrorEvent) => void; - /** - * Signals an event to be logged for a `mongocryptd` process. - */ - 'mongosh:mongocryptd-log': (ev: MongocryptdLogEvent) => void; + 'mongosh:csfle-load-found': (ev: CSFLELibraryFoundEvent) => void; /** * Signals that the CLI REPL's `close` method has completed. * _ONLY AVAILABLE FOR TESTING._ diff --git a/packaging/msi-template/README.md b/packaging/msi-template/README.md index 9ebddd5cb6..0f53a0d089 100644 --- a/packaging/msi-template/README.md +++ b/packaging/msi-template/README.md @@ -16,4 +16,4 @@ Build by executing commands: - BuildFolder: Folder containing the binaries and license notices. Defaults to "..\\..\mongosh-_Version_-dev.0-win32" # Open Issues -1. mongosh.exe and mongocryptd-mongosh.exe should have a version numbers \ No newline at end of file +1. mongosh.exe and mongosh_csfle_v1.dll should have a version numbers diff --git a/testing/integration-testing-hooks.ts b/testing/integration-testing-hooks.ts index 544ec4bc1a..6f85adef75 100644 --- a/testing/integration-testing-hooks.ts +++ b/testing/integration-testing-hooks.ts @@ -10,6 +10,7 @@ import { URL } from 'url'; import { promisify } from 'util'; import which from 'which'; import { downloadMongoDb } from '../packages/build/src/download-mongodb'; +import { downloadCsfleLibrary } from '../packages/build/src/packaging/download-csfle-library'; const execFile = promisify(child_process.execFile); @@ -389,6 +390,10 @@ export async function ensureMongodAvailable(mongodVersion = process.env.MONGOSH_ } } +export async function downloadCurrentCsfleSharedLibrary(): Promise { + return downloadCsfleLibrary('host'); +} + /** * Starts a local server unless the `MONGOSH_TEST_SERVER_URL` * environment variable is set. @@ -511,29 +516,6 @@ export function skipIfApiStrict(): void { }); } -/** - * Add the server tarball's bin/ directrory to the PATH for this section. - * This enables using e.g. mongocryptd if available. - * - * describe('...', () => { - * useBinaryPath(testServer) - * }); - */ -export function useBinaryPath(server: MongodSetup): void { - let pathBefore: string; - before(async() => { - await server.start(); - pathBefore = process.env.PATH ?? ''; - const extraPath = server.bindir; - if (extraPath !== null) { - process.env.PATH += path.delimiter + extraPath; - } - }); - after(() => { - process.env.PATH = pathBefore; - }); -} - /** * Skip tests in the suite if the test server version * (configured as environment variable or the currently installed one)