feat: add csfle shared library MONGOSH-949

.evergreen.yml

@@ -5033,6 +5033,8 @@ functions:
           set -e
           set -x
           {
+          export NODE_JS_VERSION=${node_js_version}
+          . .evergreen/setup-env.sh
           . preload.sh
           ./scripts/docker/build.sh ${dockerfile}
           ./scripts/docker/run.sh ${dockerfile} --smokeTests
@@ -8394,8 +8396,13 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: ubuntu18.04-deb
   - name: pkg_test_docker_ubuntu20_04_deb
     tags: ["smoke-test"]
@@ -8408,8 +8415,13 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: ubuntu20.04-deb
   - name: pkg_test_docker_debian9_deb
     tags: ["smoke-test"]
@@ -8422,8 +8434,13 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: debian9-deb
   - name: pkg_test_docker_debian10_deb
     tags: ["smoke-test"]
@@ -8436,8 +8453,13 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: debian10-deb
   - name: pkg_test_docker_debian11_deb
     tags: ["smoke-test"]
@@ -8450,8 +8472,13 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: debian11-deb
   - name: pkg_test_docker_centos7_rpm
     tags: ["smoke-test"]
@@ -8464,8 +8491,13 @@ tasks:
         vars:
           source_distribution_build_variant: rhel7-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: centos7-rpm
   - name: pkg_test_docker_amazonlinux2_rpm
     tags: ["smoke-test"]
@@ -8478,8 +8510,13 @@ tasks:
         vars:
           source_distribution_build_variant: rhel7-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: amazonlinux2-rpm
   - name: pkg_test_docker_rocky8_rpm
     tags: ["smoke-test"]
@@ -8492,8 +8529,13 @@ tasks:
         vars:
           source_distribution_build_variant: rhel8-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: rocky8-rpm
   - name: pkg_test_docker_fedora34_rpm
     tags: ["smoke-test"]
@@ -8506,8 +8548,13 @@ tasks:
         vars:
           source_distribution_build_variant: rhel8-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: fedora34-rpm
   - name: pkg_test_docker_suse12_rpm
     tags: ["smoke-test"]
@@ -8520,8 +8567,13 @@ tasks:
         vars:
           source_distribution_build_variant: suse-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: suse12-rpm
   - name: pkg_test_docker_suse15_rpm
     tags: ["smoke-test"]
@@ -8534,8 +8586,13 @@ tasks:
         vars:
           source_distribution_build_variant: suse-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: suse15-rpm
   - name: pkg_test_docker_amazonlinux1_rpm
     tags: ["smoke-test"]
@@ -8548,8 +8605,13 @@ tasks:
         vars:
           source_distribution_build_variant: amzn1-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.19.1"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "14.19.1"
           dockerfile: amazonlinux1-rpm
   - name: pkg_test_debextract_debian_arm64
     tags: ["smoke-test"]

.evergreen/evergreen.yml.in

@@ -521,6 +521,8 @@ functions:
           set -e
           set -x
           {
+          export NODE_JS_VERSION=${node_js_version}
+          . .evergreen/setup-env.sh
           . preload.sh
           ./scripts/docker/build.sh ${dockerfile}
           ./scripts/docker/run.sh ${dockerfile} --smokeTests
@@ -819,8 +821,13 @@ tasks:
           preload_script_path: preload.sh
     <% }; break;
        case 'docker': { %>
+      - func: install
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_14) %>"
+          npm_deps_mode: cli_build
       - func: test_artifact_docker
         vars:
+          node_js_version: "<% out(NODE_JS_VERSION_14) %>"
           dockerfile: <% out(dockerfile) %>
     <% }; break;
     // We don't have docker for platforms other than x64, so for those we just

config/build.conf.js

@@ -3,6 +3,10 @@
 const path = require('path');
 const os = require('os');
 
+const SHARED_LIBRARY_SUFFIX =
+  process.platform === 'win32' ? 'dll' :
+  process.platform === 'darwin' ? 'dylib' : 'so';
+
 /**
  * The project root.
  */
@@ -45,11 +49,11 @@ const OUTPUT_DIR = path.join(ROOT, 'dist');
 const EXECUTABLE_PATH = path.join(OUTPUT_DIR, process.platform === 'win32' ? 'mongosh.exe' : 'mongosh');
 
 /**
- * The name of the downloaded mongocryptd executable.
- * We use the name mongocryptd-mongosh to avoid conflicts with users
- * potentially installing the 'proper' mongocryptd package.
+ * The path to the downloaded csfe shared library.
+ * We use the name mongosh_csfle_v1 to avoid conflicts with users
+ * potentially installing the 'proper' csfle shared library.
  */
-const MONGOCRYPTD_PATH = path.resolve(TMP_DIR, 'mongocryptd-mongosh' + (process.platform === 'win32' ? '.exe' : ''));
+const CSFLE_LIBRARY_PATH = path.resolve(TMP_DIR, 'mongosh_csfle_v1.' + SHARED_LIBRARY_SUFFIX);
 
 /**
  * Build info JSON data file.
@@ -105,7 +109,7 @@ module.exports = {
     repo: 'mongosh'
   },
   artifactUrlFile: process.env.ARTIFACT_URL_FILE,
-  mongocryptdPath: MONGOCRYPTD_PATH,
+  csfleLibraryPath: CSFLE_LIBRARY_PATH,
   packageInformation: {
     binaries: [
       {
@@ -120,11 +124,11 @@ module.exports = {
         }
       },
       {
-        sourceFilePath: MONGOCRYPTD_PATH,
-        category: 'libexec',
+        sourceFilePath: CSFLE_LIBRARY_PATH,
+        category: 'lib',
         license: {
-          sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-mongocryptd'),
-          packagedFilePath: 'LICENSE-mongocryptd',
+          sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-csfle'),
+          packagedFilePath: 'LICENSE-csfle',
           debCopyright: COPYRIGHT,
           debIdentifier: 'Proprietary',
           rpmIdentifier: 'Proprietary'

lerna.json

@@ -1,6 +1,7 @@
 {
   "packages": [
-    "packages/*"
+    "packages/*",
+    "scripts/docker"
   ],
   "version": "0.0.0-dev.0"
 }

package.json

@@ -124,7 +124,7 @@
     "lerna": "^4.0.0",
     "mocha": "^7.1.2",
     "mongodb": "^4.6.0",
-    "mongodb-download-url": "^1.1.2",
+    "mongodb-download-url": "^1.2.0",
     "mongodb-js-precommit": "^2.0.0",
     "nock": "^13.0.11",
     "node-codesign": "^0.3.3",

packages/arg-parser/src/arg-mapper.ts

@@ -32,6 +32,16 @@ function setAutoEncrypt<Key extends keyof AutoEncryptionOptions>(
   return setDriver(i, 'autoEncryption', autoEncryption);
 }
 
+type AutoEncryptionExtraOptions = NonNullable<AutoEncryptionOptions['extraOptions']>;
+function setAutoEncryptExtra<Key extends keyof AutoEncryptionExtraOptions>(
+  i: Readonly<ConnectionInfo>,
+  key: Key,
+  value: AutoEncryptionExtraOptions[Key]): ConnectionInfo {
+  const extraOptions = i.driverOptions.autoEncryption?.extraOptions ?? {};
+  extraOptions[key] = value;
+  return setAutoEncrypt(i, 'extraOptions', extraOptions);
+}
+
 type AWSKMSOptions = NonNullable<NonNullable<AutoEncryptionOptions['kmsProviders']>['aws']>;
 function setAWSKMS<Key extends keyof AWSKMSOptions>(
   i: Readonly<ConnectionInfo>,
@@ -95,6 +105,7 @@ const MAPPINGS: {
   awsSecretAccessKey: (i, v) => setAWSKMS(i, 'secretAccessKey', v),
   awsSessionToken: (i, v) => setAWSKMS(i, 'sessionToken', v),
   awsIamSessionToken: (i, v) => setAuthMechProp(i, 'AWS_SESSION_TOKEN', v),
+  csfleLibraryPath: (i, v) => setAutoEncryptExtra(i, 'csflePath', v),
   gssapiServiceName: (i, v) => setAuthMechProp(i, 'SERVICE_NAME', v),
   sspiRealmOverride: (i, v) => setAuthMechProp(i, 'SERVICE_REALM', v),
   sspiHostnameCanonicalization:

packages/arg-parser/src/cli-options.ts

@@ -16,6 +16,7 @@ export interface CliOptions {
   awsIamSessionToken?: string;
   awsSecretAccessKey?: string;
   awsSessionToken?: string;
+  csfleLibraryPath?: string;
   db?: string;
   eval?: string;
   gssapiServiceName?: string;

packages/build/src/compile/signable-compiler.ts

@@ -73,6 +73,10 @@ export class SignableCompiler {
       path: await findModulePath('service-provider-server', 'os-dns-native'),
       requireRegexp: /\bos_dns_native\.node$/
     };
+    const csfleLibraryVersionAddon = {
+      path: await findModulePath('cli-repl', 'mongodb-csfle-library-version'),
+      requireRegexp: /\bmongodb_csfle_library_version\.node$/
+    };
     // Warning! Until https://jira.mongodb.org/browse/MONGOSH-990,
     // packages/service-provider-server *also* has a copy of these.
     // We use the versions included in packages/cli-repl here, so these
@@ -110,7 +114,8 @@ export class SignableCompiler {
       addons: [
         fleAddon,
         osDnsAddon,
-        kerberosAddon
+        kerberosAddon,
+        csfleLibraryVersionAddon
       ].concat(winCAAddon ? [
         winCAAddon
       ] : []).concat(winConsoleProcessListAddon ? [

packages/build/src/config/config.ts

@@ -40,7 +40,7 @@ export interface Config {
   };
   isPatch?: boolean;
   triggeringGitTag?: string;
-  mongocryptdPath: string;
+  csfleLibraryPath: string;
   packageInformation?: PackageInformation;
   artifactUrlFile?: string;
   manpage?: ManPageConfig;

packages/build/src/config/redact-config.ts

@@ -17,7 +17,7 @@ export function redactConfig(config: Config): Partial<Config> {
     repo: config.repo,
     isPatch: config.isPatch,
     packageInformation: config.packageInformation,
-    mongocryptdPath: config.mongocryptdPath,
+    csfleLibraryPath: config.csfleLibraryPath,
     artifactUrlFile: config.artifactUrlFile
   };
 }