feat: add packaging for CSFLE shared library MONGOSH-1116

.evergreen.yml

@@ -7937,6 +7937,9 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: ubuntu18.04-deb
@@ -7951,6 +7954,9 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: ubuntu20.04-deb
@@ -7965,6 +7971,9 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: debian9-deb
@@ -7979,6 +7988,9 @@ tasks:
         vars:
           source_distribution_build_variant: debian-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: debian10-deb
@@ -7993,6 +8005,9 @@ tasks:
         vars:
           source_distribution_build_variant: rhel7-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: centos7-rpm
@@ -8007,6 +8022,9 @@ tasks:
         vars:
           source_distribution_build_variant: rhel7-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: amazonlinux2-rpm
@@ -8021,6 +8039,9 @@ tasks:
         vars:
           source_distribution_build_variant: rhel8-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: rocky8-rpm
@@ -8035,6 +8056,9 @@ tasks:
         vars:
           source_distribution_build_variant: rhel8-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: fedora34-rpm
@@ -8049,6 +8073,9 @@ tasks:
         vars:
           source_distribution_build_variant: suse-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: suse12-rpm
@@ -8063,6 +8090,9 @@ tasks:
         vars:
           source_distribution_build_variant: suse-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: suse15-rpm
@@ -8077,6 +8107,9 @@ tasks:
         vars:
           source_distribution_build_variant: amzn1-x64
       - func: write_preload_script
+      - func: install
+        vars:
+          node_js_version: "14.18.3"
       - func: test_artifact_docker
         vars:
           dockerfile: amazonlinux1-rpm

.evergreen/evergreen.yml.in

@@ -793,6 +793,9 @@ tasks:
           preload_script_path: preload.sh
     <% }; break;
        case 'docker': { %>
+      - func: install
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_14) %>"
       - func: test_artifact_docker
         vars:
           dockerfile: <% out(dockerfile) %>

config/build.conf.js

@@ -3,6 +3,10 @@
 const path = require('path');
 const os = require('os');
 
+const SHARED_LIBRARY_SUFFIX =
+  process.platform === 'win32' ? 'dll' :
+  process.platform === 'darwin' ? 'dylib' : 'so';
+
 /**
  * The project root.
  */
@@ -45,11 +49,11 @@ const OUTPUT_DIR = path.join(ROOT, 'dist');
 const EXECUTABLE_PATH = path.join(OUTPUT_DIR, process.platform === 'win32' ? 'mongosh.exe' : 'mongosh');
 
 /**
- * The name of the downloaded mongocryptd executable.
- * We use the name mongocryptd-mongosh to avoid conflicts with users
- * potentially installing the 'proper' mongocryptd package.
+ * The path to the downloaded csfe shared library.
+ * We use the name mongosh_csfle_v1 to avoid conflicts with users
+ * potentially installing the 'proper' csfle shared library.
  */
-const MONGOCRYPTD_PATH = path.resolve(TMP_DIR, 'mongocryptd-mongosh' + (process.platform === 'win32' ? '.exe' : ''));
+const CSFLE_LIBRARY_PATH = path.resolve(TMP_DIR, 'mongosh_csfle_v1.' + SHARED_LIBRARY_SUFFIX);
 
 /**
  * Build info JSON data file.
@@ -105,7 +109,7 @@ module.exports = {
     repo: 'mongosh'
   },
   artifactUrlFile: process.env.ARTIFACT_URL_FILE,
-  mongocryptdPath: MONGOCRYPTD_PATH,
+  csfleLibraryPath: CSFLE_LIBRARY_PATH,
   packageInformation: {
     binaries: [
       {
@@ -120,11 +124,11 @@ module.exports = {
         }
       },
       {
-        sourceFilePath: MONGOCRYPTD_PATH,
-        category: 'libexec',
+        sourceFilePath: CSFLE_LIBRARY_PATH,
+        category: 'lib',
         license: {
-          sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-mongocryptd'),
-          packagedFilePath: 'LICENSE-mongocryptd',
+          sourceFilePath: path.resolve(__dirname, '..', 'packaging', 'LICENSE-csfle'),
+          packagedFilePath: 'LICENSE-csfle',
           debCopyright: COPYRIGHT,
           debIdentifier: 'Proprietary',
           rpmIdentifier: 'Proprietary'

lerna.json

@@ -1,6 +1,7 @@
 {
   "packages": [
-    "packages/*"
+    "packages/*",
+    "scripts/docker"
   ],
   "version": "0.0.0-dev.0"
 }

package.json

@@ -123,7 +123,7 @@
     "lerna": "^4.0.0",
     "mocha": "^7.1.2",
     "mongodb": "^4.4.0",
-    "mongodb-download-url": "^1.1.2",
+    "mongodb-download-url": "^1.2.0",
     "mongodb-js-precommit": "^2.0.0",
     "nock": "^13.0.11",
     "node-codesign": "^0.3.3",

packages/build/src/config/config.ts

@@ -40,7 +40,7 @@ export interface Config {
   };
   isPatch?: boolean;
   triggeringGitTag?: string;
-  mongocryptdPath: string;
+  csfleLibraryPath: string;
   packageInformation?: PackageInformation;
   artifactUrlFile?: string;
   manpage?: ManPageConfig;

packages/build/src/config/redact-config.ts

@@ -17,7 +17,7 @@ export function redactConfig(config: Config): Partial<Config> {
     repo: config.repo,
     isPatch: config.isPatch,
     packageInformation: config.packageInformation,
-    mongocryptdPath: config.mongocryptdPath,
+    csfleLibraryPath: config.csfleLibraryPath,
     artifactUrlFile: config.artifactUrlFile
   };
 }

packages/build/src/download-mongodb.ts

@@ -1,8 +1,12 @@
 /* eslint-disable no-return-assign, no-empty */
 /* istanbul ignore file */
+import fetch from 'node-fetch';
+import tar from 'tar';
+import { promisify } from 'util';
 import { promises as fs } from 'fs';
 import path from 'path';
 import download from 'download';
+import { pipeline } from 'stream';
 import getDownloadURL from 'mongodb-download-url';
 import type { Options as DownloadOptions } from 'mongodb-download-url';
 
@@ -21,7 +25,7 @@ export async function downloadMongoDb(tmpdir: string, targetVersionSemverSpecifi
 
   await fs.mkdir(tmpdir, { recursive: true });
   if (targetVersionSemverSpecifier === 'latest-alpha') {
-    return await doDownload(tmpdir, 'latest-alpha', lookupDownloadUrl);
+    return await doDownload(tmpdir, !!options.csfle, 'latest-alpha', lookupDownloadUrl);
   }
 
   if (/-community$/.test(targetVersionSemverSpecifier)) {
@@ -31,28 +35,44 @@ export async function downloadMongoDb(tmpdir: string, targetVersionSemverSpecifi
 
   return await doDownload(
     tmpdir,
+    !!options.csfle,
     targetVersionSemverSpecifier + (wantsEnterprise ? '-enterprise' : '-community'),
     () => lookupDownloadUrl());
 }
 
 const downloadPromises: Record<string, Promise<string>> = {};
-async function doDownload(tmpdir: string, version: string, lookupDownloadUrl: () => Promise<string>) {
+async function doDownload(
+  tmpdir: string,
+  isCsfle: boolean,
+  version: string,
+  lookupDownloadUrl: () => Promise<string>) {
   const downloadTarget = path.resolve(
     tmpdir,
     `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${process.arch}-${version}`
       .replace(/[^a-zA-Z0-9_-]/g, ''));
   return downloadPromises[downloadTarget] ??= (async() => {
-    const bindir = path.resolve(downloadTarget, 'bin');
+    const bindir = path.resolve(downloadTarget, isCsfle ? 'lib' : 'bin');
     try {
       await fs.stat(bindir);
       console.info(`Skipping download because ${downloadTarget} exists`);
       return bindir;
     } catch {}
 
     await fs.mkdir(downloadTarget, { recursive: true });
-    const downloadInfo = await lookupDownloadUrl();
-    console.info('Downloading...', downloadInfo);
-    await download(downloadInfo, downloadTarget, { extract: true, strip: 1 });
+    const url = await lookupDownloadUrl();
+    console.info('Downloading...', url);
+
+    if (url.match(/\.tgz$|\.tar(\.[^.]+)?$/)) {
+      // the server's tarballs can contain hard links, which the (unmaintained?)
+      // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93)
+      const response = await fetch(url);
+      await promisify(pipeline)(
+        response.body,
+        tar.x({ cwd: downloadTarget, strip: isCsfle ? 0 : 1 })
+      );
+    } else {
+      await download(url, downloadTarget, { extract: true, strip: 1 });
+    }
 
     await fs.stat(bindir); // Make sure it exists.
     return bindir;

packages/build/src/packaging/download-mongocryptd.ts → packages/build/src/packaging/download-csfle-library.ts

@@ -4,26 +4,41 @@ import { promises as fs, constants as fsConstants } from 'fs';
 import { downloadMongoDb, DownloadOptions } from '../download-mongodb';
 import { BuildVariant, getDistro, getArch } from '../config';
 
-export async function downloadMongocrypt(variant: BuildVariant): Promise<string> {
+export async function downloadCsfleLibrary(variant: BuildVariant): Promise<string> {
   const opts: DownloadOptions = {};
   opts.arch = getArch(variant);
   opts.distro = lookupReleaseDistro(variant);
   opts.enterprise = true;
-  opts.cryptd = true;
-  console.info('mongosh: downloading latest mongocryptd for inclusion in package:', JSON.stringify(opts));
+  opts.csfle = true;
+  console.info('mongosh: downloading latest csfle shared library for inclusion in package:', JSON.stringify(opts));
 
-  const bindir = await downloadMongoDb(
-    path.resolve(__dirname, '..', '..', '..', '..', 'tmp', 'mongocryptd-store', variant),
-    '*',
-    opts); // Download mongodb for latest server version.
-  let mongocryptd = path.join(bindir, 'mongocryptd');
+  let libdir = '';
+  const csfleTmpTargetDir = path.resolve(__dirname, '..', '..', '..', '..', 'tmp', 'csfle-store', variant);
+  // Download mongodb for latest server version. Since the CSFLE shared
+  // library is not part of a non-rc release yet and 5.3.0 not released yet, try:
+  // 1. release server version, 2. '5.3.0' specifically, 3. any version at all
+  let error: Error | undefined;
+  for (const version of [ 'stable', '5.3.0', 'unstable' ]) {
+    try {
+      libdir = await downloadMongoDb(csfleTmpTargetDir, version, opts);
+      break;
+    } catch (e: any) {
+      error = e;
+    }
+  }
+  if (!libdir) throw error;
+  let csfleLibrary = path.join(libdir, 'mongo_csfle_v1');
   if (opts.distro === 'win32') {
-    mongocryptd += '.exe';
+    csfleLibrary += '.dll';
+  } else if (opts.distro === 'darwin') {
+    csfleLibrary += '.dylib';
+  } else {
+    csfleLibrary += '.so';
   }
-  // Make sure that the binary exists and is executable.
-  await fs.access(mongocryptd, fsConstants.X_OK);
-  console.info('mongosh: downloaded', mongocryptd);
-  return mongocryptd;
+  // Make sure that the binary exists and is readable.
+  await fs.access(csfleLibrary, fsConstants.R_OK);
+  console.info('mongosh: downloaded', csfleLibrary);
+  return csfleLibrary;
 }
 
 // eslint-disable-next-line complexity

packages/build/src/packaging/package/debian.spec.ts

@@ -25,7 +25,7 @@ describe('tarball debian', () => {
     {
       const { stdout } = await execFile('dpkg', ['-c', tarball.path]);
       expect(stdout).to.match(/^-rwxr.xr-x.+\/usr\/bin\/foo$/m);
-      expect(stdout).to.match(/^-rwxr.xr-x.+\/usr\/libexec\/bar$/m);
+      expect(stdout).to.match(/^-rwxr.xr-x.+\/usr\/lib\/bar$/m);
       expect(stdout).to.match(/^-rw-r.-r--.+\/usr\/share\/doc\/foobar\/LICENSE_bar$/m);
       expect(stdout).to.match(/^-rw-r.-r--.+\/usr\/share\/doc\/foobar\/LICENSE_foo$/m);
       expect(stdout).to.match(/^-rw-r.-r--.+\/usr\/share\/doc\/foobar\/README$/m);

packages/build/src/packaging/package/package-information.ts

@@ -15,7 +15,7 @@ type ManPage = DocumentationFile;
 export interface PackageInformation {
     binaries: {
         sourceFilePath: string;
-        category: 'bin' | 'libexec';
+        category: 'bin' | 'lib';
         license: LicenseInformation;
     }[];
     otherDocFilePaths: DocumentationFile[];

packages/build/src/packaging/package/redhat.spec.ts

@@ -34,7 +34,7 @@ describe('tarball redhat', () => {
     expect(stdout).to.match(/URL\s+:\s+https:\/\/example.org/);
     expect(stdout).to.match(/Summary\s+:\s+Dummy package/);
     expect(stdout).to.match(/^\/usr\/bin\/foo$/m);
-    expect(stdout).to.match(/^\/usr\/libexec\/bar$/m);
+    expect(stdout).to.match(/^\/usr\/lib\/bar$/m);
     expect(stdout).to.match(/^\/usr\/share\/doc\/foobar-1.0.0\/README$/m);
     expect(stdout).to.match(/^\/usr\/share\/licenses\/foobar-1.0.0\/LICENSE_bar$/m);
     expect(stdout).to.match(/^\/usr\/share\/licenses\/foobar-1.0.0\/LICENSE_foo$/m);