From c4bb0f03e1211ef3603a54406703a23acaf24d03 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 18 May 2022 16:08:26 +0200 Subject: [PATCH 1/2] feat(ci): create shared-openssl binaries MONGOSH-1223 --- .evergreen.yml | 25 ++++++++++ .evergreen/compile-artifact.sh | 48 ++++++++++++++++++- .evergreen/evergreen.yml.in | 25 ++++++++++ .../build/src/compile/signable-compiler.ts | 14 +++--- packages/cli-repl/src/build-info.ts | 21 ++++---- 5 files changed, 116 insertions(+), 17 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 3539505e0d..096ea61acd 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -4877,6 +4877,7 @@ functions: binary: bash .evergreen/compile-artifact.sh env: NODE_JS_VERSION: ${node_js_version} + MONGOSH_SHARED_OPENSSL: ${mongosh_shared_openssl} - command: s3.put params: aws_key: ${aws_key} @@ -9057,6 +9058,22 @@ buildvariants: executable_os_id: linux-x64 tasks: - name: compile_artifact + - name: linux_x64_build_openssl11 + display_name: "RHEL 7.0 x64 (build, shared OpenSSL 1.1)" + run_on: rhel70-build + expansions: + executable_os_id: linux-x64-openssl11 + mongosh_shared_openssl: openssl11 + tasks: + - name: compile_artifact + - name: linux_x64_build_openssl3 + display_name: "RHEL 7.0 x64 (build, shared OpenSSL 3)" + run_on: rhel70-build + expansions: + executable_os_id: linux-x64-openssl3 + mongosh_shared_openssl: openssl3 + tasks: + - name: compile_artifact - name: linux_arm64_build display_name: "Amazon 2 arm64 (build)" run_on: amazon2-arm64-small @@ -9064,6 +9081,14 @@ buildvariants: executable_os_id: linux-arm64 tasks: - name: compile_artifact + - name: linux_arm64_build_openssl11 + display_name: "Amazon 2 arm64 (build, shared OpenSSL 1.1)" + run_on: amazon2-arm64-small + expansions: + executable_os_id: linux-arm64-openssl11 + mongosh_shared_openssl: openssl11 + tasks: + - name: compile_artifact - name: linux_ppc64le_build display_name: "RHEL 8.1 PPC (build)" run_on: rhel81-power8-small diff --git a/.evergreen/compile-artifact.sh b/.evergreen/compile-artifact.sh index 0b237d7413..cc5d4d363b 100755 --- a/.evergreen/compile-artifact.sh +++ b/.evergreen/compile-artifact.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +set -x cd $(pwd) @@ -13,9 +14,52 @@ if uname -a | grep -q 'Linux.*x86_64'; then export CXX="$PWD/tmp/.sccache/sccache g++" fi +rm -rf /tmp/m && mkdir -pv /tmp/m # Node.js compilation can fail on long path prefixes +trap "rm -rf /tmp/m" EXIT +export TMP=/tmp/m +export TMPDIR=/tmp/m + +# The CI machines we have for Windows and x64 macOS are not +# able to compile OpenSSL with assembly support, +# so we revert back to the slower version. +if [ "$OS" == "Windows_NT" ]; then + export NODE_JS_CONFIGURE_ARGS='["openssl-no-asm"]' +elif uname -a | grep -q 'Darwin.*x86_64'; then + export NODE_JS_CONFIGURE_ARGS='["--openssl-no-asm"]' +elif [ -n "$MONGOSH_SHARED_OPENSSL" ]; then + pushd /tmp/m + if [ "$MONGOSH_SHARED_OPENSSL" == "openssl11" ]; then + curl -sSfLO https://www.openssl.org/source/openssl-1.1.1o.tar.gz + MONGOSH_OPENSSL_LIBNAME=:libcrypto.so.1.1,:libssl.so.1.1 + elif [ "$MONGOSH_SHARED_OPENSSL" == "openssl3" ]; then + curl -sSfLO https://www.openssl.org/source/openssl-3.0.2.tar.gz + MONGOSH_OPENSSL_LIBNAME=:libcrypto.so.3,:libssl.so.3 + else + echo "Unknown MONGOSH_SHARED_OPENSSL value: $MONGOSH_SHARED_OPENSSL" + exit 1 + fi + + tar xzvf openssl-*.tar.gz + pushd openssl-* + ./config --prefix=/tmp/m/opt --libdir=lib shared + make -j12 + make -j12 install install_ssldirs + + popd # openssl-* + popd # /tmp/m + + export NODE_JS_CONFIGURE_ARGS='[ + "--shared-openssl", + "--shared-openssl-includes=/tmp/m/opt/include", + "--shared-openssl-libpath=/tmp/m/opt/lib", + "--shared-openssl-libname='"$MONGOSH_OPENSSL_LIBNAME"'", + "--shared-zlib" + ]' + export LD_LIBRARY_PATH=/tmp/m/opt/lib +fi + export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" -mkdir -pv /tmp/m # Node.js compilation can fail on long path prefixes -(env TMP=/tmp/m TMPDIR=/tmp/m npm run evergreen-release compile && rm -rf /tmp/m) || (rm -rf /tmp/m; false) +npm run evergreen-release compile dist/mongosh --version dist/mongosh --build-info dist/mongosh --build-info | grep -q '"distributionKind": "compiled"' diff --git a/.evergreen/evergreen.yml.in b/.evergreen/evergreen.yml.in index e73854e08e..49d0b7fa02 100644 --- a/.evergreen/evergreen.yml.in +++ b/.evergreen/evergreen.yml.in @@ -365,6 +365,7 @@ functions: binary: bash .evergreen/compile-artifact.sh env: NODE_JS_VERSION: ${node_js_version} + MONGOSH_SHARED_OPENSSL: ${mongosh_shared_openssl} - command: s3.put params: aws_key: ${aws_key} @@ -965,6 +966,22 @@ buildvariants: executable_os_id: linux-x64 tasks: - name: compile_artifact + - name: linux_x64_build_openssl11 + display_name: "RHEL 7.0 x64 (build, shared OpenSSL 1.1)" + run_on: rhel70-build + expansions: + executable_os_id: linux-x64-openssl11 + mongosh_shared_openssl: openssl11 + tasks: + - name: compile_artifact + - name: linux_x64_build_openssl3 + display_name: "RHEL 7.0 x64 (build, shared OpenSSL 3)" + run_on: rhel70-build + expansions: + executable_os_id: linux-x64-openssl3 + mongosh_shared_openssl: openssl3 + tasks: + - name: compile_artifact - name: linux_arm64_build display_name: "Amazon 2 arm64 (build)" run_on: amazon2-arm64-small @@ -972,6 +989,14 @@ buildvariants: executable_os_id: linux-arm64 tasks: - name: compile_artifact + - name: linux_arm64_build_openssl11 + display_name: "Amazon 2 arm64 (build, shared OpenSSL 1.1)" + run_on: amazon2-arm64-small + expansions: + executable_os_id: linux-arm64-openssl11 + mongosh_shared_openssl: openssl11 + tasks: + - name: compile_artifact - name: linux_ppc64le_build display_name: "RHEL 8.1 PPC (build)" run_on: rhel81-power8-small diff --git a/packages/build/src/compile/signable-compiler.ts b/packages/build/src/compile/signable-compiler.ts index 551544fe87..6d0abba129 100644 --- a/packages/build/src/compile/signable-compiler.ts +++ b/packages/build/src/compile/signable-compiler.ts @@ -1,5 +1,3 @@ -/* eslint-disable no-nested-ternary */ -import os from 'os'; import { promises as fs } from 'fs'; import Module from 'module'; import pkgUp from 'pkg-up'; @@ -116,13 +114,15 @@ export class SignableCompiler { requireRegexp: /\bget_console_process_list\.node$/ } : null; + let configureArgs: string[] = []; + + if (process.env.NODE_JS_CONFIGURE_ARGS) { + configureArgs = JSON.parse(process.env.NODE_JS_CONFIGURE_ARGS); + } + // This compiles the executable along with Node from source. - // Evergreen and XCode don't have up to date libraries to compile - // open ssl with asm so we revert back to the slower version. await compileJSFileAsBinary({ - configureArgs: - os.platform() === 'win32' ? ['openssl-no-asm'] : - os.platform() === 'darwin' ? ['--openssl-no-asm'] : [], + configureArgs, sourceFile: this.sourceFile, targetFile: this.targetFile, nodeVersionRange: this.nodeVersionRange, diff --git a/packages/cli-repl/src/build-info.ts b/packages/cli-repl/src/build-info.ts index 8c10d99e6b..9e64ffb04a 100644 --- a/packages/cli-repl/src/build-info.ts +++ b/packages/cli-repl/src/build-info.ts @@ -1,14 +1,14 @@ -import os from 'os'; - export type BuildInfo = { version: string; nodeVersion: string; distributionKind: 'unpackaged' | 'packaged' | 'compiled'; - buildArch: string; - buildPlatform: string; + buildArch: typeof process['arch']; + buildPlatform: typeof process['platform']; buildTarget: string; buildTime: string | null; gitVersion: string | null; + opensslVersion: string; + sharedOpenssl: boolean; }; /** @@ -16,8 +16,13 @@ export type BuildInfo = { * in particular, when it was built and how. */ export function buildInfo(): BuildInfo { + const runtimeData = { + nodeVersion: process.version, + opensslVersion: process.versions.openssl, + sharedOpenssl: !!process.config.variables.node_shared_openssl + }; try { - const buildInfo = { ...require('./build-info.json'), nodeVersion: process.version }; + const buildInfo = { ...require('./build-info.json'), ...runtimeData }; delete buildInfo.segmentApiKey; return buildInfo; } catch { @@ -25,12 +30,12 @@ export function buildInfo(): BuildInfo { return { version, distributionKind: 'unpackaged', - buildArch: os.arch(), - buildPlatform: os.platform(), + buildArch: process.arch, + buildPlatform: process.platform, buildTarget: 'unknown', buildTime: null, gitVersion: null, - nodeVersion: process.version + ...runtimeData }; } } From 55cd18858663d83361f9381407a1b96d764f8037 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Fri, 20 May 2022 15:27:07 +0200 Subject: [PATCH 2/2] fixup: adjust e2e test for new buildInfo output --- packages/cli-repl/test/e2e.spec.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cli-repl/test/e2e.spec.ts b/packages/cli-repl/test/e2e.spec.ts index 5bd9cc7cc4..6a1917d906 100644 --- a/packages/cli-repl/test/e2e.spec.ts +++ b/packages/cli-repl/test/e2e.spec.ts @@ -39,7 +39,8 @@ describe('e2e', function() { const data = JSON.parse(shell.output); expect(Object.keys(data)).to.deep.equal([ 'version', 'distributionKind', 'buildArch', 'buildPlatform', - 'buildTarget', 'buildTime', 'gitVersion', 'nodeVersion' + 'buildTarget', 'buildTime', 'gitVersion', 'nodeVersion', + 'opensslVersion', 'sharedOpenssl' ]); expect(data.version).to.be.a('string'); expect(data.nodeVersion).to.be.a('string'); @@ -47,7 +48,8 @@ describe('e2e', function() { expect(['unpackaged', 'packaged', 'compiled'].includes(data.distributionKind)).to.be.true; expect(data.buildArch).to.be.a('string'); expect(data.buildPlatform).to.be.a('string'); - expect(data.buildTarget).to.be.a('string'); + expect(data.opensslVersion).to.be.a('string'); + expect(data.sharedOpenssl).to.be.a('boolean'); if (data.distributionKind !== 'unpackaged') { expect(data.buildTime).to.be.a('string'); expect(data.gitVersion).to.be.a('string');