diff --git a/.evergreen.yml b/.evergreen.yml index 1eb15d9c17..0129128b74 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -43,6 +43,7 @@ functions: { export NODE_JS_VERSION=${node_js_version} source .evergreen/.setup_env + npm --unsafe-perm=true run bootstrap-ci npm run check-ci } test: @@ -59,6 +60,7 @@ functions: { export NODE_JS_VERSION=${node_js_version} source .evergreen/.setup_env + npm --unsafe-perm=true run bootstrap-ci npm run test-ci echo "Creating coverage tarball..." tar cvzf coverage.tgz coverage @@ -83,6 +85,9 @@ functions: script: | set -e { + export NODE_JS_VERSION=${node_js_version} + source .evergreen/.setup_env + npm --unsafe-perm=true run bootstrap-ci (cd scripts/docker && docker build -t ubuntu18.04-xvfb -f ubuntu18.04-xvfb.Dockerfile .) docker run \ --rm -v $PWD:/tmp/build ubuntu18.04-xvfb \ @@ -102,6 +107,7 @@ functions: { export NODE_JS_VERSION=${node_js_version} source .evergreen/.setup_env + npm --unsafe-perm=true run bootstrap-ci npm run test-connectivity } @@ -536,6 +542,8 @@ tasks: vars: node_js_version: "12.20.0" - func: test_vscode + vars: + node_js_version: "12.20.0" - name: test_connectivity tags: ["extra-integration-test"] commands: @@ -573,11 +581,27 @@ tasks: - func: run_e2e_tests vars: node_js_version: "14.15.1" - - name: e2e_tests_linux + - name: e2e_tests_linux_x64 + tags: ["e2e-test"] + depends_on: + - name: compile_artifact + variant: linux_x64_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "14.15.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-x64 + - func: run_e2e_tests + vars: + node_js_version: "14.15.1" + - name: e2e_tests_linux_arm64 tags: ["e2e-test"] depends_on: - name: compile_artifact - variant: linux_build + variant: linux_arm64_build commands: - func: checkout - func: install @@ -585,7 +609,39 @@ tasks: node_js_version: "14.15.1" - func: download_compiled_artifact vars: - executable_os_id: linux + executable_os_id: linux-arm64 + - func: run_e2e_tests + vars: + node_js_version: "14.15.1" + - name: e2e_tests_linux_ppc64le + tags: ["e2e-test"] + depends_on: + - name: compile_artifact + variant: linux_ppc64le_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "14.15.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-ppc64le + - func: run_e2e_tests + vars: + node_js_version: "14.15.1" + - name: e2e_tests_linux_s390x + tags: ["e2e-test"] + depends_on: + - name: compile_artifact + variant: linux_s390x_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "14.15.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-s390x - func: run_e2e_tests vars: node_js_version: "14.15.1" @@ -633,7 +689,7 @@ tasks: - name: ".unit-test" variant: linux - name: compile_artifact - variant: linux_build + variant: linux_x64_build commands: - func: checkout - func: install @@ -650,7 +706,7 @@ tasks: - name: ".unit-test" variant: linux - name: compile_artifact - variant: linux_build + variant: linux_x64_build commands: - func: checkout - func: install @@ -667,7 +723,7 @@ tasks: - name: ".unit-test" variant: linux - name: compile_artifact - variant: linux_build + variant: linux_x64_build commands: - func: checkout - func: install @@ -909,10 +965,10 @@ buildvariants: - name: e2e_tests_darwin - name: linux - display_name: "Ubuntu 18.04" + display_name: "Ubuntu 18.04 x64" run_on: ubuntu1804-small expansions: - executable_os_id: linux + executable_os_id: linux-x64 tasks: - name: check - name: test_m40xc_n12 @@ -934,44 +990,80 @@ buildvariants: - name: package_and_upload_artifact_linux - name: package_and_upload_artifact_debian - name: package_and_upload_artifact_rhel - - name: linux_build - display_name: "Ubuntu 18.04 (build)" - run_on: ubuntu1804-build + - name: linux_x64_build + display_name: "RHEL 7.0 x64 (build)" + run_on: rhel70-build + expansions: + executable_os_id: linux-x64 + tasks: + - name: compile_artifact + - name: linux_arm64_build + display_name: "Ubuntu 18.04 arm64 (build)" + run_on: ubuntu1804-arm64-small + expansions: + executable_os_id: linux-arm64 + tasks: + - name: compile_artifact + - name: linux_ppc64le_build + display_name: "RHEL 7.1 PPC (build)" + run_on: rhel71-power8-small expansions: - executable_os_id: linux + executable_os_id: linux-ppc64le + tasks: + - name: compile_artifact + - name: linux_s390x_build + display_name: "RHEL 7.2 s390x (build)" + run_on: rhel72-zseries-small + expansions: + executable_os_id: linux-s390x tasks: - name: compile_artifact - - name: rhel70 - display_name: "RHEL 7.0 (E2E Tests)" + - name: e2e_rhel70_x64 + display_name: "RHEL 7.0 x64 (E2E Tests)" run_on: rhel70-small tasks: - - name: e2e_tests_linux - - name: rhel76 - display_name: "RHEL 7.6 (E2E Tests)" + - name: e2e_tests_linux_x64 + - name: e2e_rhel76_x64 + display_name: "RHEL 7.6 x64 (E2E Tests)" run_on: rhel76-small tasks: - - name: e2e_tests_linux - - name: ubuntu1604 - display_name: "Ubuntu 16.04 (E2E Tests)" + - name: e2e_tests_linux_x64 + - name: e2e_ubuntu1604_x64 + display_name: "Ubuntu 16.04 x64 (E2E Tests)" run_on: ubuntu1604-small tasks: - - name: e2e_tests_linux - - name: ubuntu1804 - display_name: "Ubuntu 18.04 (E2E Tests)" + - name: e2e_tests_linux_x64 + - name: e2e_ubuntu1804_x64 + display_name: "Ubuntu 18.04 x64 (E2E Tests)" run_on: ubuntu1804-small tasks: - - name: e2e_tests_linux - - name: ubuntu2004 - display_name: "Ubuntu 20.04 (E2E Tests)" + - name: e2e_tests_linux_x64 + - name: e2e_ubuntu2004_x64 + display_name: "Ubuntu 20.04 x64 (E2E Tests)" run_on: ubuntu2004-small tasks: - - name: e2e_tests_linux - - name: debian10 - display_name: "Debian 10 (E2E Tests)" + - name: e2e_tests_linux_x64 + - name: e2e_debian10_x64 + display_name: "Debian 10 x64 (E2E Tests)" run_on: debian10-small tasks: - - name: e2e_tests_linux + - name: e2e_tests_linux_x64 + - name: e2e_ubuntu1804_arm64 + display_name: "Ubuntu 18.04 arm64 (E2E Tests)" + run_on: ubuntu1804-arm64-small + tasks: + - name: e2e_tests_linux_arm64 + - name: e2e_rhel71_ppc64le + display_name: "RHEL 7.1 PPC (E2E Tests)" + run_on: rhel71-power8-small + tasks: + - name: e2e_tests_linux_ppc64le + - name: e2e_rhel72_s390x + display_name: "RHEL 7.2 s390x (E2E Tests)" + run_on: rhel72-zseries-small + tasks: + - name: e2e_tests_linux_s390x - name: win32 display_name: "Windows VS 2019" diff --git a/.evergreen/.install_node b/.evergreen/.install_node index 0c538c7477..567580679d 100644 --- a/.evergreen/.install_node +++ b/.evergreen/.install_node @@ -5,19 +5,22 @@ export BASEDIR="$PWD/.evergreen" if [ "$OS" == "Windows_NT" ]; then powershell "$(cygpath -w "$BASEDIR")"/InstallNode.ps1 else + if which realpath; then # No realpath on macOS, but also not needed there + export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink + fi export NVM_DIR="$HOME/.nvm" - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash echo "Setting NVM environment home: $NVM_DIR" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" - nvm install $NODE_JS_VERSION + nvm install --no-progress $NODE_JS_VERSION nvm alias default $NODE_JS_VERSION npm install -g npm@6 fi . "$BASEDIR/.setup_env" -npm --unsafe-perm=true run bootstrap-ci +npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/build --scope @mongosh/cli-repl --include-dependencies diff --git a/.evergreen/.setup_env b/.evergreen/.setup_env index bbf99e4d99..4407d2312d 100755 --- a/.evergreen/.setup_env +++ b/.evergreen/.setup_env @@ -1,14 +1,32 @@ set -e set -x export BASEDIR="$PWD/.evergreen" -export PATH="$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/python/3.6/bin:/opt/chefdk/gitbin:/cygdrive/c/Program Files/Git/bin:/cygdrive/c/Program Files/Git/mingw32/libexec/git-core:/cygdrive/c/Python39/Scripts:/cygdrive/c/Python39:/cygdrive/c/cmake/bin:$PATH" +export PATH="$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/python/3.6/bin:/opt/chefdk/gitbin:/cygdrive/c/Program Files/Git/bin:/cygdrive/c/Program Files/Git/mingw32/libexec/git-core:/cygdrive/c/Python39/Scripts:/cygdrive/c/Python39:/cygdrive/c/cmake/bin:/opt/mongodbtoolchain/v3/bin:$PATH" export IS_MONGOSH_EVERGREEN_CI=1 if [ "$OS" != "Windows_NT" ]; then + if which realpath; then # No realpath on macOS, but also not needed there + export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink + fi export NVM_DIR="$HOME/.nvm" echo "Setting NVM environment home: $NVM_DIR" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" nvm use $NODE_JS_VERSION + + export CC=gcc + export CXX=c++ + + echo "Using gcc version:" + gcc --version + + echo "Using g++ version:" + g++ --version +fi + +export EVERGREEN_EXPANSIONS_PATH="$BASEDIR/../../tmp/expansions.yaml" + +if [ "$OS" == "Windows_NT" ]; then + export EVERGREEN_EXPANSIONS_PATH="$(cygpath -w "$EVERGREEN_EXPANSIONS_PATH")" fi echo "Using node version:" @@ -20,8 +38,5 @@ npm --version echo "Using git version:" git --version -export EVERGREEN_EXPANSIONS_PATH="$BASEDIR/../../tmp/expansions.yaml" - -if [ "$OS" == "Windows_NT" ]; then - export EVERGREEN_EXPANSIONS_PATH="$(cygpath -w "$EVERGREEN_EXPANSIONS_PATH")" -fi +echo "Using python version:" +python --version diff --git a/.evergreen/compile-artifact.sh b/.evergreen/compile-artifact.sh index c642f546d3..1af34243e6 100755 --- a/.evergreen/compile-artifact.sh +++ b/.evergreen/compile-artifact.sh @@ -4,21 +4,37 @@ set -e cd $(pwd) source .evergreen/.setup_env -export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" -if [ "$(uname)" == Linux ]; then - mkdir -p tmp - cp "$(pwd)/../tmp/expansions.yaml" tmp/expansions.yaml - (cd scripts/docker && docker build -t centos7-build -f centos7-build.Dockerfile .) - echo Starting Docker container build - docker run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD \ - -e EVERGREEN_EXPANSIONS_PATH=/tmp/build/tmp/expansions.yaml \ - -e NODE_JS_VERSION \ - -e DISTRIBUTION_BUILD_VARIANT \ - --rm -v $PWD:/tmp/build --network host centos7-build \ - -c 'source /opt/rh/devtoolset-8/enable && cd /tmp/build && npm run evergreen-release compile && dist/mongosh --version' -else - npm run evergreen-release compile + +if uname -a | grep -q 'Linux.*x86_64'; then + rm -rf "tmp/.sccache" + mkdir -p "tmp/.sccache" + curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar -C "tmp/.sccache" -xzvf - --strip=1 sccache-0.2.13-x86_64-unknown-linux-musl/sccache + export CC="$PWD/tmp/.sccache/sccache gcc" + export CXX="$PWD/tmp/.sccache/sccache g++" fi + +if uname -p | sed -e 's/s390/matches/' -e 's/ppc/matches/' | grep -q matches; then + # On s390x and PPC machines in evergreen, there is a Python 3.x installation + # in /opt/mongodbtoolchain/v3/bin, but it's broken in the sense that its bz2 + # package cannot be loaded, which we need for building Node.js. + # We therefore do some PATH wiggling here to make sure that the python + # executable used by the Node.js configure script is python 2.x. + # This is not a sustainable long-term solution, because Node.js drops Python 2 + # support for build scripts with Node.js 15 and above. + # TODO: Open a build ticket about this, referencing patch/output from + # https://spruce.mongodb.com/version/608ad4613e8e8601d5cd3d6f/tasks. + rm -rf tmp/python-container + mkdir -p tmp/python-container + (cd tmp/python-container && ln -s /usr/bin/python python && ln -s python python2.7 && ln -s python python2) + export PATH="$PWD/tmp/python-container:$PATH" + export FORCE_PYTHON2=1 + + echo "Using python version:" + python --version +fi + +export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" +npm run evergreen-release compile dist/mongosh --version tar cvzf dist.tgz dist diff --git a/package-lock.json b/package-lock.json index 1b6a9f34b5..5da3b714b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13892,13 +13892,13 @@ } }, "mongodb-download-url": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/mongodb-download-url/-/mongodb-download-url-1.0.0.tgz", - "integrity": "sha512-bvUE9Wa0P0ZpuazQ7UFHcvE03zdnXePSsntVBngqJQIM5m6JosMeW+nrabMyBvbGqiIsgXL96UF4Bj/FakKrLw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mongodb-download-url/-/mongodb-download-url-1.0.1.tgz", + "integrity": "sha512-00qIYKoxY8O+5w7v+fJhaMs0JxCWDCk2dKbVPkQR+EKrc9UL9E5BGCQ+NxgIWd6OYQyMzjeQFOtLMsXNwPSAkQ==", "dev": true, "requires": { "debug": "^4.1.1", - "minimist": "^1.2.0", + "minimist": "^1.2.3", "node-fetch": "^2.6.1", "semver": "^7.1.1" }, diff --git a/package.json b/package.json index 06c1df5f8c..a31f30178d 100644 --- a/package.json +++ b/package.json @@ -126,7 +126,7 @@ "lerna": "^3.10.7", "mocha": "^7.1.2", "mongodb": "4.0.0-beta.3", - "mongodb-download-url": "^1.0.0", + "mongodb-download-url": "^1.0.1", "mongodb-js-precommit": "^2.0.0", "nock": "^13.0.11", "node-codesign": "^0.3.2", diff --git a/packages/cli-repl/test/e2e-direct.spec.ts b/packages/cli-repl/test/e2e-direct.spec.ts index ca0f1c6b24..1c5b5e9827 100644 --- a/packages/cli-repl/test/e2e-direct.spec.ts +++ b/packages/cli-repl/test/e2e-direct.spec.ts @@ -114,7 +114,10 @@ describe('e2e direct connection', () => { expect(await shell.executeLine('show collections')).to.include('system.version'); expect(await shell.executeLine('show dbs')).to.include('admin'); }); - it('autocompletes collection names', async() => { + it('autocompletes collection names', async function() { + if (process.arch === 's390x') { + return this.skip(); // https://jira.mongodb.org/browse/MONGOSH-746 + } const shell = TestShell.start({ args: [`${await rs1.connectionString()}/${dbname}`], forceTerminal: true }); await shell.waitForPrompt(); shell.writeInput('db.testc\u0009\u0009'); @@ -186,7 +189,10 @@ describe('e2e direct connection', () => { expect(await shell.executeLine('show collections')).to.include('system.version'); expect(await shell.executeLine('show dbs')).to.include('admin'); }); - it('autocompletes collection names', async() => { + it('autocompletes collection names', async function() { + if (process.arch === 's390x') { + return this.skip(); // https://jira.mongodb.org/browse/MONGOSH-746 + } const shell = TestShell.start({ args: [`${await rs1.connectionString()}/${dbname}`], forceTerminal: true }); await shell.waitForPrompt(); shell.writeInput('db.testc\u0009\u0009'); diff --git a/packages/cli-repl/test/e2e.spec.ts b/packages/cli-repl/test/e2e.spec.ts index 5e126a7e7c..cd5c95cccb 100644 --- a/packages/cli-repl/test/e2e.spec.ts +++ b/packages/cli-repl/test/e2e.spec.ts @@ -767,7 +767,10 @@ describe('e2e', function() { }); describe('history file', () => { - it('persists between sessions', async() => { + it('persists between sessions', async function() { + if (process.arch === 's390x') { + return this.skip(); // https://jira.mongodb.org/browse/MONGOSH-746 + } await shell.executeLine('a = 42'); shell.writeInput('.exit\n'); await shell.waitForExit(); diff --git a/scripts/docker/centos7-build.Dockerfile b/scripts/docker/centos7-build.Dockerfile deleted file mode 100644 index d62ef1ecda..0000000000 --- a/scripts/docker/centos7-build.Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM centos:7 - -RUN yum install -y centos-release-scl epel-release -RUN yum repolist -RUN yum install -y python3 devtoolset-8 cmake -RUN scl enable devtoolset-8 bash - -# Add Git from IUS repo as CentOS has git 1.8.3.1 by default (lerna requires 2.+) -RUN yum install -y https://repo.ius.io/ius-release-el7.rpm -RUN yum install -y git224 - -# Add Node.js -RUN curl -sL https://rpm.nodesource.com/setup_14.x | bash - -RUN yum install -y nodejs - -# Add sccache -RUN curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar -C /usr/local/bin -xzvf - --strip=1 sccache-0.2.13-x86_64-unknown-linux-musl/sccache - -ENV CC="sccache gcc" -ENV CXX="sccache g++" - -RUN bash -c 'source /opt/rh/devtoolset-8/enable && g++ --version && git --version' - -ENTRYPOINT [ "bash" ] diff --git a/scripts/prep-fle-addon.sh b/scripts/prep-fle-addon.sh index 6ec3227109..99960c64a3 100755 --- a/scripts/prep-fle-addon.sh +++ b/scripts/prep-fle-addon.sh @@ -34,9 +34,12 @@ echo Using libmongocrypt at git tag "$LIBMONGOCRYPT_VERSION" if [ x"$FLE_NODE_SOURCE_PATH" != x"" -a -z "$BUILD_FLE_FROM_SOURCE" ]; then # Use prebuilt binaries where available. - case `uname` in + case `uname -a` in Darwin*) PREBUILT_OSNAME=macos;; - Linux*) PREBUILT_OSNAME=rhel-70-64-bit;; + Linux*x86_64*) PREBUILT_OSNAME=rhel-70-64-bit;; + Linux*s390x*) PREBUILT_OSNAME=rhel72-zseries-test;; + Linux*aarch64*) PREBUILT_OSNAME=ubuntu1804-arm64;; + Linux*ppc64le*) PREBUILT_OSNAME=rhel-71-ppc64el;; CYGWIN*|MINGW32*|MSYS*|MINGW*) PREBUILT_OSNAME=windows-test;; esac fi diff --git a/testing/integration-testing-hooks.ts b/testing/integration-testing-hooks.ts index eabeb3366b..c2e60d1c1b 100644 --- a/testing/integration-testing-hooks.ts +++ b/testing/integration-testing-hooks.ts @@ -53,28 +53,10 @@ async function tryExtensions(base: string): Promise<[ string, Error ]> { return [ '', lastErr ]; } -// Get the path we use to spawn mlaunch, and potential environment variables -// necessary to run it successfully. This tries to install it locally if it -// cannot find an existing installation. -let mlaunchPath: { exec: string, env: Record } | undefined; -export async function getMlaunchPath(): Promise<{ exec: string, env: Record }> { - const tmpdir = await getTmpdir(); - if (mlaunchPath !== undefined) { - return mlaunchPath; - } - - try { - // If `mlaunch` is already in the PATH: Great, we're done. - return mlaunchPath = { exec: await which('mlaunch'), env: {} }; - } catch { - ciLog('Did not find mlaunch in PATH'); - } - - // Figure out where python3 might live (python3, python, $PYTHON). - let python = ''; +async function findPython3() { try { await which('python3'); - python = 'python3'; + return 'python3'; } catch { ciLog('Did not find python3 in PATH'); try { @@ -82,7 +64,7 @@ export async function getMlaunchPath(): Promise<{ exec: string, env: Record } | undefined; +export async function getMlaunchPath(): Promise<{ exec: string[], env: Record }> { + const tmpdir = await getTmpdir(); + if (mlaunchPath !== undefined) { + return mlaunchPath; } + try { + // If `mlaunch` is already in the PATH: Great, we're done. + const mlaunchBinary = await which('mlaunch'); + const filehandle: any = await fs.open(mlaunchBinary, 'r'); + try { + const startOfMlaunchFile = (await filehandle.read({ length: 100 })).buffer.toString('utf8').trim(); + console.log(startOfMlaunchFile) + if (startOfMlaunchFile.match(/^#!(\S+)python3?\r?\n/)) { + return mlaunchPath = { exec: [ await findPython3(), mlaunchBinary ], env: {} }; + } + } finally { + await filehandle.close(); + } + return mlaunchPath = { exec: [ mlaunchBinary ], env: {} }; + } catch { + ciLog('Did not find mlaunch in PATH'); + } + + // Figure out where python3 might live (python3, python, $PYTHON). + const python = await findPython3(); + // Install mlaunch, preferably locally and otherwise attempt to do so globally. try { const mlaunchPy = path.join(tmpdir, 'bin', 'mlaunch'); let [ exec ] = await tryExtensions(mlaunchPy); if (exec) { - return mlaunchPath = { exec, env: { PYTHONPATH: tmpdir } }; + return mlaunchPath = { exec: [ python, exec ], env: { PYTHONPATH: tmpdir } }; } ciLog('Trying to install mlaunch in ', tmpdir); await execFile('pip3', ['install', '--target', tmpdir, 'mtools[mlaunch]']); ciLog('Installation complete'); [ exec ] = await tryExtensions(mlaunchPy); if (exec) { - return mlaunchPath = { exec, env: { PYTHONPATH: tmpdir } }; + return mlaunchPath = { exec: [ python, exec ], env: { PYTHONPATH: tmpdir } }; } } catch {} @@ -125,7 +137,7 @@ export async function getMlaunchPath(): Promise<{ exec: string, env: Record