diff --git a/.evergreen.yml b/.evergreen.yml index ba7d4eda70..39c426f4cf 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -5,7 +5,8 @@ exec_timeout_secs: 3600 # # Current functions: # checkout - Checks out the project from git. -# install - Installs Node and all dependencies. +# compile_ts - Installs Node and all dependencies, and run all prepublish scripts. +# install - Installs Node and all dependencies, and download the result of running all prepublish scripts. # check - Performs linter and dependency checks. # test - Runs all tests. # test_vscode - Clones the vscode extension repository and runs its tests. @@ -22,6 +23,30 @@ functions: - command: git.get_project params: directory: src + compile_ts: + - command: expansions.write + params: + file: tmp/expansions.yaml + redacted: true + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + export NODE_JS_VERSION=${node_js_version} + source .evergreen/install-node.sh + npm run evergreen-release bump + npm run bootstrap-ci + tar cvzf compiled-ts.tgz packages/*/{lib,dist} + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/compiled-ts.tgz + remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz + bucket: mciuploads + permissions: public-read + content_type: application/x-gzip install: - command: expansions.write params: @@ -33,8 +58,21 @@ functions: shell: bash script: | export NODE_JS_VERSION=${node_js_version} - source .evergreen/.install_node + source .evergreen/install-node.sh npm run evergreen-release bump + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/compiled-ts.tgz + remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz + bucket: mciuploads + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + tar xvzf compiled-ts.tgz check: - command: shell.exec params: @@ -44,8 +82,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run check-ci } test: @@ -61,8 +99,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run test-ci echo "Creating coverage tarball..." tar cvzf coverage.tgz coverage @@ -88,8 +126,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish (cd scripts/docker && docker build -t ubuntu18.04-xvfb -f ubuntu18.04-xvfb.Dockerfile .) docker run \ --rm -v $PWD:/tmp/build ubuntu18.04-xvfb \ @@ -108,8 +146,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run test-connectivity } test_apistrict: @@ -125,7 +163,7 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh npm run test-apistrict-ci } @@ -364,7 +402,7 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" npm run evergreen-release draft } @@ -382,11 +420,11 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" echo "registry=https://registry.npmjs.org/" > .npmrc echo "//registry.npmjs.org/:_authToken=${devtoolsbot_npm_token}" >> .npmrc - npm --unsafe-perm=true run bootstrap-ci + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run evergreen-release publish } @@ -394,6 +432,7 @@ functions: # pass or fail. # # Current tasks: +# compile_ts - Do the initial compilation of TS sources. # check - Performs linter and dependency checks. # test_{version} - Runs all tests, against a specified mongod version. # test_vscode - Run the vscode extension integration tests. @@ -405,7 +444,17 @@ functions: # release_publish - Publishes the npm packages and uploads the tarballs. # pkg_test_* - Run tests on the release packages tasks: + - name: compile_ts + commands: + - func: checkout + - func: compile_ts + vars: + node_js_version: "14.17.1" + - name: check + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -421,6 +470,9 @@ tasks: ### - name: test_m40xc_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -433,6 +485,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m40xe_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -445,6 +500,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m42xc_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -457,6 +515,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m42xe_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -469,6 +530,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m44xc_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -481,6 +545,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m44xe_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -493,6 +560,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m50xc_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -505,6 +575,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m50xe_n12 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -517,6 +590,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_mlatest_n12 tags: ["unit-test","mlatest"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -529,6 +605,9 @@ tasks: mongosh_skip_node_version_check: "1" - name: test_m40xc_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -541,6 +620,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m40xe_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -553,6 +635,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m42xc_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -565,6 +650,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m42xe_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -577,6 +665,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m44xc_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -589,6 +680,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m44xe_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -601,6 +695,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m50xc_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -613,6 +710,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_m50xe_n14 tags: ["unit-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -625,6 +725,9 @@ tasks: mongosh_skip_node_version_check: "" - name: test_mlatest_n14 tags: ["unit-test","mlatest"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -641,6 +744,9 @@ tasks: ### - name: test_vscode tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -651,6 +757,9 @@ tasks: node_js_version: "12.20.0" - name: test_connectivity tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -659,6 +768,9 @@ tasks: - func: test_connectivity - name: test_apistrict tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -670,6 +782,9 @@ tasks: mongosh_server_test_version: "latest-alpha" mongosh_test_force_api_strict: "1" - name: compile_artifact + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -1287,6 +1402,9 @@ tasks: tags: ["publish"] git_tag_only: true exec_timeout_secs: 7200 + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -1345,6 +1463,7 @@ buildvariants: display_name: "Ubuntu 18.04 x64" run_on: ubuntu1804-small tasks: + - name: compile_ts - name: check - name: test_m40xc_n12 - name: test_m40xe_n12 diff --git a/.evergreen/compile-artifact.sh b/.evergreen/compile-artifact.sh index 9e5e411b92..f21bd731e7 100755 --- a/.evergreen/compile-artifact.sh +++ b/.evergreen/compile-artifact.sh @@ -3,7 +3,7 @@ set -e cd $(pwd) -source .evergreen/.setup_env +source .evergreen/setup-env.sh if uname -a | grep -q 'Linux.*x86_64'; then rm -rf "tmp/.sccache" diff --git a/.evergreen/evergreen.yml.in b/.evergreen/evergreen.yml.in index 113fcf5195..5a916ca322 100644 --- a/.evergreen/evergreen.yml.in +++ b/.evergreen/evergreen.yml.in @@ -88,7 +88,8 @@ exec_timeout_secs: 3600 # # Current functions: # checkout - Checks out the project from git. -# install - Installs Node and all dependencies. +# compile_ts - Installs Node and all dependencies, and run all prepublish scripts. +# install - Installs Node and all dependencies, and download the result of running all prepublish scripts. # check - Performs linter and dependency checks. # test - Runs all tests. # test_vscode - Clones the vscode extension repository and runs its tests. @@ -105,6 +106,30 @@ functions: - command: git.get_project params: directory: src + compile_ts: + - command: expansions.write + params: + file: tmp/expansions.yaml + redacted: true + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + export NODE_JS_VERSION=${node_js_version} + source .evergreen/install-node.sh + npm run evergreen-release bump + npm run bootstrap-ci + tar cvzf compiled-ts.tgz packages/*/{lib,dist} + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/compiled-ts.tgz + remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz + bucket: mciuploads + permissions: public-read + content_type: application/x-gzip install: - command: expansions.write params: @@ -116,8 +141,21 @@ functions: shell: bash script: | export NODE_JS_VERSION=${node_js_version} - source .evergreen/.install_node + source .evergreen/install-node.sh npm run evergreen-release bump + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/compiled-ts.tgz + remote_file: mongosh/binaries/${revision}/${revision_order_id}/compiled-ts.tgz + bucket: mciuploads + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + tar xvzf compiled-ts.tgz check: - command: shell.exec params: @@ -127,8 +165,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run check-ci } test: @@ -144,8 +182,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run test-ci echo "Creating coverage tarball..." tar cvzf coverage.tgz coverage @@ -171,8 +209,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish (cd scripts/docker && docker build -t ubuntu18.04-xvfb -f ubuntu18.04-xvfb.Dockerfile .) docker run \ --rm -v $PWD:/tmp/build ubuntu18.04-xvfb \ @@ -191,8 +229,8 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env - npm --unsafe-perm=true run bootstrap-ci + source .evergreen/setup-env.sh + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run test-connectivity } test_apistrict: @@ -208,7 +246,7 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh npm run test-apistrict-ci } @@ -447,7 +485,7 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" npm run evergreen-release draft } @@ -465,11 +503,11 @@ functions: set -e { export NODE_JS_VERSION=${node_js_version} - source .evergreen/.setup_env + source .evergreen/setup-env.sh export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true" echo "registry=https://registry.npmjs.org/" > .npmrc echo "//registry.npmjs.org/:_authToken=${devtoolsbot_npm_token}" >> .npmrc - npm --unsafe-perm=true run bootstrap-ci + npm --unsafe-perm=true run bootstrap-ci -- --ignore-prepublish npm run evergreen-release publish } @@ -477,6 +515,7 @@ functions: # pass or fail. # # Current tasks: +# compile_ts - Do the initial compilation of TS sources. # check - Performs linter and dependency checks. # test_{version} - Runs all tests, against a specified mongod version. # test_vscode - Run the vscode extension integration tests. @@ -488,7 +527,17 @@ functions: # release_publish - Publishes the npm packages and uploads the tarballs. # pkg_test_* - Run tests on the release packages tasks: + - name: compile_ts + commands: + - func: checkout + - func: compile_ts + vars: + node_js_version: "<% out(NODE_JS_VERSION_14) %>" + - name: check + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -506,6 +555,9 @@ tasks: for (const { shortName: mShort, versionSpec: mVersion } of MONGODB_VERSIONS) { %> - name: test_<% const id = `m${mShort}_n${nShort}`; ALL_UNIT_TESTS.push(id); out(id) %> tags: <% out(["unit-test", ...(mShort === 'latest' ? ["mlatest"] : [])]) %> + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -523,6 +575,9 @@ tasks: ### - name: test_vscode tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -533,6 +588,9 @@ tasks: node_js_version: "<% out(NODE_JS_VERSION_12) %>" - name: test_connectivity tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -541,6 +599,9 @@ tasks: - func: test_connectivity - name: test_apistrict tags: ["extra-integration-test"] + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -552,6 +613,9 @@ tasks: mongosh_server_test_version: "latest-alpha" mongosh_test_force_api_strict: "1" - name: compile_artifact + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -677,6 +741,9 @@ tasks: tags: ["publish"] git_tag_only: true exec_timeout_secs: 7200 + depends_on: + - name: compile_ts + variant: linux commands: - func: checkout - func: install @@ -720,6 +787,7 @@ buildvariants: display_name: "Ubuntu 18.04 x64" run_on: ubuntu1804-small tasks: + - name: compile_ts - name: check <% for (const test of ALL_UNIT_TESTS) { %> - name: test_<% out(test) %> diff --git a/.evergreen/.install_node b/.evergreen/install-node.sh old mode 100644 new mode 100755 similarity index 79% rename from .evergreen/.install_node rename to .evergreen/install-node.sh index ca402d157c..c40e770915 --- a/.evergreen/.install_node +++ b/.evergreen/install-node.sh @@ -28,7 +28,7 @@ else fi fi -. "$BASEDIR/.setup_env" +. "$BASEDIR/setup-env.sh" # We need the build package for various tasks, and can bootstrap the cli-repl # package on all hosts, including dependencies. @@ -39,6 +39,8 @@ fi # to installing with --ignore-scripts (i.e. do not attempt to build addons) # and only do the TypeScript compilation step, which is sufficient for the # executable compilation step. -npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/build -npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies || \ - (npm --unsafe-perm=true run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-scripts && npm --unsafe-perm=true run compile-cli) +npm ci --verbose + +npm run bootstrap-ci -- --scope @mongosh/build --ignore-prepublish +(npm run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-prepublish && test -e packages/service-provider-server/node_modules/mongodb-client-encryption) || \ + npm run bootstrap-ci -- --scope @mongosh/cli-repl --include-dependencies --ignore-prepublish --ignore-scripts diff --git a/.evergreen/package-and-upload-artifact.sh b/.evergreen/package-and-upload-artifact.sh index 6c10a5d963..6bcd5cd277 100755 --- a/.evergreen/package-and-upload-artifact.sh +++ b/.evergreen/package-and-upload-artifact.sh @@ -8,7 +8,7 @@ cd $(pwd) export NODE_JS_VERSION=${NODE_JS_VERSION} export ARTIFACT_URL_FILE="$PWD/artifact-url.txt" -source .evergreen/.setup_env +source .evergreen/setup-env.sh tar xvzf dist.tgz if [ "$(uname)" == Linux ]; then diff --git a/.evergreen/run-e2e-tests.sh b/.evergreen/run-e2e-tests.sh index b0149a9aa8..4ee7e62ffe 100755 --- a/.evergreen/run-e2e-tests.sh +++ b/.evergreen/run-e2e-tests.sh @@ -2,7 +2,7 @@ set -e export NODE_JS_VERSION=${NODE_JS_VERSION} -source .evergreen/.setup_env +source .evergreen/setup-env.sh tar xvzf dist.tgz dist/mongosh --version diff --git a/.evergreen/.setup_env b/.evergreen/setup-env.sh similarity index 92% rename from .evergreen/.setup_env rename to .evergreen/setup-env.sh index deb0424847..fcb3a54b55 100755 --- a/.evergreen/.setup_env +++ b/.evergreen/setup-env.sh @@ -34,6 +34,11 @@ if [ "$OS" == "Windows_NT" ]; then export EVERGREEN_EXPANSIONS_PATH="$(cygpath -w "$EVERGREEN_EXPANSIONS_PATH")" fi +# On RHEL hosts, we run as root for some reason +if [ `uname` = Linux ]; then + export npm_config_unsafe_perm=true +fi + echo "Full path:" echo $PATH diff --git a/.gitignore b/.gitignore index 4e690a651a..09761e286b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ tmp/expansions.yaml .evergreen/mongodb tmp/ dist.tgz +compiled-ts.tgz mongocryptd.pid diff --git a/package.json b/package.json index 3436401dd2..57ba078a56 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,7 @@ "bootstrap-with-chromium": "lerna bootstrap --concurrency=1", "prebootstrap": "npm install", "bootstrap": "cross-env PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 lerna bootstrap --ignore-prepublish --concurrency=1", - "prebootstrap-ci": "npm ci", - "bootstrap-ci": "lerna bootstrap --ci --concurrency=1", + "bootstrap-ci": "lerna bootstrap --ci", "clean": "lerna clean -y && rm -Rf node_modules", "check": "lerna run check --since HEAD --exclude-dependents", "check-ci": "lerna run check", diff --git a/packages/java-shell/src/test/js/run-tests.ts b/packages/java-shell/src/test/js/run-tests.ts index 232b988d7c..b0e7b2e353 100644 --- a/packages/java-shell/src/test/js/run-tests.ts +++ b/packages/java-shell/src/test/js/run-tests.ts @@ -6,7 +6,7 @@ import { once } from 'events'; import { startTestServer } from '../../../../../testing/integration-testing-hooks'; describe('java-shell tests', function() { - this.timeout(600_000); + this.timeout(1_000_000); const testServer = startTestServer('shared'); const packageRoot = path.resolve(__dirname, '..', '..', '..') + '/';