diff --git a/.evergreen.yml b/.evergreen.yml index 64398e5e10..5c1faee9dc 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -6488,7 +6488,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_cli_repl" @@ -6686,7 +6686,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_cli_repl" @@ -7064,7 +7064,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_java_shell" @@ -7262,7 +7262,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_java_shell" @@ -7532,7 +7532,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_mongosh" @@ -7730,7 +7730,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_mongosh" @@ -7928,7 +7928,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_node_runtime_worker_thread" @@ -8126,7 +8126,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_node_runtime_worker_thread" @@ -8360,7 +8360,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_service_provider_server" @@ -8558,7 +8558,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_service_provider_server" @@ -8756,7 +8756,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "16.20.1" mongosh_skip_node_version_check: "" mongosh_test_id: "mlatest_n16_shell_api" @@ -8954,7 +8954,7 @@ tasks: npm_deps_mode: all - func: test vars: - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" node_js_version: "14.21.3" mongosh_skip_node_version_check: "1" mongosh_test_id: "mlatest_n14_shell_api" @@ -9113,7 +9113,7 @@ tasks: - func: test_apistrict vars: node_js_version: "16.20.1" - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" mongosh_test_force_api_strict: "1" - name: compile_artifact depends_on: @@ -9163,7 +9163,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_darwin_x64 tags: ["e2e-test"] @@ -9182,7 +9182,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_darwin_x64_50x_fips tags: ["e2e-test"] @@ -9201,7 +9201,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_darwin_x64_50x tags: ["e2e-test"] @@ -9220,7 +9220,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_darwin_arm64_fips tags: ["e2e-test"] @@ -9239,7 +9239,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_darwin_arm64 tags: ["e2e-test"] @@ -9258,7 +9258,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_darwin_arm64_50x_fips tags: ["e2e-test"] @@ -9277,7 +9277,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_darwin_arm64_50x tags: ["e2e-test"] @@ -9296,7 +9296,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_fips tags: ["e2e-test"] @@ -9315,7 +9315,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64 tags: ["e2e-test"] @@ -9334,7 +9334,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_50x_fips tags: ["e2e-test"] @@ -9353,7 +9353,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64_50x tags: ["e2e-test"] @@ -9372,7 +9372,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_openssl11_fips tags: ["e2e-test"] @@ -9391,7 +9391,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64_openssl11 tags: ["e2e-test"] @@ -9410,7 +9410,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_openssl11_50x_fips tags: ["e2e-test"] @@ -9429,7 +9429,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64_openssl11_50x tags: ["e2e-test"] @@ -9448,7 +9448,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_openssl3_fips tags: ["e2e-test"] @@ -9467,7 +9467,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64_openssl3 tags: ["e2e-test"] @@ -9486,7 +9486,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_x64_openssl3_50x_fips tags: ["e2e-test"] @@ -9505,7 +9505,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_x64_openssl3_50x tags: ["e2e-test"] @@ -9524,7 +9524,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_arm64_fips tags: ["e2e-test"] @@ -9543,7 +9543,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_arm64 tags: ["e2e-test"] @@ -9562,7 +9562,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_arm64_50x_fips tags: ["e2e-test"] @@ -9581,7 +9581,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_arm64_50x tags: ["e2e-test"] @@ -9600,7 +9600,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_arm64_openssl11_fips tags: ["e2e-test"] @@ -9619,7 +9619,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_arm64_openssl11 tags: ["e2e-test"] @@ -9638,7 +9638,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_arm64_openssl11_50x_fips tags: ["e2e-test"] @@ -9657,7 +9657,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_arm64_openssl11_50x tags: ["e2e-test"] @@ -9676,7 +9676,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_ppc64le_fips tags: ["e2e-test"] @@ -9695,7 +9695,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_ppc64le tags: ["e2e-test"] @@ -9714,7 +9714,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_ppc64le_50x_fips tags: ["e2e-test"] @@ -9733,7 +9733,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_ppc64le_50x tags: ["e2e-test"] @@ -9752,7 +9752,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_s390x_fips tags: ["e2e-test"] @@ -9771,7 +9771,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_s390x tags: ["e2e-test"] @@ -9790,7 +9790,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_linux_s390x_50x_fips tags: ["e2e-test"] @@ -9809,7 +9809,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_linux_s390x_50x tags: ["e2e-test"] @@ -9828,7 +9828,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_win32_fips tags: ["e2e-test"] @@ -9847,7 +9847,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_win32 tags: ["e2e-test"] @@ -9866,7 +9866,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "stable" + mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" - name: e2e_tests_win32_50x_fips tags: ["e2e-test"] @@ -9885,7 +9885,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "1" - name: e2e_tests_win32_50x tags: ["e2e-test"] @@ -9904,7 +9904,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "16.20.1" - mongosh_server_test_version: "5.0.x" + mongosh_server_test_version: "5.0.x-enterprise" mongosh_test_e2e_force_fips: "" ### diff --git a/.evergreen/evergreen.yml.in b/.evergreen/evergreen.yml.in index 65f0124e77..de29ba3cd1 100644 --- a/.evergreen/evergreen.yml.in +++ b/.evergreen/evergreen.yml.in @@ -16,7 +16,7 @@ const MONGODB_VERSIONS = [ { shortName: '50xe', versionSpec: '5.0.x' }, { shortName: '60xc', versionSpec: '6.0.x-community' }, { shortName: '60xe', versionSpec: '6.0.x' }, - { shortName: 'latest', versionSpec: 'latest-alpha' } + { shortName: 'latest', versionSpec: 'latest-alpha-enterprise' } ]; const NODE_VERSIONS = [ { shortName: '16', versionSpec: NODE_JS_VERSION_16, skipNodeVersionCheck: '' }, @@ -810,7 +810,7 @@ tasks: - func: test_apistrict vars: node_js_version: "<% out(NODE_JS_VERSION_16) %>" - mongosh_server_test_version: "latest-alpha" + mongosh_server_test_version: "latest-alpha-enterprise" mongosh_test_force_api_strict: "1" - name: compile_artifact depends_on: @@ -864,7 +864,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "<% out(NODE_JS_VERSION_16) %>" - mongosh_server_test_version: "<% out(mVersion) %>" + mongosh_server_test_version: "<% out(mVersion) %>-enterprise" mongosh_test_e2e_force_fips: "<% out(fipsVariant === 'fips' ? '1' : '') %>" <% } } } %> diff --git a/.evergreen/setup-env.sh b/.evergreen/setup-env.sh index f7eb7c4caa..e562a43ccc 100755 --- a/.evergreen/setup-env.sh +++ b/.evergreen/setup-env.sh @@ -3,6 +3,7 @@ set -x export BASEDIR="$PWD/.evergreen" export PATH="/cygdrive/c/python/Python311/Scripts:/cygdrive/c/python/Python311:/cygdrive/c/Python311/Scripts:/cygdrive/c/Python311:/opt/python/3.6/bin:$BASEDIR/mingit/cmd:$BASEDIR/mingit/mingw64/libexec/git-core:$BASEDIR/git-2:$BASEDIR/npm-8/node_modules/.bin:$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/java/jdk16/bin:/opt/chefdk/gitbin:/cygdrive/c/cmake/bin:/opt/mongodbtoolchain/v3/bin:$PATH" export IS_MONGOSH_EVERGREEN_CI=1 +export DEBUG="mongodb*,$DEBUG" if [ "$OS" != "Windows_NT" ]; then if which realpath; then # No realpath on macOS, but also not needed there diff --git a/README.md b/README.md index bacb6f1e82..34a98c511f 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,6 @@ variable. For detailed instructions for each of our supported platforms, please ### Requirements - Node.js v16.x -- Python 3.x - - The test suite uses [mlaunch](http://blog.rueckstiess.com/mtools/mlaunch.html) - for managing running mongod, you can install that manually as well via - `pip3 install mtools[mlaunch]` if the automatic installation causes any trouble. ### Install diff --git a/package-lock.json b/package-lock.json index 97626296b5..e2c952c373 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "lerna": "^4.0.0", "mocha": "^10.2.0", "mongodb": "^5.7.0", + "mongodb-runner": "^5.4.4", "node-gyp": "^9.0.0", "nyc": "^15.1.0", "pkg-up": "^3.1.0", @@ -9592,6 +9593,18 @@ "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-constants/-/mongodb-constants-0.2.2.tgz", "integrity": "sha512-vm1G+/WRWmXGyE9ZnhDv9toe+LRu1x0F/lGEwqWESfBiUUUuVZhj25fS2o4IL7H4pJ31sFxr7/gu+ER8OkmtzA==" }, + "node_modules/@mongodb-js/mongodb-downloader": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.2.8.tgz", + "integrity": "sha512-y+mgw9QspvgTLRNHZJRS+DUTPk45RWpvYD1MaGDWhZ4ajffvxGqanY+Z4R6z01n+tIRmQvpShzF6zk+2Pr9d6w==", + "dependencies": { + "debug": "^4.3.4", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.3.0", + "node-fetch": "^2.6.11", + "tar": "^6.1.15" + } + }, "node_modules/@mongodb-js/monorepo-tools": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@mongodb-js/monorepo-tools/-/monorepo-tools-1.1.4.tgz", @@ -22160,6 +22173,63 @@ "lodash": "^4.17.15" } }, + "node_modules/mongodb-runner": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-5.4.4.tgz", + "integrity": "sha512-bODXH7sOMRXbpVceIR8zLnEgg9kv1eXW60pxW5dfNShkjWd+7wmalJY/Dm9LG1tcL7+O5H7lTZH+s32IWg+FVA==", + "dev": true, + "dependencies": { + "@mongodb-js/mongodb-downloader": "^0.2.8", + "debug": "^4.3.4", + "mongodb": "^5.6.0", + "mongodb-connection-string-url": "^2.6.0", + "yargs": "^17.7.2" + }, + "bin": { + "mongodb-runner": "bin/runner.js" + } + }, + "node_modules/mongodb-runner/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-runner/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-runner/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/mongosh": { "resolved": "packages/mongosh", "link": true @@ -22423,9 +22493,9 @@ "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==" }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -30155,13 +30225,13 @@ "dependencies": { "@mongodb-js/devtools-github-repo": "^1.0.1", "@mongodb-js/dl-center": "^1.0.1", + "@mongodb-js/mongodb-downloader": "^0.2.7", "@octokit/rest": "^17.9.0", "aws-sdk": "^2.674.0", "boxednode": "^2.0.1", "command-exists": "^1.2.9", "download": "^8.0.0", "es-aggregate-error": "^1.0.9", - "mongodb-download-url": "^1.3.0", "nock": "^13.0.11", "node-fetch": "^2.6.1", "pkg-up": "^3.1.0", @@ -38787,6 +38857,18 @@ "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-constants/-/mongodb-constants-0.2.2.tgz", "integrity": "sha512-vm1G+/WRWmXGyE9ZnhDv9toe+LRu1x0F/lGEwqWESfBiUUUuVZhj25fS2o4IL7H4pJ31sFxr7/gu+ER8OkmtzA==" }, + "@mongodb-js/mongodb-downloader": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.2.8.tgz", + "integrity": "sha512-y+mgw9QspvgTLRNHZJRS+DUTPk45RWpvYD1MaGDWhZ4ajffvxGqanY+Z4R6z01n+tIRmQvpShzF6zk+2Pr9d6w==", + "requires": { + "debug": "^4.3.4", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.3.0", + "node-fetch": "^2.6.11", + "tar": "^6.1.15" + } + }, "@mongodb-js/monorepo-tools": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@mongodb-js/monorepo-tools/-/monorepo-tools-1.1.4.tgz", @@ -39216,6 +39298,7 @@ "@mongodb-js/devtools-github-repo": "^1.0.1", "@mongodb-js/dl-center": "^1.0.1", "@mongodb-js/eslint-config-mongosh": "^1.0.0", + "@mongodb-js/mongodb-downloader": "^0.2.7", "@mongodb-js/prettier-config-devtools": "^1.0.1", "@mongodb-js/tsconfig-mongosh": "^1.0.0", "@octokit/rest": "^17.9.0", @@ -39236,7 +39319,6 @@ "download": "^8.0.0", "es-aggregate-error": "^1.0.9", "eslint": "^7.25.0", - "mongodb-download-url": "^1.3.0", "nock": "^13.0.11", "node-fetch": "^2.6.1", "pkg-up": "^3.1.0", @@ -49266,6 +49348,53 @@ "lodash": "^4.17.15" } }, + "mongodb-runner": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/mongodb-runner/-/mongodb-runner-5.4.4.tgz", + "integrity": "sha512-bODXH7sOMRXbpVceIR8zLnEgg9kv1eXW60pxW5dfNShkjWd+7wmalJY/Dm9LG1tcL7+O5H7lTZH+s32IWg+FVA==", + "dev": true, + "requires": { + "@mongodb-js/mongodb-downloader": "^0.2.8", + "debug": "^4.3.4", + "mongodb": "^5.6.0", + "mongodb-connection-string-url": "^2.6.0", + "yargs": "^17.7.2" + }, + "dependencies": { + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true + } + } + }, "mongosh": { "version": "file:packages/mongosh", "requires": { @@ -49494,9 +49623,9 @@ "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==" }, "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", "requires": { "whatwg-url": "^5.0.0" }, diff --git a/package.json b/package.json index 3e46379de8..9ddceee33a 100644 --- a/package.json +++ b/package.json @@ -20,14 +20,6 @@ "predepcheck": "npm run depalign", "depcheck": "lerna run --stream depcheck", "lint": "lerna run lint", - "clear-mlaunch": "ts-node -P configs/tsconfig-mongosh/tsconfig.common.json -r ./scripts/import-expansions scripts/clear-mlaunch", - "install-mlaunch": "ts-node -P configs/tsconfig-mongosh/tsconfig.common.json -r ./scripts/import-expansions scripts/install-mlaunch", - "pretest": "npm run clear-mlaunch && npm run install-mlaunch", - "pretest-ci": "npm run clear-mlaunch -- --killAllMongod && npm run install-mlaunch", - "pretest-e2e-ci": "npm run clear-mlaunch -- --killAllMongod && npm run install-mlaunch", - "posttest": "npm run clear-mlaunch", - "posttest-ci": "npm run clear-mlaunch -- --killAllMongod", - "posttest-e2e-ci": "npm run clear-mlaunch -- --killAllMongod", "test": "rimraf .nyc_output && lerna exec -- nyc --no-clean --cwd ../.. --reporter=none npm run test && npm run report-coverage", "test-ci": "rimraf .nyc_output && lerna exec -- nyc --no-clean --cwd ../.. --reporter=none npm run test-ci && npm run post-process-nyc", "test-ci-nocoverage": "lerna exec -- npm run test-ci", @@ -116,6 +108,7 @@ "lerna": "^4.0.0", "mocha": "^10.2.0", "mongodb": "^5.7.0", + "mongodb-runner": "^5.4.4", "node-gyp": "^9.0.0", "nyc": "^15.1.0", "pkg-up": "^3.1.0", diff --git a/packages/build/package.json b/packages/build/package.json index bd1f7bc7c1..dd7ea48c0c 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -58,6 +58,7 @@ }, "dependencies": { "@mongodb-js/devtools-github-repo": "^1.0.1", + "@mongodb-js/mongodb-downloader": "^0.2.7", "@mongodb-js/dl-center": "^1.0.1", "@octokit/rest": "^17.9.0", "aws-sdk": "^2.674.0", @@ -65,7 +66,6 @@ "command-exists": "^1.2.9", "download": "^8.0.0", "es-aggregate-error": "^1.0.9", - "mongodb-download-url": "^1.3.0", "nock": "^13.0.11", "node-fetch": "^2.6.1", "pkg-up": "^3.1.0", diff --git a/packages/build/src/download-mongodb.ts b/packages/build/src/download-mongodb.ts deleted file mode 100644 index b0c038c60c..0000000000 --- a/packages/build/src/download-mongodb.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* eslint-disable no-empty */ -/* istanbul ignore file */ -import fetch from 'node-fetch'; -import tar from 'tar'; -import { promisify } from 'util'; -import { promises as fs } from 'fs'; -import path from 'path'; -import download from 'download'; -import { pipeline } from 'stream'; -import getDownloadURL from 'mongodb-download-url'; -import type { Options as DownloadOptions } from 'mongodb-download-url'; - -export type { DownloadOptions }; - -// Download mongod + mongos and return the path to a directory containing them. -export async function downloadMongoDb( - tmpdir: string, - targetVersionSemverSpecifier = '*', - options: DownloadOptions = {} -): Promise { - let wantsEnterprise = options.enterprise ?? true; - async function lookupDownloadUrl(): Promise { - return ( - await getDownloadURL({ - version: targetVersionSemverSpecifier, - enterprise: wantsEnterprise, - ...options, - }) - ).url; - } - - await fs.mkdir(tmpdir, { recursive: true }); - if (targetVersionSemverSpecifier === 'latest-alpha') { - return await doDownload( - tmpdir, - !!options.crypt_shared, - 'latest-alpha', - lookupDownloadUrl - ); - } - - if (/-community$/.test(targetVersionSemverSpecifier)) { - wantsEnterprise = false; - targetVersionSemverSpecifier = targetVersionSemverSpecifier.replace( - /-community$/, - '' - ); - } - - return await doDownload( - tmpdir, - !!options.crypt_shared, - targetVersionSemverSpecifier + - (wantsEnterprise ? '-enterprise' : '-community'), - () => lookupDownloadUrl() - ); -} - -const downloadPromises: Record> = {}; -async function doDownload( - tmpdir: string, - isCryptLibrary: boolean, - version: string, - lookupDownloadUrl: () => Promise -) { - const downloadTarget = path.resolve( - tmpdir, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${version}`.replace(/[^a-zA-Z0-9_-]/g, '') - ); - return (downloadPromises[downloadTarget] ??= (async () => { - const bindir = path.resolve( - downloadTarget, - isCryptLibrary && process.platform !== 'win32' ? 'lib' : 'bin' - ); - try { - await fs.stat(bindir); - console.info(`Skipping download because ${downloadTarget} exists`); - return bindir; - } catch {} - - await fs.mkdir(downloadTarget, { recursive: true }); - const url = await lookupDownloadUrl(); - console.info('Downloading...', url); - - async function downloadAndExtract(withExtraStripDepth = 0): Promise { - if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { - // the server's tarballs can contain hard links, which the (unmaintained?) - // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) - const response = await fetch(url); - await promisify(pipeline)( - response.body, - tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }) - ); - } else { - await download(url, downloadTarget, { - extract: true, - strip: isCryptLibrary ? 0 : 1, - }); - } - - try { - await fs.stat(bindir); // Make sure it exists. - } catch (err) { - if (withExtraStripDepth === 0 && url.includes('macos')) { - // The server team changed how macos release artifacts are packed - // and added a `./` prefix to paths in the tarball, - // which seems like it shouldn't change anything but does - // in fact require an increased path strip depth. - console.info('Retry due to miscalculated --strip-components depth'); - return await downloadAndExtract(1); - } - throw err; - } - } - - await downloadAndExtract(); - return bindir; - })()); -} diff --git a/packages/build/src/index.ts b/packages/build/src/index.ts index 22a398dbdb..e4f0cbcdd9 100644 --- a/packages/build/src/index.ts +++ b/packages/build/src/index.ts @@ -1,6 +1,6 @@ import path from 'path'; import { validatePackageVariant } from './config'; -import { downloadMongoDb } from './download-mongodb'; +import { downloadMongoDb } from '@mongodb-js/mongodb-downloader'; import { getArtifactUrl } from './evergreen'; import { triggerRelease } from './local'; import type { ReleaseCommand } from './release'; diff --git a/packages/build/src/packaging/download-crypt-library.ts b/packages/build/src/packaging/download-crypt-library.ts index 4193991e4f..bb7688a910 100644 --- a/packages/build/src/packaging/download-crypt-library.ts +++ b/packages/build/src/packaging/download-crypt-library.ts @@ -1,8 +1,8 @@ /* istanbul ignore file */ import path from 'path'; import { promises as fs, constants as fsConstants } from 'fs'; -import type { DownloadOptions } from '../download-mongodb'; -import { downloadMongoDb } from '../download-mongodb'; +import type { DownloadOptions } from '@mongodb-js/mongodb-downloader'; +import { downloadMongoDb } from '@mongodb-js/mongodb-downloader'; import type { PackageVariant } from '../config'; import { getDistro, getArch } from '../config'; diff --git a/packages/cli-repl/src/cli-repl.spec.ts b/packages/cli-repl/src/cli-repl.spec.ts index 6e27f30c36..5b1ef84d83 100644 --- a/packages/cli-repl/src/cli-repl.spec.ts +++ b/packages/cli-repl/src/cli-repl.spec.ts @@ -1971,7 +1971,10 @@ describe('CliRepl', function () { context('with a replset node', function () { verifyAutocompletion({ - testServer: startTestServer('not-shared', '--replicaset', '--nodes', '1'), + testServer: startTestServer('not-shared', { + topology: 'replset', + secondaries: 0, + }), wantWatch: true, wantShardDistribution: false, hasCollectionNames: true, @@ -1981,13 +1984,11 @@ describe('CliRepl', function () { context('with a mongos', function () { verifyAutocompletion({ - testServer: startTestServer( - 'not-shared', - '--replicaset', - '--csrs', - '--sharded', - '0' - ), + testServer: startTestServer('not-shared', { + topology: 'sharded', + shards: 1, + secondaries: 0, + }), wantWatch: true, wantShardDistribution: true, hasCollectionNames: false, // We're only spinning up a mongos here @@ -1997,7 +1998,9 @@ describe('CliRepl', function () { context('with an auth-required mongod', function () { verifyAutocompletion({ - testServer: startTestServer('not-shared', '--auth'), + testServer: startTestServer('not-shared', { + args: ['--auth'], + }), wantWatch: false, wantShardDistribution: false, hasCollectionNames: false, diff --git a/packages/cli-repl/test/e2e-analytics.spec.ts b/packages/cli-repl/test/e2e-analytics.spec.ts index 207af6649a..7910ae2c08 100644 --- a/packages/cli-repl/test/e2e-analytics.spec.ts +++ b/packages/cli-repl/test/e2e-analytics.spec.ts @@ -6,10 +6,10 @@ import { TestShell } from './test-shell'; describe('e2e Analytics Node', function () { const replSetName = 'replicaSet'; const [rs0, rs1, rs2, rs3] = startTestCluster( - ['--single', '--replSet', replSetName], - ['--single', '--replSet', replSetName], - ['--single', '--replSet', replSetName], - ['--single', '--replSet', replSetName] + { args: ['--replSet', replSetName] }, + { args: ['--replSet', replSetName] }, + { args: ['--replSet', replSetName] }, + { args: ['--replSet', replSetName] } ); after(TestShell.cleanup); @@ -61,12 +61,7 @@ describe('e2e Analytics Node', function () { const helloResult = await shell.executeLine('db.hello()'); expect(helloResult).to.contain('isWritablePrimary: true'); - - await shell.executeLine('use admin'); - const explain = await shell.executeLine( - "db['system.users'].find().explain()" - ); - expect(explain).to.contain(`port: ${await rs0.port()}`); + expect(helloResult).not.to.contain('nodeType:'); }); }); @@ -77,12 +72,34 @@ describe('e2e Analytics Node', function () { `${await rs0.connectionString()}?replicaSet=${replSetName}&readPreference=secondary&readPreferenceTags=nodeType:ANALYTICS`, ], }); - await shell.waitForPrompt(); + + const directConnectionToAnalyticsShell = TestShell.start({ + args: [`${await rs3.connectionString()}?directConnection=true`], + }); + await Promise.all([ + shell.waitForPrompt(), + directConnectionToAnalyticsShell.waitForPrompt(), + ]); + + const helloResult = await shell.executeLine('db.hello()'); + expect(helloResult).to.contain('isWritablePrimary: false'); + expect(helloResult).to.contain("nodeType: 'ANALYTICS'"); + + await directConnectionToAnalyticsShell.executeLine( + 'const before = db.serverStatus().opcounters.query' + ); + await shell.executeLine('use admin'); - const explain = await shell.executeLine( - "db['system.users'].find().explain()" + await shell.executeLine("db['system.users'].findOne()"); + + await directConnectionToAnalyticsShell.executeLine( + 'const after = db.serverStatus().opcounters.query' ); - expect(explain).to.contain(`port: ${await rs3.port()}`); + expect( + await directConnectionToAnalyticsShell.executeLine( + '({ diff: after - before })' + ) + ).to.match(/diff: [1-9]/); // not 0 }); }); }); diff --git a/packages/cli-repl/test/e2e-auth.spec.ts b/packages/cli-repl/test/e2e-auth.spec.ts index e3b6abf4ac..062e1f4c36 100644 --- a/packages/cli-repl/test/e2e-auth.spec.ts +++ b/packages/cli-repl/test/e2e-auth.spec.ts @@ -80,7 +80,7 @@ describe('Auth e2e', function () { let db; let client; let shell: TestShell; - let dbName; + let dbName: string; let examplePrivilege1; let examplePrivilege2; @@ -843,9 +843,14 @@ describe('Auth e2e', function () { await assertUserAuth('pwd2', 'anna2'); }); it('can auth when there is login in URI', async function () { - const connectionString = await testServer.connectionString(); - const split = connectionString.split('//'); - const authConnectionString = `${split[0]}//anna2:pwd2@${split[1]}/${dbName}`; + const authConnectionString = await testServer.connectionString( + {}, + { + username: 'anna2', + password: 'pwd2', + pathname: `/${dbName}`, + } + ); shell = TestShell.start({ args: [authConnectionString] }); await shell.waitForPrompt(); shell.assertNoErrors(); @@ -862,9 +867,14 @@ describe('Auth e2e', function () { shell.assertNoErrors(); }); it('connection-resetting operations don’t undo auth', async function () { - const connectionString = await testServer.connectionString(); - const split = connectionString.split('//'); - const authConnectionString = `${split[0]}//anna2:pwd2@${split[1]}/${dbName}`; + const authConnectionString = await testServer.connectionString( + {}, + { + username: 'anna2', + password: 'pwd2', + pathname: `/${dbName}`, + } + ); shell = TestShell.start({ args: [authConnectionString] }); await shell.waitForPrompt(); shell.assertNoErrors(); @@ -1065,6 +1075,7 @@ describe('Auth e2e', function () { 'Miscellaneous failure (see text): Unable to find realm of host localhost', 'Miscellaneous failure (see text): no credential for', "Unsupported mechanism 'GSSAPI' on authentication database '$external'", + 'The specified target is unknown or unreachable', ]; expect(messages.some((msg) => shell.output.includes(msg))).to.equal( true, diff --git a/packages/cli-repl/test/e2e-direct.spec.ts b/packages/cli-repl/test/e2e-direct.spec.ts index 2301ed7427..138c1bd430 100644 --- a/packages/cli-repl/test/e2e-direct.spec.ts +++ b/packages/cli-repl/test/e2e-direct.spec.ts @@ -21,9 +21,9 @@ describe('e2e direct connection', function () { context('to a replica set', function () { const replSetId = 'replset'; const [rs0, rs1, rs2] = startTestCluster( - ['--single', '--replSet', replSetId], - ['--single', '--replSet', replSetId], - ['--single', '--replSet', replSetId] + { args: ['--replSet', replSetId] }, + { args: ['--replSet', replSetId] }, + { args: ['--replSet', replSetId] } ); [ @@ -121,7 +121,9 @@ describe('e2e direct connection', function () { it('lists collections when readPreference is in the connection string', async function () { const shell = TestShell.start({ args: [ - `${await rs1.connectionString()}?readPreference=secondaryPreferred`, + await rs1.connectionString({ + readPreference: 'secondaryPreferred', + }), ], }); await shell.waitForPrompt(); @@ -132,7 +134,7 @@ describe('e2e direct connection', function () { it('lists collections when readPreference is set via Mongo', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}`], + args: [await rs1.connectionString()], }); await shell.waitForPrompt(); await shell.executeLine('use admin'); @@ -145,7 +147,7 @@ describe('e2e direct connection', function () { it('fails to list databases without explicit readPreference', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}`], + args: [await rs1.connectionString()], }); await shell.waitForPrompt(); await shell.executeLine('use admin'); @@ -156,7 +158,9 @@ describe('e2e direct connection', function () { it('lists databases when readPreference is in the connection string', async function () { const shell = TestShell.start({ args: [ - `${await rs1.connectionString()}?readPreference=secondaryPreferred`, + await rs1.connectionString({ + readPreference: 'secondaryPreferred', + }), ], }); await shell.waitForPrompt(); @@ -167,7 +171,7 @@ describe('e2e direct connection', function () { it('lists databases when readPreference is set via Mongo', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}`], + args: [await rs1.connectionString()], }); await shell.waitForPrompt(); await shell.executeLine('use admin'); @@ -180,7 +184,7 @@ describe('e2e direct connection', function () { it('lists collections and dbs using show by default', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}`], + args: [await rs1.connectionString()], }); await shell.waitForPrompt(); await shell.executeLine('use admin'); @@ -195,7 +199,7 @@ describe('e2e direct connection', function () { return this.skip(); // https://jira.mongodb.org/browse/MONGOSH-746 } const shell = TestShell.start({ - args: [`${await rs1.connectionString()}/${dbname}`], + args: [await rs1.connectionString({}, { pathname: `/${dbname}` })], forceTerminal: true, }); await shell.waitForPrompt(); @@ -212,7 +216,16 @@ describe('e2e direct connection', function () { it('allows aggregate with $merge with secondary readpref', async function () { const shell = TestShell.start({ args: [ - `${await rs1.connectionString()}/${dbname}?readPreference=secondary&directConnection=false&serverSelectionTimeoutMS=10000`, + await rs1.connectionString( + { + readPreference: 'secondary', + directConnection: 'false', + serverSelectionTimeoutMS: '1000', + }, + { + pathname: `/${dbname}`, + } + ), ], }); await shell.waitForPrompt(); @@ -234,7 +247,7 @@ describe('e2e direct connection', function () { context('connecting to primary', function () { it('when specifying replicaSet', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}?replicaSet=${replSetId}`], + args: [await rs1.connectionString({ replicaSet: replSetId })], }); await shell.waitForPrompt(); await shell.executeLine('db.isMaster()'); @@ -244,7 +257,7 @@ describe('e2e direct connection', function () { }); it('when setting directConnection to false', async function () { const shell = TestShell.start({ - args: [`${await rs1.connectionString()}?directConnection=false`], + args: [await rs1.connectionString({ directConnection: 'false' })], }); await shell.waitForPrompt(); await shell.executeLine('db.isMaster()'); @@ -352,7 +365,7 @@ describe('e2e direct connection', function () { return this.skip(); // https://jira.mongodb.org/browse/MONGOSH-746 } const shell = TestShell.start({ - args: [`${await rs1.connectionString()}/${dbname}`], + args: [await rs1.connectionString({}, { pathname: `/${dbname}` })], forceTerminal: true, }); await shell.waitForPrompt(); diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index a7bbbe6561..e5512a4a3e 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -24,12 +24,10 @@ function isMacosTooOldForQE() { } describe('FLE tests', function () { - const testServer = startTestServer( - 'not-shared', - '--replicaset', - '--nodes', - '1' - ); + const testServer = startTestServer('not-shared', { + topology: 'replset', + secondaries: 0, + }); skipIfServerVersion(testServer, '< 4.2'); // FLE only available on 4.2+ skipIfCommunityServer(testServer); // FLE is enterprise-only let kmsServer: ReturnType; diff --git a/packages/cli-repl/test/e2e-oidc.spec.ts b/packages/cli-repl/test/e2e-oidc.spec.ts index 944aba818c..bf6fb5bfbc 100644 --- a/packages/cli-repl/test/e2e-oidc.spec.ts +++ b/packages/cli-repl/test/e2e-oidc.spec.ts @@ -1,5 +1,5 @@ import { - MlaunchSetup, + MongoRunnerSetup, skipIfApiStrict, skipIfEnvServerVersion, } from '../../../testing/integration-testing-hooks'; @@ -14,8 +14,8 @@ describe('OIDC auth e2e', function () { let getTokenPayload: typeof oidcMockProviderConfig.getTokenPayload; let tokenFetches: number; - let testServer: MlaunchSetup; - let testServer2: MlaunchSetup; + let testServer: MongoRunnerSetup; + let testServer2: MongoRunnerSetup; let oidcMockProviderConfig: OIDCMockProviderConfig; let oidcMockProvider: OIDCMockProvider; let shell: TestShell; @@ -69,20 +69,22 @@ describe('OIDC auth e2e', function () { '--setParameter', 'enableTestCommands=true', ]; - testServer = new MlaunchSetup([ - '--setParameter', - // weird quoting to work around mlaunch - `"'oidcIdentityProviders=${JSON.stringify([serverOidcConfig])}'"`, - ...commonOidcServerArgs, - ]); - testServer2 = new MlaunchSetup([ - '--setParameter', - // weird quoting to work around mlaunch - `"'oidcIdentityProviders=${JSON.stringify([ - { ...serverOidcConfig, clientId: 'testServer2' }, - ])}'"`, - ...commonOidcServerArgs, - ]); + testServer = new MongoRunnerSetup({ + args: [ + '--setParameter', + `oidcIdentityProviders=${JSON.stringify([serverOidcConfig])}`, + ...commonOidcServerArgs, + ], + }); + testServer2 = new MongoRunnerSetup({ + args: [ + '--setParameter', + `oidcIdentityProviders=${JSON.stringify([ + { ...serverOidcConfig, clientId: 'testServer2' }, + ])}`, + ...commonOidcServerArgs, + ], + }); await Promise.all([testServer.start(), testServer2.start()]); }); diff --git a/packages/cli-repl/test/e2e-tls.spec.ts b/packages/cli-repl/test/e2e-tls.spec.ts index 6fea50b0de..753076d61c 100644 --- a/packages/cli-repl/test/e2e-tls.spec.ts +++ b/packages/cli-repl/test/e2e-tls.spec.ts @@ -4,6 +4,7 @@ import path from 'path'; import { skipIfEnvServerVersion, startTestServer, + MongodSetup, } from '../../../testing/integration-testing-hooks'; import { useTmpdir, @@ -14,6 +15,22 @@ import { TestShell } from './test-shell'; import { promisify } from 'util'; import rimraf from 'rimraf'; +// TLS requires matching hostnames, so here we need to explicitly +// specify `localhost` + IPv4 instead of `127.0.0.1` +async function connectionStringWithLocalhost( + setup: MongodSetup, + searchParams: Record = {} +): Promise { + const cs = await setup.connectionStringUrl(); + cs.hosts = cs.hosts.map((host) => + host.replace(/^(127.0.0.1)(?=$|:)/, 'localhost') + ); + cs.searchParams.set('family', '4'); + for (const [key, value] of Object.entries(searchParams)) + cs.searchParams.set(key, value); + return cs.toString(); +} + function getCertPath(filename: string): string { return path.join( __dirname, @@ -111,7 +128,7 @@ describe('e2e TLS', function () { // here to work around that. const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -124,20 +141,19 @@ describe('e2e TLS', function () { }); afterEach(TestShell.cleanup); - const server = startTestServer( - 'not-shared', - '--hostname', - 'localhost', - serverTlsModeOption, - serverTlsModeValue, - serverTlsCertificateKeyFileOption, - SERVER_KEY - ); + const server = startTestServer('not-shared', { + args: [ + serverTlsModeOption, + serverTlsModeValue, + serverTlsCertificateKeyFileOption, + SERVER_KEY, + ], + }); it('works with matching CA (args)', async function () { const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -150,9 +166,10 @@ describe('e2e TLS', function () { it('works with matching CA (connection string)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?tls=true&tlsCAFile=${encodeURIComponent( - CA_CERT - )}`, + await connectionStringWithLocalhost(server, { + tls: 'true', + tlsCAFile: CA_CERT, + }), ], }); const result = await shell.waitForPromptOrExit(); @@ -162,7 +179,9 @@ describe('e2e TLS', function () { it('fails when not using --tls (args)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), ], }); const result = await shell.waitForPromptOrExit(); @@ -173,7 +192,10 @@ describe('e2e TLS', function () { it('fails when not using --tls (connection string)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500&tls=false`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + tls: 'false', + }), ], }); const result = await shell.waitForPromptOrExit(); @@ -184,7 +206,9 @@ describe('e2e TLS', function () { it('fails with invalid CA (args)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsCAFile', NON_CA_CERT, @@ -198,9 +222,11 @@ describe('e2e TLS', function () { it('fails with invalid CA (connection string)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500&tls=true&tlsCAFile=${encodeURIComponent( - NON_CA_CERT - )}`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + tls: 'true', + tlsCAFile: NON_CA_CERT, + }), ], }); const result = await shell.waitForPromptOrExit(); @@ -211,7 +237,9 @@ describe('e2e TLS', function () { it('fails when providing a CRL including the servers cert', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsCAFile', CA_CERT, @@ -235,7 +263,9 @@ describe('e2e TLS', function () { ); const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsUseSystemCA', ], @@ -268,7 +298,9 @@ describe('e2e TLS', function () { ); const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsUseSystemCA', ], @@ -300,7 +332,9 @@ describe('e2e TLS', function () { } const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsUseSystemCA', ], @@ -324,7 +358,7 @@ describe('e2e TLS', function () { after(async function () { const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -340,17 +374,16 @@ describe('e2e TLS', function () { await TestShell.cleanup.call(this); }); - const server = startTestServer( - 'not-shared', - '--hostname', - 'localhost', - serverTlsModeOption, - serverTlsModeValue, - serverTlsCertificateKeyFileOption, - SERVER_KEY, - serverTlsCAFileOption, - CA_CERT - ); + const server = startTestServer('not-shared', { + args: [ + serverTlsModeOption, + serverTlsModeValue, + serverTlsCertificateKeyFileOption, + SERVER_KEY, + serverTlsCAFileOption, + CA_CERT, + ], + }); const certUser = 'emailAddress=tester@example.com,CN=Wonderwoman,OU=DevTools Testers,O=MongoDB'; @@ -361,7 +394,9 @@ describe('e2e TLS', function () { /* connect with cert to create user */ const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--tls', '--tlsCAFile', CA_CERT, @@ -383,7 +418,9 @@ describe('e2e TLS', function () { it('works with valid cert (args)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--authenticationMechanism', 'MONGODB-X509', '--tls', @@ -412,7 +449,9 @@ describe('e2e TLS', function () { it('works with valid cert (args, encrypted)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--authenticationMechanism', 'MONGODB-X509', '--tls', @@ -448,11 +487,13 @@ describe('e2e TLS', function () { it('works with valid cert (connection string)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500` + - '&authMechanism=MONGODB-X509' + - `&tls=true&tlsCAFile=${encodeURIComponent( - CA_CERT - )}&tlsCertificateKeyFile=${encodeURIComponent(CLIENT_CERT)}`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + authMechanism: 'MONGODB-X509', + tls: 'true', + tlsCAFile: CA_CERT, + tlsCertificateKeyFile: CLIENT_CERT, + }), ], }); const prompt = await shell.waitForPromptOrExit(); @@ -474,15 +515,14 @@ describe('e2e TLS', function () { it('works with valid cert (connection string, encrypted)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500` + - '&authMechanism=MONGODB-X509' + - `&tls=true&tlsCAFile=${encodeURIComponent(CA_CERT)}` + - `&tlsCertificateKeyFile=${encodeURIComponent( - CLIENT_CERT_ENCRYPTED - )}` + - `&tlsCertificateKeyFilePassword=${encodeURIComponent( - CLIENT_CERT_PASSWORD - )}`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + authMechanism: 'MONGODB-X509', + tls: 'true', + tlsCAFile: CA_CERT, + tlsCertificateKeyFile: CLIENT_CERT_ENCRYPTED, + tlsCertificateKeyFilePassword: CLIENT_CERT_PASSWORD, + }), ], env, }); @@ -509,7 +549,9 @@ describe('e2e TLS', function () { it('fails with invalid cert (args)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--authenticationMechanism', 'MONGODB-X509', '--tls', @@ -527,13 +569,13 @@ describe('e2e TLS', function () { it('fails with invalid cert (connection string)', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500` + - '&authMechanism=MONGODB-X509' + - `&tls=true&tlsCAFile=${encodeURIComponent( - CA_CERT - )}&tlsCertificateKeyFile=${encodeURIComponent( - INVALID_CLIENT_CERT - )}`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + authMechanism: 'MONGODB-X509', + tls: 'true', + tlsCAFile: CA_CERT, + tlsCertificateKeyFile: INVALID_CLIENT_CERT, + }), ], }); const exit = await shell.waitForPromptOrExit(); @@ -558,7 +600,9 @@ describe('e2e TLS', function () { ); const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--authenticationMechanism', 'MONGODB-X509', '--tls', @@ -581,7 +625,9 @@ describe('e2e TLS', function () { it('fails with an invalid tlsCertificateSelector', async function () { const shell = TestShell.start({ args: [ - `${await server.connectionString()}?serverSelectionTimeoutMS=1500`, + await connectionStringWithLocalhost(server, { + serverSelectionTimeoutMS: '1500', + }), '--authenticationMechanism', 'MONGODB-X509', '--tls', @@ -616,7 +662,7 @@ describe('e2e TLS', function () { // here to work around that. const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -628,20 +674,19 @@ describe('e2e TLS', function () { await TestShell.killall(); }); - const server = startTestServer( - 'not-shared', - '--hostname', - 'localhost', - serverTlsModeOption, - serverTlsModeValue, - serverTlsCertificateKeyFileOption, - SERVER_INVALIDHOST_KEY - ); + const server = startTestServer('not-shared', { + args: [ + serverTlsModeOption, + serverTlsModeValue, + serverTlsCertificateKeyFileOption, + SERVER_INVALIDHOST_KEY, + ], + }); it('works with allowInvalidCertificates', async function () { const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -655,7 +700,7 @@ describe('e2e TLS', function () { it('works with allowInvalidHostnames', async function () { const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, @@ -669,7 +714,7 @@ describe('e2e TLS', function () { it('fails when no additional args are provided', async function () { const shell = TestShell.start({ args: [ - await server.connectionString(), + await connectionStringWithLocalhost(server), '--tls', '--tlsCAFile', CA_CERT, diff --git a/packages/cli-repl/test/e2e.spec.ts b/packages/cli-repl/test/e2e.spec.ts index f703e0c316..0b6ea9c02b 100644 --- a/packages/cli-repl/test/e2e.spec.ts +++ b/packages/cli-repl/test/e2e.spec.ts @@ -1392,7 +1392,7 @@ describe('e2e', function () { it('errors if an API version is specified', async function () { const shell = TestShell.start({ args: [ - `${await testServer.connectionString()}/${dbName}`, + await testServer.connectionString({}, { pathname: `/${dbName}` }), '--apiVersion', '1', ], @@ -1410,7 +1410,7 @@ describe('e2e', function () { it('can specify an API version', async function () { const shell = TestShell.start({ args: [ - `${await testServer.connectionString()}/${dbName}`, + await testServer.connectionString({}, { pathname: `/${dbName}` }), '--apiVersion', '1', ], @@ -1426,7 +1426,7 @@ describe('e2e', function () { it('can specify an API version and strict mode', async function () { const shell = TestShell.start({ args: [ - `${await testServer.connectionString()}/${dbName}`, + await testServer.connectionString({}, { pathname: `/${dbName}` }), '--apiVersion', '1', '--apiStrict', @@ -1445,7 +1445,7 @@ describe('e2e', function () { // Make sure SERVER-55593 doesn't happen to us. const shell = TestShell.start({ args: [ - `${await testServer.connectionString()}/${dbName}`, + await testServer.connectionString({}, { pathname: `/${dbName}` }), '--apiVersion', '1', ], diff --git a/packages/java-shell/src/test/js/run-tests.ts b/packages/java-shell/src/test/js/run-tests.ts index b0e7b2e353..9331c9400c 100644 --- a/packages/java-shell/src/test/js/run-tests.ts +++ b/packages/java-shell/src/test/js/run-tests.ts @@ -1,6 +1,4 @@ -'use strict'; -import child_process from 'child_process'; -import fs from 'fs'; +import { spawn } from 'child_process'; import path from 'path'; import { once } from 'events'; import { startTestServer } from '../../../../../testing/integration-testing-hooks'; @@ -11,10 +9,10 @@ describe('java-shell tests', function() { const packageRoot = path.resolve(__dirname, '..', '..', '..') + '/'; before(async function () { - process.env.JAVA_SHELL_MONGOSH_TEST_URI = await testServer.connectionString(); + process.env.JAVA_SHELL_MONGOSH_TEST_URI = (await testServer.connectionString()).replace(/\/$/, ''); const connectionString = await testServer.connectionString(); - const mongosh = child_process.spawn( + const mongosh = spawn( process.execPath, [ path.resolve(packageRoot, '..', 'cli-repl', 'bin', 'mongosh.js'), connectionString ], { stdio: [ 'pipe', 'inherit', 'inherit' ] }); @@ -26,12 +24,15 @@ describe('java-shell tests', function() { await once(mongosh, 'exit'); }); - it('passes the JavaShell tests', () => { + it('passes the JavaShell tests', async function() { + const opts = { stdio: 'inherit', cwd: packageRoot, shell: true } as const; + let proc; if (process.platform !== 'win32') { - child_process.execSync('./gradlew test --info', { stdio: 'inherit', cwd: packageRoot }); + proc = spawn('./gradlew test --info', [], opts); } else { - child_process.execSync('.\\gradlew.bat test --info', { stdio: 'inherit', cwd: packageRoot }); + proc = spawn('.\\gradlew.bat test --info', [], opts); } + await once(proc, 'exit'); }); }); diff --git a/packages/service-provider-server/src/cli-service-provider.integration.spec.ts b/packages/service-provider-server/src/cli-service-provider.integration.spec.ts index b08703e5c0..5feddc7c6e 100644 --- a/packages/service-provider-server/src/cli-service-provider.integration.spec.ts +++ b/packages/service-provider-server/src/cli-service-provider.integration.spec.ts @@ -798,7 +798,7 @@ describe('CliServiceProvider [integration]', function () { describe('#getURI', function () { it('returns the current URI', function () { - expect(serviceProvider.getURI()).to.equal(connectionString + '/'); + expect(serviceProvider.getURI()).to.equal(connectionString); }); }); diff --git a/packages/shell-api/src/change-stream-cursor.spec.ts b/packages/shell-api/src/change-stream-cursor.spec.ts index 375ac15bd4..863d88ec57 100644 --- a/packages/shell-api/src/change-stream-cursor.spec.ts +++ b/packages/shell-api/src/change-stream-cursor.spec.ts @@ -117,7 +117,7 @@ describe('ChangeStreamCursor', function () { }); }); describe('integration', function () { - const [srv0] = startTestCluster(['--replicaset']); + const [srv0] = startTestCluster({ topology: 'replset' }); let serviceProvider: CliServiceProvider; let instanceState: ShellInstanceState; let mongo: Mongo; diff --git a/packages/shell-api/src/field-level-encryption.spec.ts b/packages/shell-api/src/field-level-encryption.spec.ts index d5848bb3a2..a1c868a66d 100644 --- a/packages/shell-api/src/field-level-encryption.spec.ts +++ b/packages/shell-api/src/field-level-encryption.spec.ts @@ -799,7 +799,7 @@ describe('Field Level Encryption', function () { beforeEach(async function () { dbname = `test_fle_${Date.now()}`; - uri = `${await testServer.connectionString()}/${dbname}`; + uri = await testServer.connectionString(); serviceProvider = await CliServiceProvider.connect( uri, dummyOptions, diff --git a/packages/shell-api/src/replica-set.spec.ts b/packages/shell-api/src/replica-set.spec.ts index baee646fbc..494ee033fb 100644 --- a/packages/shell-api/src/replica-set.spec.ts +++ b/packages/shell-api/src/replica-set.spec.ts @@ -857,10 +857,10 @@ describe('ReplicaSet', function () { const replId = 'rs0'; const [srv0, srv1, srv2, srv3] = startTestCluster( - ['--single', '--replSet', replId], - ['--single', '--replSet', replId], - ['--single', '--replSet', replId], - ['--single', '--replSet', replId] + { args: ['--replSet', replId] }, + { args: ['--replSet', replId] }, + { args: ['--replSet', replId] }, + { args: ['--replSet', replId] } ); let cfg: Partial; @@ -1103,9 +1103,9 @@ describe('ReplicaSet', function () { const replId = 'rspsa'; const [srv0, srv1, srv2] = startTestCluster( - ['--single', '--replSet', replId], - ['--single', '--replSet', replId], - ['--single', '--replSet', replId] + { args: ['--replSet', replId] }, + { args: ['--replSet', replId] }, + { args: ['--replSet', replId] } ); let serviceProvider: CliServiceProvider; diff --git a/packages/shell-api/src/session.spec.ts b/packages/shell-api/src/session.spec.ts index 2ad9c6f6ea..38ac2c3dd2 100644 --- a/packages/shell-api/src/session.spec.ts +++ b/packages/shell-api/src/session.spec.ts @@ -176,7 +176,7 @@ describe('Session', function () { }); }); describe('integration', function () { - const [srv0] = startTestCluster(['--replicaset']); + const [srv0] = startTestCluster({ topology: 'replset' }); let serviceProvider: CliServiceProvider; let instanceState: ShellInstanceState; let mongo: Mongo; diff --git a/packages/shell-api/src/shard.spec.ts b/packages/shell-api/src/shard.spec.ts index 58ede6ae81..1007aa7146 100644 --- a/packages/shell-api/src/shard.spec.ts +++ b/packages/shell-api/src/shard.spec.ts @@ -1792,10 +1792,13 @@ describe('Shard', function () { const shardId = 'rs-shard0'; const [mongos, rs0, rs1] = startTestCluster( - // --sharded 0 creates a setup without any initial shards - ['--replicaset', '--sharded', '0', '--csrs'], - ['--replicaset', '--name', `${shardId}-0`, '--shardsvr'], - ['--replicaset', '--name', `${shardId}-1`, '--shardsvr'] + // shards: 0 creates a setup without any initial shards + { topology: 'sharded', shards: 0 }, + { + topology: 'replset', + args: ['--replSet', `${shardId}-0`, '--shardsvr'], + }, + { topology: 'replset', args: ['--replSet', `${shardId}-1`, '--shardsvr'] } ); before(async function () { diff --git a/scripts/clear-mlaunch.ts b/scripts/clear-mlaunch.ts deleted file mode 100755 index dd1d920b0b..0000000000 --- a/scripts/clear-mlaunch.ts +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env ts-node -import { clearMlaunch } from '../testing/integration-testing-hooks'; -const killAllMongod = process.argv.includes('--killAllMongod'); -clearMlaunch({ killAllMongod }).catch(err => process.nextTick(() => { throw err; })); diff --git a/scripts/install-mlaunch.ts b/scripts/install-mlaunch.ts deleted file mode 100755 index 1d388959f0..0000000000 --- a/scripts/install-mlaunch.ts +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env ts-node -import { getMlaunchPath, ensureMongodAvailable } from '../testing/integration-testing-hooks'; -(async () => { - const mlaunchpath = await getMlaunchPath(); - const binarypath = await ensureMongodAvailable(); - console.log({ mlaunchpath, binarypath }); -})().catch(err => process.nextTick(() => { throw err; })); diff --git a/testing/integration-testing-hooks.ts b/testing/integration-testing-hooks.ts index 52209f2c37..e59a6994e7 100644 --- a/testing/integration-testing-hooks.ts +++ b/testing/integration-testing-hooks.ts @@ -1,15 +1,14 @@ import child_process from 'child_process'; -import crypto from 'crypto'; -import { once } from 'events'; import { promises as fs } from 'fs'; import { MongoClient, MongoClientOptions } from 'mongodb'; import path from 'path'; -import rimraf from 'rimraf'; import semver from 'semver'; import { URL } from 'url'; import { promisify } from 'util'; import which from 'which'; -import { downloadMongoDb } from '../packages/build/src/download-mongodb'; +import { ConnectionString } from 'mongodb-connection-string-url'; +import { MongoCluster, MongoClusterOptions } from 'mongodb-runner'; +import { downloadMongoDb } from '@mongodb-js/mongodb-downloader'; import { downloadCryptLibrary } from '../packages/build/src/packaging/download-crypt-library'; const execFile = promisify(child_process.execFile); @@ -21,18 +20,6 @@ function ciLog(...args: any[]) { } } -// Return the stat results or, if the file does not exist, `undefined`. -async function statIfExists(path: string): Promise | undefined> { - try { - return await fs.stat(path); - } catch (err: any) { - if (err.code === 'ENOENT') { - return undefined; - } - throw err; - } -} - // Return the path to the temporary directory and ensure that it exists. async function getTmpdir(): Promise { const tmpdir = path.resolve(__dirname, '..', 'tmp'); @@ -40,175 +27,6 @@ async function getTmpdir(): Promise { return tmpdir; } -async function tryExtensions(base: string): Promise<[ string, Error ]> { - let lastErr = new Error('unreachable'); - for (const ext of ['', '.exe', '.bat']) { - try { - await fs.stat(base + ext); - return [ base + ext, lastErr ]; - } catch (err: any) { - lastErr = err; - ciLog('File does not exist or is inaccessible', base + ext); - } - } - return [ '', lastErr ]; -} - -async function findPython3() { - try { - if (process.env.DISTRO_ID?.startsWith('windows-')) { - throw new Error('python3 on Windows in evergreen CI is broken but `python` is working python3'); - } - await which('python3'); - return 'python3'; - } catch { - ciLog('Did not find python3 in PATH'); - try { - // Fun fact on the side: Python 2.x writes the version to stderr, - // Python 3.x writes to stdout. - const { stdout } = await execFile('python', ['-V']); - if (stdout.includes('Python 3')) { - return 'python'; - } else { - throw new Error('python is not Python 3.x'); - } - } catch { - ciLog('Did not find python as Python 3.x in PATH'); - const pythonEnv = process.env.PYTHON; - if (pythonEnv) { - const { stdout } = await execFile(pythonEnv as string, ['-V']); - if (stdout.includes('Python 3')) { - return pythonEnv as string; - } - } - } - } - throw new Error('Could not find Python 3.x installation, install mlaunch manually'); -} - -// Get the path we use to spawn mlaunch, and potential environment variables -// necessary to run it successfully. This tries to install it locally if it -// cannot find an existing installation. -let mlaunchPath: { exec: string[], env: Record } | undefined; -export async function getMlaunchPath(): Promise<{ exec: string[], env: Record }> { - const tmpdir = await getTmpdir(); - if (mlaunchPath !== undefined) { - return mlaunchPath; - } - - try { - // If `mlaunch` is already in the PATH: Great, we're done. - const mlaunchBinary = await which('mlaunch'); - const filehandle: any = await fs.open(mlaunchBinary, 'r'); - try { - const startOfMlaunchFile = (await filehandle.read({ length: 100 })).buffer.toString('utf8').trim(); - if (startOfMlaunchFile.match(/^#!(\S+)python3?\r?\n/)) { - return mlaunchPath = { exec: [ await findPython3(), mlaunchBinary ], env: {} }; - } - } finally { - await filehandle.close(); - } - return mlaunchPath = { exec: [ mlaunchBinary ], env: {} }; - } catch { - ciLog('Did not find mlaunch in PATH'); - } - - // Figure out where python3 might live (python3, python, $PYTHON). - const python = await findPython3(); - - // Install mlaunch, preferably locally and otherwise attempt to do so globally. - try { - const mlaunchPy = path.join(tmpdir, 'bin', 'mlaunch'); - let [ exec ] = await tryExtensions(mlaunchPy); - if (exec) { - return mlaunchPath = { exec: [ python, exec ], env: { PYTHONPATH: tmpdir } }; - } - ciLog('Trying to install mlaunch in ', tmpdir); - // Pin pymongo to 3.12.2 because mlaunch does not seem to be compatible - // with 4.0, see https://jira.mongodb.org/browse/MONGOSH-1072 - // Also pin mlaunch to 1.6.4, since we have Python 3.6 in CI and 1.7.0 - // drops Python 3.6 support. - await execFile('pip3', ['install', '--no-cache-dir', '--target', tmpdir, 'mtools[mlaunch]==1.6.4', 'pymongo==3.12.2']); - ciLog('Installation complete'); - [ exec ] = await tryExtensions(mlaunchPy); - if (exec) { - return mlaunchPath = { exec: [ python, exec ], env: { PYTHONPATH: tmpdir } }; - } - } catch {} - - // Figure out the most likely target path for pip3 --user and use mlaunch - // from there. - const pythonBase = (await execFile(python, ['-m', 'site', '--user-base'])).stdout.trim(); - const pythonPath = (await execFile(python, ['-m', 'site', '--user-site'])).stdout.trim(); - const mlaunchExec = path.join(pythonBase, 'bin', 'mlaunch'); - { - const [ exec ] = await tryExtensions(mlaunchExec); - if (exec) { - return { exec: [ python, exec ], env: { PYTHONPATH: pythonPath } }; - } - } - ciLog('Trying to install mlaunch in ', { pythonBase, pythonPath }); - await execFile('pip3', ['install', '--no-cache-dir', '--user', 'mtools[mlaunch]']); - ciLog('Installation complete'); - const [ exec, lastErr ] = await tryExtensions(mlaunchExec); - if (exec) { - return { exec: [ python, exec ], env: { PYTHONPATH: pythonPath } }; - } - throw lastErr; -} - -type MlaunchCommand = 'init' | 'start' | 'stop' | 'list' | 'kill'; -// Run a specific mlaunch command, trying to ensure that `mlaunch` is accessible -// and installing it if necessary. -async function execMlaunch(command: MlaunchCommand, ...args: string[]): Promise { - const mlaunchPath = await getMlaunchPath(); - - // command could be part of args, but the extra typechecking here has helped - // me at least once. - const fullCmd = [...mlaunchPath.exec, command, ...args]; - // console.info('Running command', fullCmd.join(' ')); - const proc = child_process.spawn(fullCmd[0], fullCmd.slice(1), { - env: { ...process.env, ...mlaunchPath.env }, - stdio: 'inherit' - }); - await once(proc, 'exit'); - // console.info('Successfully ran command', fullCmd.join(' ')); -} - -// Remove all potential leftover mlaunch instances. -export async function clearMlaunch({ killAllMongod = false } = {}) { - if (killAllMongod) { - for (const proc of ['mongod', 'mongos']) { - try { - if (process.platform === 'win32') { - await execFile('taskkill', ['/IM', `${proc}.exe`, '/F']); - } else { - await execFile('killall', [proc]); - } - } catch (err: any) { - console.warn(`Cleaning up ${proc} instances failed:`, err); - } - } - } - const tmpdir = await getTmpdir(); - for await (const { name } of await fs.opendir(tmpdir)) { - if (name.startsWith('mlaunch-')) { - const fullPath = path.join(tmpdir, name); - try { - await execMlaunch('kill', '--dir', fullPath); - } catch (err: any) { - console.warn(`mlaunch kill in ${fullPath} failed:`, err); - } - try { - await promisify(rimraf)(fullPath); - } catch (err: any) { - console.warn(`Removing ${fullPath} failed:`, err); - } - } - } - await promisify(rimraf)(path.join(tmpdir, '.current-port')); -} - // Represents one running test server instance. export class MongodSetup { _connectionString: Promise; @@ -236,21 +54,29 @@ export class MongodSetup { throw new Error('Server not managed'); } - async connectionString(): Promise { - return this._connectionString; + async connectionString(searchParams: Partial> = {}, uriOptions: Partial = {}): Promise { + if (Object.keys(searchParams).length + Object.keys(uriOptions).length === 0) { + return this._connectionString; + } + + const url = await this.connectionStringUrl(); + for (const [key, value] of Object.entries(searchParams)) + url.searchParams.set(key, value); + for (const [key, value] of Object.entries(uriOptions)) + (url as any)[key] = value; + return url.toString(); } - async host(): Promise { - return new URL(await this.connectionString()).hostname; + async connectionStringUrl(): Promise { + return new ConnectionString(await this.connectionString()); } async port(): Promise { - // 27017 is the default port for mongodb:// URLs. - return new URL(await this.connectionString()).port ?? '27017'; + return (await this.hostport()).split(':').reverse()[0]; } async hostport(): Promise { - return `${await this.host()}:${await this.port()}`; + return (await this.connectionStringUrl()).hosts[0]; } async serverVersion(): Promise { @@ -295,77 +121,31 @@ export class MongodSetup { } } -// Spawn mlaunch with a specific set of arguments. -export class MlaunchSetup extends MongodSetup { - _args: string[]; - _mlaunchdir = ''; +// Spawn a mongodb-runner-managed instance with a specific set of arguments. +export class MongoRunnerSetup extends MongodSetup { + _opts: Partial; + _cluster: MongoCluster | undefined; - constructor(args: string[] = []) { + constructor(opts: Partial = {}) { super(); - this._args = args; + this._opts = opts; } async start(): Promise { - if (this._mlaunchdir) return; - const random = (await promisify(crypto.randomBytes)(16)).toString('hex'); - const tag = `${process.pid}-${random}`; - - const tmpdir = await getTmpdir(); - this._mlaunchdir = path.join(tmpdir, `mlaunch-${tag}`); - - const args = this._args; - if (!args.includes('--replicaset') && !args.includes('--single')) { - args.push('--single'); - } - - let port: number; - if (args.includes('--port')) { - const index = args.indexOf('--port'); - port = +args.splice(index, 2)[1]; - } else { - // If no port was specified, we pick one in the range [30000, 40000]. - // We pick by writing to a file, looking up the index at which we wrote, - // and adding that to the base port, so that there is a low likelihood of - // port collisions between different test runs even when two tests call - // startMlaunch() at the same time. - // Ideally, we would handle failures from port conflicts that occur when - // mlaunch starts, but we don't currently have access to that information - // until .start() is called. - const portfile = path.join(tmpdir, '.current-port'); - await fs.appendFile(portfile, `${tag}\n`); - const portfileContent = (await fs.readFile(portfile, 'utf8')).split('\n'); - const writeIndex = portfileContent.indexOf(tag); - if (writeIndex === -1) { - throw new Error(`Could not figure out port number, ${portfile} may be corrupt`); - } - port = 30000 + (writeIndex * 30) % 10000; - } - - // Make sure mongod and mongos are accessible - const binarypath = await ensureMongodAvailable(); - if (binarypath) { - args.unshift('--binarypath', binarypath); - this._bindir = binarypath; - } - - if (await statIfExists(this._mlaunchdir)) { - // There might be leftovers from previous runs. Remove them. - await execMlaunch('kill', '--dir', this._mlaunchdir); - await promisify(rimraf)(this._mlaunchdir); - } - await fs.mkdir(this._mlaunchdir, { recursive: true }); - await execMlaunch('init', '--dir', this._mlaunchdir, '--port', `${port}`, ...args); - this._setConnectionString(`mongodb://localhost:${port}`); + if (this._cluster) return; + this._cluster = await MongoCluster.start({ + topology: 'standalone', + tmpDir: await getTmpdir(), + version: process.env.MONGOSH_SERVER_TEST_VERSION, + ...this._opts + }) + + this._setConnectionString(this._cluster.connectionString); } async stop(): Promise { - if (!this._mlaunchdir) return; - await execMlaunch('stop', '--dir', this._mlaunchdir); - try { - await promisify(rimraf)(this._mlaunchdir); - } catch (err: any) { - console.error(`Cannot remove directory ${this._mlaunchdir}`, err); - } + await this._cluster?.close(); + this._cluster = undefined; } } @@ -376,23 +156,6 @@ async function getInstalledMongodVersion(): Promise { return version; } -export async function ensureMongodAvailable(mongodVersion = process.env.MONGOSH_SERVER_TEST_VERSION): Promise { - try { - if (/-community/.test(String(mongodVersion))) { - console.info(`Explicitly requesting community server with ${mongodVersion}, downloading...`); - throw new Error(); - } - const version = await getInstalledMongodVersion(); - if (mongodVersion && !semver.satisfies(version, mongodVersion)) { - console.info(`global mongod is ${version}, wanted ${mongodVersion}, downloading...`); - throw new Error(); - } - return null; - } catch { - return await downloadMongoDb(path.resolve(__dirname, '..', 'tmp'), mongodVersion); - } -} - export async function downloadCurrentCryptSharedLibrary(): Promise { if (process.platform === 'linux') { return await downloadCryptLibrary(`linux-${process.arch.replace('ppc64', 'ppc64le')}` as any); @@ -419,19 +182,19 @@ export async function downloadCurrentCryptSharedLibrary(): Promise { * @returns {MongodSetup} - Object with information about the started server. */ let sharedSetup : MongodSetup | null = null; -export function startTestServer(shareMode: 'shared' | 'not-shared', ...args: string[]): MongodSetup { +export function startTestServer(shareMode: 'shared' | 'not-shared', args: Partial = {}): MongodSetup { if (shareMode === 'shared' && process.env.MONGOSH_TEST_SERVER_URL) { return new MongodSetup(process.env.MONGOSH_TEST_SERVER_URL); } let server : MongodSetup; if (shareMode === 'shared') { - if (args.length > 0) { + if (Object.keys(args).length > 0) { throw new Error('Cannot specify arguments for shared mongod'); } - server = sharedSetup ?? (sharedSetup = new MlaunchSetup()); + server = sharedSetup ?? (sharedSetup = new MongoRunnerSetup()); } else { - server = new MlaunchSetup(args); + server = new MongoRunnerSetup(args); } before(async function() { @@ -459,8 +222,8 @@ global.after?.(async function() { // The same as startTestServer(), except that this starts multiple servers // in parallel in the same before() call. -export function startTestCluster(...argLists: string[][]): MongodSetup[] { - const servers = argLists.map(args => new MlaunchSetup(args)); +export function startTestCluster(...argLists: Partial[]): MongodSetup[] { + const servers = argLists.map(args => new MongoRunnerSetup(args)); before(async function() { this.timeout(90_000 + 30_000 * servers.length);