diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2f22ffff28..c4b57b543e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -79,17 +79,6 @@ functions: export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" export PIP_QUIET=1 - ENSURE_TOOLCHAIN_PYTHON_BINARY: | - # Make sure PYTHON_BINARY is set to a suitable toolchain python. - if [ -z "$PYTHON_BINARY" ]; then - if [ "$(uname -s)" = "Darwin" ]; then - export PYTHON_BINARY=/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - export PYTHON_BINARY=/cygdrive/c/python/Python39/python - else - export PYTHON_BINARY=/opt/python/3.9/bin/python3 - fi - fi EOT # Load the expansion file to make an evergreen variable with the current unique version @@ -502,12 +491,8 @@ functions: export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" fi - export PYTHON_BINARY=${PYTHON_BINARY} - if [ -z "$PYTHON_BINARY" ]; then - export PYTHON_BINARY=${python3_binary} - fi - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ + PYTHON_BINARY=${PYTHON_BINARY} \ C_EXTENSIONS=${C_EXTENSIONS} \ COVERAGE=${COVERAGE} \ COMPRESSORS=${COMPRESSORS} \ @@ -678,7 +663,6 @@ functions: include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} - ${ENSURE_TOOLCHAIN_PYTHON_BINARY} bash .evergreen/run-mongodb-oidc-test.sh "run aws auth test with aws credentials as environment variables": @@ -1630,8 +1614,6 @@ tasks: tags: ["no-server"] commands: - func: "run tests" - vars: - PYTHON_BINARY: /opt/python/3.7/bin/python3 - name: "atlas-connect" tags: ["atlas-connect"] @@ -2108,7 +2090,6 @@ tasks: vars: VERSION: "latest" TOPOLOGY: "server" - PYTHON_BINARY: "/opt/mongodbtoolchain/v3/bin/python3" - func: "run tests" # }}} - name: "coverage-report" @@ -2260,7 +2241,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100 display_name: "macOS 11.00" @@ -2269,7 +2249,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100-arm64 display_name: "macOS 11.00 Arm64" @@ -2278,21 +2257,18 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: rhel7 display_name: "RHEL 7.x" run_on: rhel79-small batchtime: 10080 # 7 days variables: - python3_binary: "/opt/python/3.8/bin/python3" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel8 display_name: "RHEL 8.x" run_on: rhel87-small batchtime: 10080 # 7 days variables: - python3_binary: "/opt/python/3.8/bin/python3" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel80-fips display_name: "RHEL 8.0 FIPS" @@ -2304,14 +2280,10 @@ axes: display_name: "Ubuntu 22.04" run_on: ubuntu2204-small batchtime: 10080 # 7 days - variables: - python3_binary: python3 - id: ubuntu-18.04 display_name: "Ubuntu 18.04" run_on: ubuntu1804-small batchtime: 10080 # 7 days - variables: - python3_binary: python3 - id: rhel83-zseries display_name: "RHEL 8.3 (zSeries)" run_on: rhel83-zseries-small @@ -2334,7 +2306,6 @@ axes: skip_ECS_auth_test: true skip_EC2_auth_test: true skip_web_identity_auth_test: true - python3_binary: "C:/python/Python38/python.exe" venv_bin_dir: "Scripts" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz @@ -2476,14 +2447,6 @@ axes: variables: PYTHON_BINARY: "/opt/python/pypy3.10/bin/pypy3" - - id: python-version-mac - display_name: "Python" - values: - - id: "system-python3" - display_name: "Python3" - variables: - PYTHON_BINARY: "python3" - - id: python-version-windows display_name: "Python" values: @@ -3166,7 +3129,6 @@ buildvariants: - matrix_name: "aws-auth-test-mac" matrix_spec: platform: [macos-1014] - python-version-mac: ["system-python3"] display_name: "MONGODB-AWS Auth ${platform} ${python-version-mac}" tasks: - name: "aws-auth-test-4.4" diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 36c4591469..5c37e33434 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -18,6 +18,9 @@ echo "Running MONGODB-AWS authentication tests" # ensure no secrets are printed in log files set +x +. ./.evergreen/utils.sh +PYTHON_BINARY=$(find_python3) + # Try to source exported AWS Secrets if [ -f ./secrets-export.sh ]; then source ./secrets-export.sh diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index cc92a50658..75fafb448b 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -13,13 +13,13 @@ fi # Get the drivers secrets. Use an existing secrets file first. if [ ! -f "./secrets-export.sh" ]; then - bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc + bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc fi source ./secrets-export.sh # # If the file did not have our creds, get them from the vault. if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then - bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc + bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc source ./secrets-export.sh fi diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 683b3b59f9..808787d63c 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -1,18 +1,11 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail +set -x . .evergreen/utils.sh if [ -z "$PYTHON_BINARY" ]; then - # Use Python 3 from the server toolchain to test on ARM, POWER or zSeries if a - # system python3 doesn't exist or exists but is older than 3.7. - if is_python_37 "$(command -v python3)"; then - PYTHON_BINARY=$(command -v python3) - elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then - PYTHON_BINARY=$(command -v /opt/mongodbtoolchain/v3/bin/python3) - else - echo "Cannot test without python3.7+ installed!" - fi + PYTHON_BINARY=$(find_python3) fi if $PYTHON_BINARY -m tox --version; then diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index b3fa45e4b6..35005c0d6a 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -2,6 +2,40 @@ set -o xtrace +find_python3() { + PYTHON="" + # Add a fallback system python3 if it is available and Python 3.7+. + if is_python_37 "$(command -v python3)"; then + PYTHON="$(command -v python3)" + fi + # Find a suitable toolchain version, if available. + if [ "$(uname -s)" = "Darwin" ]; then + # macos 11.00 + if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then + PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" + # macos 10.14 + elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then + PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" + fi + elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + PYTHON="C:/python/Python37/python.exe" + else + # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.7+. + if [ -f "/opt/python/3.7/bin/python3" ]; then + PYTHON="/opt/python/3.7/bin/python3" + elif is_python_37 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then + PYTHON="/opt/mongodbtoolchain/v4/bin/python3" + elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then + PYTHON="/opt/mongodbtoolchain/v3/bin/python3" + fi + fi + if [ -z "$PYTHON" ]; then + echo "Cannot test without python3.7+ installed!" + exit 1 + fi + echo "$PYTHON" +} + # Usage: # createvirtualenv /path/to/python /output/path/for/venv # * param1: Python binary to use for the virtualenv @@ -9,17 +43,18 @@ set -o xtrace createvirtualenv () { PYTHON=$1 VENVPATH=$2 - if $PYTHON -m virtualenv --version; then + # Prefer venv + VENV="$PYTHON -m venv" + if [ "$(uname -s)" = "Darwin" ]; then VIRTUALENV="$PYTHON -m virtualenv" - elif $PYTHON -m venv -h > /dev/null; then - # System virtualenv might not be compatible with the python3 on our path - VIRTUALENV="$PYTHON -m venv" else - echo "Cannot test without virtualenv" - exit 1 + VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv") + VIRTUALENV="$VIRTUALENV -p $PYTHON" + fi + if ! $VENV $VENVPATH 2>/dev/null; then + # Workaround for bug in older versions of virtualenv. + $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH fi - # Workaround for bug in older versions of virtualenv. - $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH if [ "Windows_NT" = "$OS" ]; then # Workaround https://bugs.python.org/issue32451: # mongovenv/Scripts/activate: line 3: $'\r': command not found @@ -31,7 +66,7 @@ createvirtualenv () { export PIP_QUIET=1 python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel tox + python -m pip install --upgrade setuptools tox } # Usage: diff --git a/test/test_index_management.py b/test/test_index_management.py index e64fe0c27e..9db9a22aea 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -67,9 +67,13 @@ def setUpClass(cls) -> None: password = os.environ["DB_PASSWORD"] cls.client = MongoClient(url, username=username, password=password) cls.client.drop_database(_NAME) - cls.addClassCleanup(cls.client.close) cls.db = cls.client.test_search_index_prose + @classmethod + def tearDownClass(cls): + cls.client.drop_database(_NAME) + cls.client.close() + def wait_for_ready(self, coll, name=_NAME, predicate=None): """Wait for a search index to be ready.""" indices: list[Mapping[str, Any]] = []