From 19dd40b822a0b4e1c6df623baf80f7bfd3234425 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 04:15:39 -0500 Subject: [PATCH 01/35] clean up python binary handling --- .evergreen/config.yml | 44 ++---------------------- .evergreen/run-mongodb-oidc-test.sh | 8 +++-- .evergreen/tox.sh | 17 +++++----- .evergreen/utils.sh | 52 ++++------------------------- 4 files changed, 22 insertions(+), 99 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0277cad140..b447be4db1 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -79,17 +79,6 @@ functions: export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" export PIP_QUIET=1 - ENSURE_TOOLCHAIN_PYTHON_BINARY: | - # Make sure PYTHON_BINARY is set to a suitable toolchain python. - if [ -z "$PYTHON_BINARY" ]; then - if [ "$(uname -s)" = "Darwin" ]; then - export PYTHON_BINARY=/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 - elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - export PYTHON_BINARY=/cygdrive/c/python/Python39/python - else - export PYTHON_BINARY=/opt/python/3.9/bin/python3 - fi - fi EOT # Load the expansion file to make an evergreen variable with the current unique version @@ -108,7 +97,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone --branch cleanup-python-binary-handlin https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config @@ -369,7 +358,6 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace - export PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-mockupdb "run doctests": @@ -380,7 +368,7 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace - PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test + bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test "run tests": # If testing FLE, start the KMS mock servers, first create the virtualenv. @@ -502,11 +490,6 @@ functions: export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" fi - export PYTHON_BINARY=${PYTHON_BINARY} - if [ -z "$PYTHON_BINARY" ]; then - export PYTHON_BINARY=${python3_binary} - fi - GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ C_EXTENSIONS=${C_EXTENSIONS} \ COVERAGE=${COVERAGE} \ @@ -678,7 +661,6 @@ functions: include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} - ${ENSURE_TOOLCHAIN_PYTHON_BINARY} bash .evergreen/run-mongodb-oidc-test.sh "run aws auth test with aws credentials as environment variables": @@ -1637,8 +1619,6 @@ tasks: tags: ["no-server"] commands: - func: "run tests" - vars: - PYTHON_BINARY: /opt/python/3.7/bin/python3 - name: "atlas-connect" tags: ["atlas-connect"] @@ -2115,7 +2095,6 @@ tasks: vars: VERSION: "latest" TOPOLOGY: "server" - PYTHON_BINARY: "/opt/mongodbtoolchain/v3/bin/python3" - func: "run tests" # }}} - name: "coverage-report" @@ -2262,7 +2241,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100 display_name: "macOS 11.00" @@ -2271,7 +2249,6 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100-arm64 display_name: "macOS 11.00 Arm64" @@ -2280,21 +2257,18 @@ axes: skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: rhel7 display_name: "RHEL 7.x" run_on: rhel79-small batchtime: 10080 # 7 days variables: - python3_binary: "/opt/python/3.8/bin/python3" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel8 display_name: "RHEL 8.x" run_on: rhel87-small batchtime: 10080 # 7 days variables: - python3_binary: "/opt/python/3.8/bin/python3" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz - id: rhel80-fips display_name: "RHEL 8.0 FIPS" @@ -2306,14 +2280,10 @@ axes: display_name: "Ubuntu 22.04" run_on: ubuntu2204-small batchtime: 10080 # 7 days - variables: - python3_binary: python3 - id: ubuntu-18.04 display_name: "Ubuntu 18.04" run_on: ubuntu1804-small batchtime: 10080 # 7 days - variables: - python3_binary: python3 - id: rhel83-zseries display_name: "RHEL 8.3 (zSeries)" run_on: rhel83-zseries-small @@ -2336,7 +2306,6 @@ axes: skip_ECS_auth_test: true skip_EC2_auth_test: true skip_web_identity_auth_test: true - python3_binary: "C:/python/Python38/python.exe" venv_bin_dir: "Scripts" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz @@ -2478,14 +2447,6 @@ axes: variables: PYTHON_BINARY: "/opt/python/pypy3.10/bin/pypy3" - - id: python-version-mac - display_name: "Python" - values: - - id: "system-python3" - display_name: "Python3" - variables: - PYTHON_BINARY: "python3" - - id: python-version-windows display_name: "Python" values: @@ -3168,7 +3129,6 @@ buildvariants: - matrix_name: "aws-auth-test-mac" matrix_spec: platform: [macos-1014] - python-version-mac: ["system-python3"] display_name: "MONGODB-AWS Auth ${platform} ${python-version-mac}" tasks: - name: "aws-auth-test-4.4" diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index cc92a50658..b3a741e82b 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -13,20 +13,21 @@ fi # Get the drivers secrets. Use an existing secrets file first. if [ ! -f "./secrets-export.sh" ]; then - bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc + bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc fi source ./secrets-export.sh # # If the file did not have our creds, get them from the vault. if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then - bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc + bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc source ./secrets-export.sh fi # Make the OIDC tokens. set -x pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc -. ./oidc_get_tokens.sh +. ./activate-authoidcvenv.sh +python oidc_get_tokens.py popd # Set up variables and run the test. @@ -45,4 +46,5 @@ fi export TEST_AUTH_OIDC=1 export COVERAGE=1 export AUTH="auth" +export PYTHON_BINARY=python bash ./.evergreen/tox.sh -m test-eg diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 683b3b59f9..19f528e7b1 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -1,18 +1,17 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail +# Make sure DRIVERS_TOOLS is set. +if [ -z "$DRIVERS_TOOLS" ]; then + echo "Must specify DRIVERS_TOOLS" + exit 1 +fi + +. $DRIVERS_TOOLS/find-python3.sh . .evergreen/utils.sh if [ -z "$PYTHON_BINARY" ]; then - # Use Python 3 from the server toolchain to test on ARM, POWER or zSeries if a - # system python3 doesn't exist or exists but is older than 3.7. - if is_python_37 "$(command -v python3)"; then - PYTHON_BINARY=$(command -v python3) - elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then - PYTHON_BINARY=$(command -v /opt/mongodbtoolchain/v3/bin/python3) - else - echo "Cannot test without python3.7+ installed!" - fi + PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi if $PYTHON_BINARY -m tox --version; then diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index b3fa45e4b6..0155b37cce 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -2,37 +2,14 @@ set -o xtrace -# Usage: -# createvirtualenv /path/to/python /output/path/for/venv -# * param1: Python binary to use for the virtualenv -# * param2: Path to the virtualenv to create -createvirtualenv () { - PYTHON=$1 - VENVPATH=$2 - if $PYTHON -m virtualenv --version; then - VIRTUALENV="$PYTHON -m virtualenv" - elif $PYTHON -m venv -h > /dev/null; then - # System virtualenv might not be compatible with the python3 on our path - VIRTUALENV="$PYTHON -m venv" - else - echo "Cannot test without virtualenv" - exit 1 - fi - # Workaround for bug in older versions of virtualenv. - $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH - if [ "Windows_NT" = "$OS" ]; then - # Workaround https://bugs.python.org/issue32451: - # mongovenv/Scripts/activate: line 3: $'\r': command not found - dos2unix $VENVPATH/Scripts/activate || true - . $VENVPATH/Scripts/activate - else - . $VENVPATH/bin/activate - fi +# Make sure DRIVERS_TOOLS is set. +if [ -z "$DRIVERS_TOOLS" ]; then + echo "Must specify DRIVERS_TOOLS" + exit 1 +fi - export PIP_QUIET=1 - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel tox -} +# Import createvirtualenv from $DRIVERS_TOOLS +. $DRIVERS_TOOLS/venv-utils.sh # Usage: # testinstall /path/to/python /path/to/.whl ["no-virtualenv"] @@ -60,18 +37,3 @@ testinstall () { rm -rf venvtestinstall fi } - -# Function that returns success if the provided Python binary is version 3.7 or later -# Usage: -# is_python_37 /path/to/python -# * param1: Python binary -is_python_37() { - if [ -z "$1" ]; then - return 1 - elif $1 -c "import sys; exit(sys.version_info[:2] < (3, 7))"; then - # runs when sys.version_info[:2] >= (3, 7) - return 0 - else - return 1 - fi -} From dac5f7a6f9fc8f142a1dab38c84e078212f8cbcb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 04:38:59 -0500 Subject: [PATCH 02/35] fix paths --- .evergreen/tox.sh | 2 +- .evergreen/utils.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 19f528e7b1..66e51a7e97 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -7,7 +7,7 @@ if [ -z "$DRIVERS_TOOLS" ]; then exit 1 fi -. $DRIVERS_TOOLS/find-python3.sh +. $DRIVERS_TOOLS/.evergreen/find-python3.sh . .evergreen/utils.sh if [ -z "$PYTHON_BINARY" ]; then diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 0155b37cce..1868605d17 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -9,7 +9,7 @@ if [ -z "$DRIVERS_TOOLS" ]; then fi # Import createvirtualenv from $DRIVERS_TOOLS -. $DRIVERS_TOOLS/venv-utils.sh +. $DRIVERS_TOOLS/.evergreen/venv-utils.sh # Usage: # testinstall /path/to/python /path/to/.whl ["no-virtualenv"] From f6c2a81820364144c1a14fa6d61c30fed1442bfa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 04:56:10 -0500 Subject: [PATCH 03/35] clean up --- .evergreen/build-mac.sh | 2 +- .evergreen/config.yml | 2 +- .evergreen/tox.sh | 19 +++++++++---------- .evergreen/utils.sh | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh index 4e8be8cf58..afea71a918 100755 --- a/.evergreen/build-mac.sh +++ b/.evergreen/build-mac.sh @@ -13,7 +13,7 @@ VERSION=${VERSION:-3.10} PYTHON=/Library/Frameworks/Python.framework/Versions/$VERSION/bin/python3 rm -rf build -createvirtualenv $PYTHON releasevenv +venvcreate $PYTHON releasevenv python -m pip install build python -m build --wheel . deactivate || true diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b447be4db1..3a685f8f2a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2150,7 +2150,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 PYTHON_BINARY=python3 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 66e51a7e97..fbff4a72c1 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -1,16 +1,14 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail -# Make sure DRIVERS_TOOLS is set. -if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 -fi - -. $DRIVERS_TOOLS/.evergreen/find-python3.sh -. .evergreen/utils.sh - if [ -z "$PYTHON_BINARY" ]; then + # Make sure DRIVERS_TOOLS is set. + if [ -z "$DRIVERS_TOOLS" ]; then + echo "Must specify DRIVERS_TOOLS" + exit 1 + fi + + . $DRIVERS_TOOLS/.evergreen/find-python3.sh PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi @@ -19,7 +17,8 @@ if $PYTHON_BINARY -m tox --version; then $PYTHON_BINARY -m tox "$@" } else # No toolchain present, set up virtualenv before installing tox - createvirtualenv "$PYTHON_BINARY" toxenv + . .evergreen/utils.sh + venvcreate "$PYTHON_BINARY" toxenv trap "deactivate; rm -rf toxenv" EXIT HUP python -m pip install -q tox run_tox() { diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 1868605d17..afca45267b 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -8,7 +8,7 @@ if [ -z "$DRIVERS_TOOLS" ]; then exit 1 fi -# Import createvirtualenv from $DRIVERS_TOOLS +# Import venvcreate from $DRIVERS_TOOLS . $DRIVERS_TOOLS/.evergreen/venv-utils.sh # Usage: From c416506a43016f390f80609417dc85eec7786d65 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 05:24:15 -0500 Subject: [PATCH 04/35] clean up --- .evergreen/run-mongodb-oidc-test.sh | 6 +++--- .evergreen/tox.sh | 3 ++- .evergreen/utils.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index b3a741e82b..662360a3ec 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -23,11 +23,11 @@ if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then source ./secrets-export.sh fi -# Make the OIDC tokens. +# Make the OIDC tokens - also activate the oidc venv. set -x pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc -. ./activate-authoidcvenv.sh -python oidc_get_tokens.py +. oidc_get_tokens.sh +pip install -q tox popd # Set up variables and run the test. diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index fbff4a72c1..4f3d367b01 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -12,7 +12,8 @@ if [ -z "$PYTHON_BINARY" ]; then PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi -if $PYTHON_BINARY -m tox --version; then +TOX=$($PYTHON_BINARY -m tox --version || "") +if [ -n "$TOX"]; then run_tox() { $PYTHON_BINARY -m tox "$@" } diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index afca45267b..9e6575b04f 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -22,7 +22,7 @@ testinstall () { NO_VIRTUALENV=$3 if [ -z "$NO_VIRTUALENV" ]; then - createvirtualenv $PYTHON venvtestinstall + venvcreate $PYTHON venvtestinstall PYTHON=python fi From be17b0dab5384a12fc07040169b31e7fa16360ef Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 05:41:02 -0500 Subject: [PATCH 05/35] clean up --- .evergreen/tox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 4f3d367b01..c698cff565 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -12,7 +12,7 @@ if [ -z "$PYTHON_BINARY" ]; then PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi -TOX=$($PYTHON_BINARY -m tox --version || "") +TOX=$($PYTHON_BINARY -m tox --version || true) if [ -n "$TOX"]; then run_tox() { $PYTHON_BINARY -m tox "$@" From 90f62c44ac50747e0cd31d8363778a28e47ae9c0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 05:59:14 -0500 Subject: [PATCH 06/35] clean up --- .evergreen/config.yml | 2 +- .evergreen/tox.sh | 2 +- .evergreen/utils.sh | 17 ++++++++--------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3a685f8f2a..4cddcf70ce 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2203,7 +2203,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ + AZUREKMS_CMD="PYTHON_BINARY=python3 KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index c698cff565..934d538e9e 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -12,7 +12,7 @@ if [ -z "$PYTHON_BINARY" ]; then PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi -TOX=$($PYTHON_BINARY -m tox --version || true) +TOX=$($PYTHON_BINARY -m tox --version 2>/dev/null || true) if [ -n "$TOX"]; then run_tox() { $PYTHON_BINARY -m tox "$@" diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 9e6575b04f..91cae32fea 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -2,15 +2,6 @@ set -o xtrace -# Make sure DRIVERS_TOOLS is set. -if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 -fi - -# Import venvcreate from $DRIVERS_TOOLS -. $DRIVERS_TOOLS/.evergreen/venv-utils.sh - # Usage: # testinstall /path/to/python /path/to/.whl ["no-virtualenv"] # * param1: Python binary to test @@ -22,6 +13,14 @@ testinstall () { NO_VIRTUALENV=$3 if [ -z "$NO_VIRTUALENV" ]; then + # Make sure DRIVERS_TOOLS is set. + if [ -z "$DRIVERS_TOOLS" ]; then + echo "Must specify DRIVERS_TOOLS" + exit 1 + fi + + # Import venvcreate from $DRIVERS_TOOLS + . $DRIVERS_TOOLS/.evergreen/venv-utils.sh venvcreate $PYTHON venvtestinstall PYTHON=python fi From bc24f0bce22f28da4f9e05c0f14e7f9d9e3f454c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 06:03:35 -0500 Subject: [PATCH 07/35] clean up --- .evergreen/utils.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 91cae32fea..beed9cf149 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -2,6 +2,11 @@ set -o xtrace +if [ -n "$DRIVERS_TOOLS" ]; then + # Import venvcreate from $DRIVERS_TOOLS + . $DRIVERS_TOOLS/.evergreen/venv-utils.sh +fi + # Usage: # testinstall /path/to/python /path/to/.whl ["no-virtualenv"] # * param1: Python binary to test @@ -18,9 +23,6 @@ testinstall () { echo "Must specify DRIVERS_TOOLS" exit 1 fi - - # Import venvcreate from $DRIVERS_TOOLS - . $DRIVERS_TOOLS/.evergreen/venv-utils.sh venvcreate $PYTHON venvtestinstall PYTHON=python fi From c3c9c3a8a9d3b0eb50223dc561dba1d3f8ef1fd2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 06:09:32 -0500 Subject: [PATCH 08/35] debug --- .evergreen/tox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 934d538e9e..577af40f8a 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -1,5 +1,6 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail +set -x if [ -z "$PYTHON_BINARY" ]; then # Make sure DRIVERS_TOOLS is set. From e02b0ba5366aff6b47abb7e940f15fb41a081664 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 06:14:46 -0500 Subject: [PATCH 09/35] clean up --- .evergreen/tox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 577af40f8a..e749489648 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -13,8 +13,8 @@ if [ -z "$PYTHON_BINARY" ]; then PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi -TOX=$($PYTHON_BINARY -m tox --version 2>/dev/null || true) -if [ -n "$TOX"]; then +TOX=$($PYTHON_BINARY -m tox --version 2>/dev/null || echo "") +if [ -n "$TOX" ]; then run_tox() { $PYTHON_BINARY -m tox "$@" } From db2ab54342b20cc22fbb9bc91c76b5bea82353ca Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 06:35:35 -0500 Subject: [PATCH 10/35] clean up --- .evergreen/tox.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index e749489648..57d80db8b7 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -19,9 +19,13 @@ if [ -n "$TOX" ]; then $PYTHON_BINARY -m tox "$@" } else # No toolchain present, set up virtualenv before installing tox - . .evergreen/utils.sh - venvcreate "$PYTHON_BINARY" toxenv - trap "deactivate; rm -rf toxenv" EXIT HUP + # If DRIVERS_TOOLS is set, we are on an EVG host, use a virtual env. + # Otherwise we are on temporary host, we can install directly. + if [ -n "$DRIVERS_TOOLS" ]; then + . .evergreen/utils.sh + venvcreate "$PYTHON_BINARY" toxenv + trap "deactivate; rm -rf toxenv" EXIT HUP + fi python -m pip install -q tox run_tox() { python -m tox "$@" From 870f531f3e5430ac806b3cba25c9abc4da9d30e1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 06:39:56 -0500 Subject: [PATCH 11/35] fix aws --- .evergreen/run-mongodb-aws-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 36c4591469..dc215b9a86 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -30,6 +30,8 @@ if [ -n "$1" ]; then cd - fi +export PYTHON_BINARY=python + if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" fi From 100455344c4c5e9d0fc26913414f3b5dde76da47 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 07:21:41 -0500 Subject: [PATCH 12/35] use python3 --- .evergreen/run-mongodb-aws-test.sh | 2 +- .evergreen/tox.sh | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index dc215b9a86..a0adc53ce5 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -30,7 +30,7 @@ if [ -n "$1" ]; then cd - fi -export PYTHON_BINARY=python +export PYTHON_BINARY=python3 if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 57d80db8b7..95523167de 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -3,14 +3,13 @@ set -o errexit # Exit the script with error if any of the commands fail set -x if [ -z "$PYTHON_BINARY" ]; then - # Make sure DRIVERS_TOOLS is set. + # If DRIVERS_TOOLS is not set, use the current python3. if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 + PYTHON_BINARY=python3 + else + . $DRIVERS_TOOLS/.evergreen/find-python3.sh + PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi - - . $DRIVERS_TOOLS/.evergreen/find-python3.sh - PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 fi TOX=$($PYTHON_BINARY -m tox --version 2>/dev/null || echo "") From 4e66acba9155de41b65ebb546440e4cce89b02fa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 07:39:29 -0500 Subject: [PATCH 13/35] more cleanup --- .evergreen/run-mongodb-aws-test.sh | 2 +- .evergreen/tox.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index a0adc53ce5..dc215b9a86 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -30,7 +30,7 @@ if [ -n "$1" ]; then cd - fi -export PYTHON_BINARY=python3 +export PYTHON_BINARY=python if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index 95523167de..a8a6f50c1c 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -24,10 +24,11 @@ else # No toolchain present, set up virtualenv before installing tox . .evergreen/utils.sh venvcreate "$PYTHON_BINARY" toxenv trap "deactivate; rm -rf toxenv" EXIT HUP + $PYTHON_BINARY=python fi - python -m pip install -q tox + $PYTHON_BINARY -m pip install -q tox run_tox() { - python -m tox "$@" + $PYTHON_BINARY -m tox "$@" } fi From 423c0d20a565e92dabe9c2227c5762c802a790b3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 07:46:48 -0500 Subject: [PATCH 14/35] try again --- .evergreen/tox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index a8a6f50c1c..f18d19f234 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -24,7 +24,7 @@ else # No toolchain present, set up virtualenv before installing tox . .evergreen/utils.sh venvcreate "$PYTHON_BINARY" toxenv trap "deactivate; rm -rf toxenv" EXIT HUP - $PYTHON_BINARY=python + PYTHON_BINARY=python fi $PYTHON_BINARY -m pip install -q tox run_tox() { From 93246dcd6b4f67ec57d1af35c5987b4c12929c23 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 07:52:52 -0500 Subject: [PATCH 15/35] try again --- .evergreen/tox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index f18d19f234..fe74543864 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -5,7 +5,7 @@ set -x if [ -z "$PYTHON_BINARY" ]; then # If DRIVERS_TOOLS is not set, use the current python3. if [ -z "$DRIVERS_TOOLS" ]; then - PYTHON_BINARY=python3 + PYTHON_BINARY=$(command -v python3 || command -v python) else . $DRIVERS_TOOLS/.evergreen/find-python3.sh PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 @@ -24,7 +24,7 @@ else # No toolchain present, set up virtualenv before installing tox . .evergreen/utils.sh venvcreate "$PYTHON_BINARY" toxenv trap "deactivate; rm -rf toxenv" EXIT HUP - PYTHON_BINARY=python + PYTHON_BINARY=$(command -v python3 || command -v python) fi $PYTHON_BINARY -m pip install -q tox run_tox() { From 306c34c36ced46229886850060795e1a14f9f870 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 10:50:24 -0500 Subject: [PATCH 16/35] try a fix for aws --- .evergreen/run-mongodb-aws-test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index dc215b9a86..b8fa6c48d3 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -30,10 +30,8 @@ if [ -n "$1" ]; then cd - fi -export PYTHON_BINARY=python - if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then - [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" + [ -s PYTHON_BINARY=python "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" fi if [ -n "$USE_ENV_VAR_CREDS" ]; then From 28efe45abebaf3ef2327ec810cbde169208455f2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 12:50:52 -0500 Subject: [PATCH 17/35] try with our own functions --- .evergreen/config.yml | 2 +- .evergreen/run-mongodb-aws-test.sh | 7 ++- .evergreen/tox.sh | 27 +++------ .evergreen/utils.sh | 91 ++++++++++++++++++++++++++---- 4 files changed, 96 insertions(+), 31 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4cddcf70ce..6b27181e16 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -97,7 +97,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone --branch cleanup-python-binary-handlin https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index b8fa6c48d3..74541e9ebf 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -28,10 +28,15 @@ if [ -n "$1" ]; then . ./activate-authawsvenv.sh python aws_tester.py "$1" cd - + PYTHON_BINARY=$(command -v python) +else + . ./.evergreen/utils.sh + PYTHON_BINARY=$(find_python3) fi + if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then - [ -s PYTHON_BINARY=python "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" + [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" fi if [ -n "$USE_ENV_VAR_CREDS" ]; then diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh index fe74543864..808787d63c 100644 --- a/.evergreen/tox.sh +++ b/.evergreen/tox.sh @@ -2,33 +2,22 @@ set -o errexit # Exit the script with error if any of the commands fail set -x +. .evergreen/utils.sh + if [ -z "$PYTHON_BINARY" ]; then - # If DRIVERS_TOOLS is not set, use the current python3. - if [ -z "$DRIVERS_TOOLS" ]; then - PYTHON_BINARY=$(command -v python3 || command -v python) - else - . $DRIVERS_TOOLS/.evergreen/find-python3.sh - PYTHON_BINARY="$(find_python3 2>/dev/null)" || exit 1 - fi + PYTHON_BINARY=$(find_python3) fi -TOX=$($PYTHON_BINARY -m tox --version 2>/dev/null || echo "") -if [ -n "$TOX" ]; then +if $PYTHON_BINARY -m tox --version; then run_tox() { $PYTHON_BINARY -m tox "$@" } else # No toolchain present, set up virtualenv before installing tox - # If DRIVERS_TOOLS is set, we are on an EVG host, use a virtual env. - # Otherwise we are on temporary host, we can install directly. - if [ -n "$DRIVERS_TOOLS" ]; then - . .evergreen/utils.sh - venvcreate "$PYTHON_BINARY" toxenv - trap "deactivate; rm -rf toxenv" EXIT HUP - PYTHON_BINARY=$(command -v python3 || command -v python) - fi - $PYTHON_BINARY -m pip install -q tox + createvirtualenv "$PYTHON_BINARY" toxenv + trap "deactivate; rm -rf toxenv" EXIT HUP + python -m pip install -q tox run_tox() { - $PYTHON_BINARY -m tox "$@" + python -m tox "$@" } fi diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index beed9cf149..bc52abd705 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -2,10 +2,71 @@ set -o xtrace -if [ -n "$DRIVERS_TOOLS" ]; then - # Import venvcreate from $DRIVERS_TOOLS - . $DRIVERS_TOOLS/.evergreen/venv-utils.sh -fi +find_python3() { + PYTHON="" + # Add a fallback system python3 if it is available and Python 3.7+. + if is_python_37 "$(command -v python3)"; then + PYTHON="$(command -v python3)" + fi + # Find a suitable toolchain version, if available. + if [ "$(uname -s)" = "Darwin" ]; then + # macos 10.14 + if [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then + PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" + # macos 11.00 + elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then + PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" + fi + elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + PYTHON="C:/python/Python37/python.exe" + else + # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.7+. + if [ -f "/opt/python/3.7/bin/python3" ]; then + PYTHON="/opt/python/3.7/bin/python3" + elif is_python_37 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then + PYTHON="/opt/mongodbtoolchain/v4/bin/python3" + elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then + PYTHON="/opt/mongodbtoolchain/v3/bin/python3" + fi + fi + if [ -z "$PYTHON" ]; then + echo "Cannot test without python3.7+ installed!" + exit 1 + fi + echo "$PYTHON" +} + +# Usage: +# createvirtualenv /path/to/python /output/path/for/venv +# * param1: Python binary to use for the virtualenv +# * param2: Path to the virtualenv to create +createvirtualenv () { + PYTHON=$1 + VENVPATH=$2 + # Prefer venv + if $PYTHON -m venv -h > /dev/null; then + VIRTUALENV="$PYTHON -m venv --system-site-packages" + elif $PYTHON -m virtualenv --version > /dev/null; then + VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" + else + echo "Cannot test without virtual env" + exit 1 + fi + # Workaround for bug in older versions of virtualenv. + $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH + if [ "Windows_NT" = "$OS" ]; then + # Workaround https://bugs.python.org/issue32451: + # mongovenv/Scripts/activate: line 3: $'\r': command not found + dos2unix $VENVPATH/Scripts/activate || true + . $VENVPATH/Scripts/activate + else + . $VENVPATH/bin/activate + fi + + export PIP_QUIET=1 + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools tox +} # Usage: # testinstall /path/to/python /path/to/.whl ["no-virtualenv"] @@ -18,12 +79,7 @@ testinstall () { NO_VIRTUALENV=$3 if [ -z "$NO_VIRTUALENV" ]; then - # Make sure DRIVERS_TOOLS is set. - if [ -z "$DRIVERS_TOOLS" ]; then - echo "Must specify DRIVERS_TOOLS" - exit 1 - fi - venvcreate $PYTHON venvtestinstall + createvirtualenv $PYTHON venvtestinstall PYTHON=python fi @@ -38,3 +94,18 @@ testinstall () { rm -rf venvtestinstall fi } + +# Function that returns success if the provided Python binary is version 3.7 or later +# Usage: +# is_python_37 /path/to/python +# * param1: Python binary +is_python_37() { + if [ -z "$1" ]; then + return 1 + elif $1 -c "import sys; exit(sys.version_info[:2] < (3, 7))"; then + # runs when sys.version_info[:2] >= (3, 7) + return 0 + else + return 1 + fi +} From 46172b08b6f47101ade02477c25c2d8a704129bd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 12:52:52 -0500 Subject: [PATCH 18/35] cleanup --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6b27181e16..16bc185165 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2150,7 +2150,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 PYTHON_BINARY=python3 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -2203,7 +2203,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="PYTHON_BINARY=python3 KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ + AZUREKMS_CMD="KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task From 85a0f33071ea95f4f0d9de045cfbeb2d40323f16 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 15:16:01 -0500 Subject: [PATCH 19/35] fix handling of venv --- .evergreen/utils.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index bc52abd705..205205b576 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -44,16 +44,12 @@ createvirtualenv () { PYTHON=$1 VENVPATH=$2 # Prefer venv - if $PYTHON -m venv -h > /dev/null; then - VIRTUALENV="$PYTHON -m venv --system-site-packages" - elif $PYTHON -m virtualenv --version > /dev/null; then - VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" - else - echo "Cannot test without virtual env" - exit 1 + VENV="$PYTHON -m venv --system-site-packages" + VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" + if ! $VENV $$VENVPATH; then + # Workaround for bug in older versions of virtualenv. + $VIRTUALENV $VENVPATH || $VIRTUALENV $VENVPATH fi - # Workaround for bug in older versions of virtualenv. - $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH if [ "Windows_NT" = "$OS" ]; then # Workaround https://bugs.python.org/issue32451: # mongovenv/Scripts/activate: line 3: $'\r': command not found From a43b34cdeabb4461b88dc88ce1e9f5bd33c8e8ae Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 15:20:59 -0500 Subject: [PATCH 20/35] cleanup --- .evergreen/build-mac.sh | 2 +- .evergreen/run-mongodb-aws-test.sh | 8 +++----- .evergreen/run-mongodb-oidc-test.sh | 8 ++------ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh index afea71a918..4e8be8cf58 100755 --- a/.evergreen/build-mac.sh +++ b/.evergreen/build-mac.sh @@ -13,7 +13,7 @@ VERSION=${VERSION:-3.10} PYTHON=/Library/Frameworks/Python.framework/Versions/$VERSION/bin/python3 rm -rf build -venvcreate $PYTHON releasevenv +createvirtualenv $PYTHON releasevenv python -m pip install build python -m build --wheel . deactivate || true diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 74541e9ebf..5c37e33434 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -18,6 +18,9 @@ echo "Running MONGODB-AWS authentication tests" # ensure no secrets are printed in log files set +x +. ./.evergreen/utils.sh +PYTHON_BINARY=$(find_python3) + # Try to source exported AWS Secrets if [ -f ./secrets-export.sh ]; then source ./secrets-export.sh @@ -28,13 +31,8 @@ if [ -n "$1" ]; then . ./activate-authawsvenv.sh python aws_tester.py "$1" cd - - PYTHON_BINARY=$(command -v python) -else - . ./.evergreen/utils.sh - PYTHON_BINARY=$(find_python3) fi - if [ -z "${SKIP_PREPARE_AWS_ENV}" ]; then [ -s "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" ] && source "${DRIVERS_TOOLS}/.evergreen/auth_aws/prepare_aws_env.sh" fi diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 662360a3ec..7ea2018793 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -23,12 +23,9 @@ if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then source ./secrets-export.sh fi -# Make the OIDC tokens - also activate the oidc venv. +# Make the OIDC tokens. set -x -pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc -. oidc_get_tokens.sh -pip install -q tox -popd +bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/oidc_get_tokens.sh # Set up variables and run the test. if [ -n "$LOCAL_OIDC_SERVER" ]; then @@ -46,5 +43,4 @@ fi export TEST_AUTH_OIDC=1 export COVERAGE=1 export AUTH="auth" -export PYTHON_BINARY=python bash ./.evergreen/tox.sh -m test-eg From d87e6535e3781b7ab132438258213a7eabbfac25 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 15:26:07 -0500 Subject: [PATCH 21/35] fixups --- .evergreen/run-mongodb-oidc-test.sh | 4 +++- .evergreen/utils.sh | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 7ea2018793..6ded3d3021 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -25,7 +25,9 @@ fi # Make the OIDC tokens. set -x -bash ${DRIVERS_TOOLS}/.evergreen/auth_oidc/oidc_get_tokens.sh +pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc +bash /oidc_get_tokens.sh +popd # Set up variables and run the test. if [ -n "$LOCAL_OIDC_SERVER" ]; then diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 205205b576..d404ad9f95 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -46,9 +46,9 @@ createvirtualenv () { # Prefer venv VENV="$PYTHON -m venv --system-site-packages" VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" - if ! $VENV $$VENVPATH; then + if ! $($VENV $$VENVPATH); then # Workaround for bug in older versions of virtualenv. - $VIRTUALENV $VENVPATH || $VIRTUALENV $VENVPATH + $($VIRTUALENV $VENVPATH) || $($VIRTUALENV $VENVPATH) fi if [ "Windows_NT" = "$OS" ]; then # Workaround https://bugs.python.org/issue32451: From dfbca6e8ecb395bf0c7e1cbbe9caefc778004ca7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 15:59:04 -0500 Subject: [PATCH 22/35] more cleanup --- .evergreen/config.yml | 3 ++- .evergreen/run-mongodb-oidc-test.sh | 2 +- .evergreen/utils.sh | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 16bc185165..07046fd8e4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -358,6 +358,7 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace + export PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-mockupdb "run doctests": @@ -368,7 +369,7 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace - bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test + PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test "run tests": # If testing FLE, start the KMS mock servers, first create the virtualenv. diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 6ded3d3021..288767752f 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -26,7 +26,7 @@ fi # Make the OIDC tokens. set -x pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc -bash /oidc_get_tokens.sh +bash ./oidc_get_tokens.sh popd # Set up variables and run the test. diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index d404ad9f95..3db5386706 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -46,9 +46,9 @@ createvirtualenv () { # Prefer venv VENV="$PYTHON -m venv --system-site-packages" VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" - if ! $($VENV $$VENVPATH); then + if ! "$VENV $$VENVPATH"; then # Workaround for bug in older versions of virtualenv. - $($VIRTUALENV $VENVPATH) || $($VIRTUALENV $VENVPATH) + "$VIRTUALENV $VENVPATH" || "$VIRTUALENV $VENVPATH" fi if [ "Windows_NT" = "$OS" ]; then # Workaround https://bugs.python.org/issue32451: From b3fde222f71260bd908c2d0765d947e3c9a9f4e7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 16:05:58 -0500 Subject: [PATCH 23/35] more cleanup --- .evergreen/run-mongodb-oidc-test.sh | 1 + .evergreen/utils.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 288767752f..5556db3030 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -26,6 +26,7 @@ fi # Make the OIDC tokens. set -x pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc +export OIDC_TOKEN_DIR=/tmp/tokens bash ./oidc_get_tokens.sh popd diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 3db5386706..98883b3344 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -46,7 +46,7 @@ createvirtualenv () { # Prefer venv VENV="$PYTHON -m venv --system-site-packages" VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" - if ! "$VENV $$VENVPATH"; then + if ! "$VENV $VENVPATH"; then # Workaround for bug in older versions of virtualenv. "$VIRTUALENV $VENVPATH" || "$VIRTUALENV $VENVPATH" fi From f2fd053da9944c133d7b608ec8120df1cba9fc34 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 16:13:19 -0500 Subject: [PATCH 24/35] try virtualenv again --- .evergreen/utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 98883b3344..557d566490 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -45,7 +45,8 @@ createvirtualenv () { VENVPATH=$2 # Prefer venv VENV="$PYTHON -m venv --system-site-packages" - VIRTUALENV="$PYTHON -m virtualenv -p $PYTHON --system-site-packages" + VIRTUALENV=$(command -v virtualenv || echo "$PYTHON -m virtualenv") + VIRTUALENV="$VIRTUALENV -p $PYTHON --system-site-packages" if ! "$VENV $VENVPATH"; then # Workaround for bug in older versions of virtualenv. "$VIRTUALENV $VENVPATH" || "$VIRTUALENV $VENVPATH" From c0a7f7ca090fd11555e30d44f54f5475cc3705f1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 16:25:49 -0500 Subject: [PATCH 25/35] more venv cleanup --- .evergreen/utils.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 557d566490..1986511c9b 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -44,10 +44,10 @@ createvirtualenv () { PYTHON=$1 VENVPATH=$2 # Prefer venv - VENV="$PYTHON -m venv --system-site-packages" - VIRTUALENV=$(command -v virtualenv || echo "$PYTHON -m virtualenv") - VIRTUALENV="$VIRTUALENV -p $PYTHON --system-site-packages" - if ! "$VENV $VENVPATH"; then + VENV="$PYTHON -m venv" + VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv") + VIRTUALENV="$VIRTUALENV -p $PYTHON" + if ! "$VENV $VENVPATH" 2>/dev/null; then # Workaround for bug in older versions of virtualenv. "$VIRTUALENV $VENVPATH" || "$VIRTUALENV $VENVPATH" fi From b8f86c7eae87ddb0a602820f22383b7d926d598a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 16:35:56 -0500 Subject: [PATCH 26/35] more venv cleanup --- .evergreen/utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 1986511c9b..73c972c3c9 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -47,9 +47,9 @@ createvirtualenv () { VENV="$PYTHON -m venv" VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv") VIRTUALENV="$VIRTUALENV -p $PYTHON" - if ! "$VENV $VENVPATH" 2>/dev/null; then + if ! $VENV $VENVPATH 2>/dev/null; then # Workaround for bug in older versions of virtualenv. - "$VIRTUALENV $VENVPATH" || "$VIRTUALENV $VENVPATH" + $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH fi if [ "Windows_NT" = "$OS" ]; then # Workaround https://bugs.python.org/issue32451: From 3c66cdb3e37c729dfd6e10e8fbcea392541d9902 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:10:33 -0500 Subject: [PATCH 27/35] fix use of virtualenv on macos --- .evergreen/utils.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 73c972c3c9..5dbb8d356b 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -11,10 +11,10 @@ find_python3() { # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then # macos 10.14 - if [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then + if [[ "$(sw_vers -productVersion)" == "10.?" ]]; PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" - # macos 11.00 - elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then + else + # macos 11.00 PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" fi elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin @@ -45,8 +45,12 @@ createvirtualenv () { VENVPATH=$2 # Prefer venv VENV="$PYTHON -m venv" - VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv") - VIRTUALENV="$VIRTUALENV -p $PYTHON" + if [ "$(uname -s)" = "Darwin" ]; then + VIRTUALENV="$PYTHON -m virtualenv" + else + VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv") + VIRTUALENV="$VIRTUALENV -p $PYTHON" + fi if ! $VENV $VENVPATH 2>/dev/null; then # Workaround for bug in older versions of virtualenv. $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH From 3577a221ebbb6a3d6e7f668a40e517854c505177 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:14:31 -0500 Subject: [PATCH 28/35] syntax --- .evergreen/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 5dbb8d356b..1d9f877aa5 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -11,7 +11,7 @@ find_python3() { # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then # macos 10.14 - if [[ "$(sw_vers -productVersion)" == "10.?" ]]; + if [[ "$(sw_vers -productVersion)" == "10.?" ]]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" else # macos 11.00 From 4ff5a4d33117e6b6b5f125c8fd97385cea94c956 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:25:11 -0500 Subject: [PATCH 29/35] fixup --- .evergreen/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 1d9f877aa5..f806e42c30 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -11,7 +11,7 @@ find_python3() { # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then # macos 10.14 - if [[ "$(sw_vers -productVersion)" == "10.?" ]]; then + if [[ "$(sw_vers -productVersion)" == "10.*" ]]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" else # macos 11.00 From d39a5a0101dbec5020e2a6f5f51880468a833f80 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:27:50 -0500 Subject: [PATCH 30/35] fixup --- .evergreen/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index f806e42c30..eb76ac3ba8 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -11,7 +11,7 @@ find_python3() { # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then # macos 10.14 - if [[ "$(sw_vers -productVersion)" == "10.*" ]]; then + if [[ "$(sw_vers -productVersion)" ~= "10.*" ]]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" else # macos 11.00 From 44a41eda2721b265168b31c76ad831e27be212a9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:30:37 -0500 Subject: [PATCH 31/35] fixup --- .evergreen/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index eb76ac3ba8..1efcb37111 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -11,7 +11,7 @@ find_python3() { # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then # macos 10.14 - if [[ "$(sw_vers -productVersion)" ~= "10.*" ]]; then + if [[ "$(sw_vers -productVersion)" =~ "10.*" ]]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" else # macos 11.00 From b534e50fb032ec2286fce160dbc39784d9b16236 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 05:39:30 -0500 Subject: [PATCH 32/35] try this approach --- .evergreen/utils.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index 1efcb37111..35005c0d6a 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -10,12 +10,12 @@ find_python3() { fi # Find a suitable toolchain version, if available. if [ "$(uname -s)" = "Darwin" ]; then + # macos 11.00 + if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then + PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" # macos 10.14 - if [[ "$(sw_vers -productVersion)" =~ "10.*" ]]; then + elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" - else - # macos 11.00 - PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3" fi elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin PYTHON="C:/python/Python37/python.exe" From 8d228a751b7ec29b260102e977b8eb4ecfb29ba8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 19:09:12 -0500 Subject: [PATCH 33/35] try all fixups --- .evergreen/run-mongodb-oidc-test.sh | 3 +-- .evergreen/run-tests.sh | 2 +- test/test_index_management.py | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 5556db3030..75fafb448b 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -26,8 +26,7 @@ fi # Make the OIDC tokens. set -x pushd ${DRIVERS_TOOLS}/.evergreen/auth_oidc -export OIDC_TOKEN_DIR=/tmp/tokens -bash ./oidc_get_tokens.sh +. ./oidc_get_tokens.sh popd # Set up variables and run the test. diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 21cba87ecc..46d2f6189d 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -258,7 +258,7 @@ python -c 'import sys; print(sys.version)' # Only cover CPython. PyPy reports suspiciously low coverage. PYTHON_IMPL=$($PYTHON -c "import platform; print(platform.python_implementation())") if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then - python -m pip install pytest-cov + python -m pip install pytest-cov coverage==7.3.1 TEST_ARGS="$TEST_ARGS --cov pymongo --cov-branch --cov-report term-missing:skip-covered" fi diff --git a/test/test_index_management.py b/test/test_index_management.py index 4d8605f243..7a69f51620 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -67,9 +67,14 @@ def setUpClass(cls) -> None: password = os.environ["DB_PASSWORD"] cls.client = MongoClient(url, username=username, password=password) cls.client.drop_database(_NAME) - cls.addClassCleanup(cls.client.close) + cls.addCleanup(cls.client.close) cls.db = cls.client.test_search_index_prose + @classmethod + def tearDownClass(cls): + cls.client.drop_database(_NAME) + cls.client.close() + def wait_for_ready(self, coll, name=_NAME, predicate=None): """Wait for a search index to be ready.""" indices: list[Mapping[str, Any]] = [] @@ -214,12 +219,13 @@ def test_case_5(self): coll0.drop_search_index("foo") -globals().update( - generate_test_classes( - _TEST_PATH, - module=__name__, +if os.environ.get("TEST_INDEX_MANAGEMENT"): + globals().update( + generate_test_classes( + _TEST_PATH, + module=__name__, + ) ) -) if __name__ == "__main__": unittest.main() From e898c7eca6fbe75a8758fad5e7a492a330b1cee5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 19:25:45 -0500 Subject: [PATCH 34/35] fixups --- .evergreen/config.yml | 1 + .evergreen/run-tests.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 49d9640b1f..8c62081585 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -492,6 +492,7 @@ functions: fi GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ + PYTHON_BINARY=${PYTHON_BINARY} \ C_EXTENSIONS=${C_EXTENSIONS} \ COVERAGE=${COVERAGE} \ COMPRESSORS=${COMPRESSORS} \ diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 46d2f6189d..0ef12b2855 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -258,7 +258,7 @@ python -c 'import sys; print(sys.version)' # Only cover CPython. PyPy reports suspiciously low coverage. PYTHON_IMPL=$($PYTHON -c "import platform; print(platform.python_implementation())") if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then - python -m pip install pytest-cov coverage==7.3.1 + python -m pip install pytest-cov "coverage<=7.3.1" TEST_ARGS="$TEST_ARGS --cov pymongo --cov-branch --cov-report term-missing:skip-covered" fi From dcb4d223be24daa40c9faad7350118086ecb8cdc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 3 Oct 2023 20:44:59 -0500 Subject: [PATCH 35/35] More fixups --- .evergreen/run-tests.sh | 2 +- test/test_index_management.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 0ef12b2855..815d5fdfda 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -258,7 +258,7 @@ python -c 'import sys; print(sys.version)' # Only cover CPython. PyPy reports suspiciously low coverage. PYTHON_IMPL=$($PYTHON -c "import platform; print(platform.python_implementation())") if [ -n "$COVERAGE" ] && [ "$PYTHON_IMPL" = "CPython" ]; then - python -m pip install pytest-cov "coverage<=7.3.1" + python -m pip install pytest-cov "coverage<7.3.0" TEST_ARGS="$TEST_ARGS --cov pymongo --cov-branch --cov-report term-missing:skip-covered" fi diff --git a/test/test_index_management.py b/test/test_index_management.py index 7a69f51620..d4a83803da 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -67,7 +67,6 @@ def setUpClass(cls) -> None: password = os.environ["DB_PASSWORD"] cls.client = MongoClient(url, username=username, password=password) cls.client.drop_database(_NAME) - cls.addCleanup(cls.client.close) cls.db = cls.client.test_search_index_prose @classmethod