From d450f2fb21564ec367cac8bdaaa490d93334bf83 Mon Sep 17 00:00:00 2001 From: julius Date: Mon, 9 May 2022 19:11:00 -0700 Subject: [PATCH 01/21] initial commit --- pymongo/encryption.py | 6 +++++- pymongo/encryption_options.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 1e06f7062d..8360a75ae6 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -296,7 +296,11 @@ def _get_internal_client(encrypter, mongo_client): io_callbacks = _EncryptionIO(metadata_client, key_vault_coll, mongocryptd_client, opts) self._auto_encrypter = AutoEncrypter( - io_callbacks, MongoCryptOptions(opts._kms_providers, schema_map) + io_callbacks, + MongoCryptOptions(opts._kms_providers, schema_map), + self.opts._csfle_path, + self.opts._csfle_required, + self.opts._bypass_auto_encryption, ) self._closed = False diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 2ac12bc4b4..0adc9e8622 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -45,6 +45,7 @@ def __init__( mongocryptd_spawn_path: str = "mongocryptd", mongocryptd_spawn_args: Optional[List[str]] = None, kms_tls_options: Optional[Mapping[str, Any]] = None, + extra_options: Optional[Mapping[str, Any]] = None, ) -> None: """Options to configure automatic client-side field level encryption. @@ -140,7 +141,9 @@ def __init__( Or to supply a client certificate:: kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} - + - `extra_options` (optional): A map of values containing extra options. + - `csflePath`: Override the path to load the CSFLE library. + - `csfleRequired`: If 'true', refuse to continue encryption without a CSFLE library .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. @@ -152,7 +155,11 @@ def __init__( "install a compatible version with: " "python -m pip install 'pymongo[encryption]'" ) - + if extra_options is not None: + self._csfle_path = extra_options.get("csflePath") + self._csfle_required = extra_options.get("csfleRequired", False) + else: + self._csfle_path, self._csfle_required = None, False self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client From 0efaa2fd91b7ffce6ac6d0b063e8c61f4d4d7a67 Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 24 May 2022 17:24:45 -0700 Subject: [PATCH 02/21] fix docs --- pymongo/encryption_options.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 0adc9e8622..79dcef9195 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -45,7 +45,8 @@ def __init__( mongocryptd_spawn_path: str = "mongocryptd", mongocryptd_spawn_args: Optional[List[str]] = None, kms_tls_options: Optional[Mapping[str, Any]] = None, - extra_options: Optional[Mapping[str, Any]] = None, + csfle_path: Optional[str] = None, + csfle_required: Optional[bool] = False, ) -> None: """Options to configure automatic client-side field level encryption. @@ -141,9 +142,10 @@ def __init__( Or to supply a client certificate:: kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} - - `extra_options` (optional): A map of values containing extra options. - - `csflePath`: Override the path to load the CSFLE library. - - `csfleRequired`: If 'true', refuse to continue encryption without a CSFLE library + - `csfle_path` (optional): Override the path to load the CSFLE library. + - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE + library + .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. @@ -155,11 +157,8 @@ def __init__( "install a compatible version with: " "python -m pip install 'pymongo[encryption]'" ) - if extra_options is not None: - self._csfle_path = extra_options.get("csflePath") - self._csfle_required = extra_options.get("csfleRequired", False) - else: - self._csfle_path, self._csfle_required = None, False + self._csfle_path = csfle_path + self._csfle_required = csfle_required self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client From 5cb948252c123b10eb57ae43a3286dc6cc11c2ec Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 24 May 2022 17:28:02 -0700 Subject: [PATCH 03/21] fix mypy --- pymongo/encryption.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 8360a75ae6..6cdfd9c2fa 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -298,9 +298,9 @@ def _get_internal_client(encrypter, mongo_client): self._auto_encrypter = AutoEncrypter( io_callbacks, MongoCryptOptions(opts._kms_providers, schema_map), - self.opts._csfle_path, - self.opts._csfle_required, - self.opts._bypass_auto_encryption, + opts._csfle_path, + opts._csfle_required, + opts._bypass_auto_encryption, ) self._closed = False From 3505295d92625c2f40d104143484faa153830ead Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 24 May 2022 17:34:33 -0700 Subject: [PATCH 04/21] fix docs --- pymongo/encryption_options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 79dcef9195..a808e1ae21 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -146,6 +146,9 @@ def __init__( - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE library + .. versionchanged:: 4.2 + Added `csfle_path` and `csfle_required` parameters + .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. From f1be8d238871497b88f4bd28fd491b5657b3b705 Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 24 May 2022 19:38:59 -0700 Subject: [PATCH 05/21] fix indentation I hope --- pymongo/encryption_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index a808e1ae21..4c8bc1140e 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -145,7 +145,7 @@ def __init__( - `csfle_path` (optional): Override the path to load the CSFLE library. - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE library - + .. versionchanged:: 4.2 Added `csfle_path` and `csfle_required` parameters From a0d96b295563ee596de5fa81ad4ebc6b452e2031 Mon Sep 17 00:00:00 2001 From: julius Date: Wed, 25 May 2022 15:49:03 -0700 Subject: [PATCH 06/21] mixed up arguments, remove optional for bool, align with backtick --- pymongo/encryption.py | 11 +++++++---- pymongo/encryption_options.py | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 6cdfd9c2fa..4470f1cafd 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -297,10 +297,13 @@ def _get_internal_client(encrypter, mongo_client): io_callbacks = _EncryptionIO(metadata_client, key_vault_coll, mongocryptd_client, opts) self._auto_encrypter = AutoEncrypter( io_callbacks, - MongoCryptOptions(opts._kms_providers, schema_map), - opts._csfle_path, - opts._csfle_required, - opts._bypass_auto_encryption, + MongoCryptOptions( + opts._kms_providers, + schema_map, + opts._csfle_path, + opts._csfle_required, + opts._bypass_auto_encryption, + ), ) self._closed = False diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 4c8bc1140e..0ce828ae4c 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -39,14 +39,14 @@ def __init__( key_vault_namespace: str, key_vault_client: Optional["MongoClient"] = None, schema_map: Optional[Mapping[str, Any]] = None, - bypass_auto_encryption: Optional[bool] = False, + bypass_auto_encryption: bool = False, mongocryptd_uri: str = "mongodb://localhost:27020", mongocryptd_bypass_spawn: bool = False, mongocryptd_spawn_path: str = "mongocryptd", mongocryptd_spawn_args: Optional[List[str]] = None, kms_tls_options: Optional[Mapping[str, Any]] = None, csfle_path: Optional[str] = None, - csfle_required: Optional[bool] = False, + csfle_required: bool = False, ) -> None: """Options to configure automatic client-side field level encryption. @@ -144,8 +144,8 @@ def __init__( kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} - `csfle_path` (optional): Override the path to load the CSFLE library. - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE - library - + library + .. versionchanged:: 4.2 Added `csfle_path` and `csfle_required` parameters From 77f57eddd5f1b33cd1b2bfe1553c4b996b2e7e0f Mon Sep 17 00:00:00 2001 From: julius Date: Wed, 25 May 2022 20:12:52 -0700 Subject: [PATCH 07/21] add test to make sure we can load csfle --- .evergreen/config.yml | 10 +++++++++- .evergreen/run-tests.sh | 11 ++++++++++- pymongo/encryption.py | 6 +++--- test/test_encryption.py | 11 +++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2576307364..01f7c54213 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2099,6 +2099,14 @@ axes: variables: test_encryption: true batchtime: 10080 # 7 days + - id: "encryption_with_csfle" + display_name: "Encryption with CSFLE" + tags: ["encryption_tag", "csfle"] + variables: + test_encryption: true + TEST_CSFLE: true + batchtime: 10080 # 7 days + # Run pyopenssl tests? - id: pyopenssl @@ -2235,7 +2243,7 @@ buildvariants: auth: "auth" ssl: "nossl" encryption: "*" - display_name: "Encryption ${platform} ${auth} ${ssl}" + display_name: "Encryption ${platform} ${auth} ${ssl} ${encryption}" tasks: *encryption-server-versions # Test one server version (4.2) with zSeries, POWER8, and ARM. diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4a48b4a33b..764ab63814 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -27,6 +27,7 @@ COVERAGE=${COVERAGE:-} COMPRESSORS=${COMPRESSORS:-} MONGODB_API_VERSION=${MONGODB_API_VERSION:-} TEST_ENCRYPTION=${TEST_ENCRYPTION:-} +TEST_CSFLE=${TEST_CSFLE:-} LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-} DATA_LAKE=${DATA_LAKE:-} @@ -153,7 +154,15 @@ if [ -z "$DATA_LAKE" ]; then else TEST_ARGS="-s test.test_data_lake" fi - +if [ -n "$TEST_CSFLE" ]; then + git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git + $PYTHON drivers-evergreen-tools/.evergreen/mongodl.py --component csfle \ + --version latest --out ../csfle/ + export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH + export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH + export PATH=../csfle/bin:$PATH + TEST_ARGS="-s test.test_encryption" +fi # Don't download unittest-xml-reporting from pypi, which often fails. if $PYTHON -c "import xmlrunner"; then # The xunit output dir must be a Python style absolute path. diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 4470f1cafd..1a29131890 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -300,9 +300,9 @@ def _get_internal_client(encrypter, mongo_client): MongoCryptOptions( opts._kms_providers, schema_map, - opts._csfle_path, - opts._csfle_required, - opts._bypass_auto_encryption, + csfle_path=opts._csfle_path, + csfle_required=opts._csfle_required, + bypass_encryption=opts._bypass_auto_encryption, ), ) self._closed = False diff --git a/test/test_encryption.py b/test/test_encryption.py index 366c406b03..799f28a503 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,6 +82,17 @@ def get_client_opts(client): class TestAutoEncryptionOpts(PyMongoTestCase): + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + @unittest.skipUnless(os.environ.get("TEST_CSFLE"), "csfle is not installed") + def test_csfle(self): + # Test that we can pick up csfle automatically + MongoClient( + auto_encryption_opts=AutoEncryptionOpts( + KMS_PROVIDERS, "keyvault.datakeys", csfle_required=True + ), + connect=False, + ) + @unittest.skipIf(_HAVE_PYMONGOCRYPT, "pymongocrypt is installed") def test_init_requires_pymongocrypt(self): with self.assertRaises(ConfigurationError): From e56dfa592a27b4984ccf0d83b55ee1ebad2c6527 Mon Sep 17 00:00:00 2001 From: julius Date: Wed, 25 May 2022 23:45:10 -0700 Subject: [PATCH 08/21] have to skip prose test because it does not work when csfle loaded --- .evergreen/config.yml | 5 ++++- .evergreen/run-tests.sh | 5 ++++- test/test_encryption.py | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 01f7c54213..a8bcc2eea7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -450,6 +450,9 @@ functions: export LIBMONGOCRYPT_URL="${libmongocrypt_url}" export TEST_ENCRYPTION=1 fi + if [ -n "${test_csfle}" ]; then + export TEST_CSFLE=1 + fi if [ -n "${test_pyopenssl}" ]; then export TEST_PYOPENSSL=1 fi @@ -2104,7 +2107,7 @@ axes: tags: ["encryption_tag", "csfle"] variables: test_encryption: true - TEST_CSFLE: true + test_csfle: true batchtime: 10080 # 7 days diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 764ab63814..5c0c78f7cd 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -11,6 +11,7 @@ set -o errexit # Exit the script with error if any of the commands fail # COVERAGE If non-empty, run the test suite with coverage. # TEST_ENCRYPTION If non-empty, install pymongocrypt. # LIBMONGOCRYPT_URL The URL to download libmongocrypt. +# TEST_CSFLE If non-empty, install CSFLE if [ -n "${SET_XTRACE_ON}" ]; then set -o xtrace @@ -154,7 +155,9 @@ if [ -z "$DATA_LAKE" ]; then else TEST_ARGS="-s test.test_data_lake" fi -if [ -n "$TEST_CSFLE" ]; then +if [ -z $TEST_CSFLE ]; then + echo "CSFLE not being tested" +else git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git $PYTHON drivers-evergreen-tools/.evergreen/mongodl.py --component csfle \ --version latest --out ../csfle/ diff --git a/test/test_encryption.py b/test/test_encryption.py index 799f28a503..fb299aa768 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -1764,6 +1764,10 @@ def test_case_8(self): # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.rst#bypass-spawning-mongocryptd class TestBypassSpawningMongocryptdProse(EncryptionIntegrationTest): + @unittest.skipIf( + os.environ.get("TEST_CSFLE"), + "this prose test does not work when CSFLE is " "on a system dynamic library search path.", + ) def test_mongocryptd_bypass_spawn(self): # Lower the mongocryptd timeout to reduce the test run time. self._original_timeout = encryption._MONGOCRYPTD_TIMEOUT_MS From 8a18adfa26fe82ea7fae354ffcc112773455baea Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 00:51:05 -0700 Subject: [PATCH 09/21] should fix it on amazon linux --- .evergreen/run-tests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 5c0c78f7cd..5aeec814e3 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -159,8 +159,13 @@ if [ -z $TEST_CSFLE ]; then echo "CSFLE not being tested" else git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git + if [[ "$(cat /etc/system-release)" == *"Amazon Linux"* ]]; then + MONGODL_ARGS="--target amazon2" + else + MONGODL_ARGS="" + fi $PYTHON drivers-evergreen-tools/.evergreen/mongodl.py --component csfle \ - --version latest --out ../csfle/ + --version latest --out ../csfle/ $MONGODL_ARGS export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH export PATH=../csfle/bin:$PATH From da553ca1ff48cd239ab20a7dc01a506afec6b036 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 08:52:57 -0700 Subject: [PATCH 10/21] add the encryption status to all the names --- .evergreen/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a8bcc2eea7..694d39b727 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2186,7 +2186,7 @@ buildvariants: - awslinux auth-ssl: "*" encryption: "*" - display_name: "Encryption ${platform} ${auth-ssl}" + display_name: "Encryption ${platform} ${auth-ssl} ${test_csfle} ${encryption}" tasks: &encryption-server-versions - ".latest" - ".5.0" @@ -2320,7 +2320,7 @@ buildvariants: # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config # coverage: "*" encryption: "*" - display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" + display_name: "Encryption ${python-version} ${platform} ${auth-ssl} ${encryption}" tasks: *encryption-server-versions - matrix_name: "tests-python-version-ubuntu18-without-c-extensions" @@ -2416,7 +2416,7 @@ buildvariants: python-version-windows: "*" auth-ssl: "*" encryption: "*" - display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}" + display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl} ${encryption}" tasks: *encryption-server-versions # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7. From bdb6c74c27db560ef25610d3deb38c97f6b44131 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 10:16:24 -0700 Subject: [PATCH 11/21] remove amazon linux tests --- .evergreen/config.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 694d39b727..e299c40b05 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2179,21 +2179,6 @@ buildvariants: - ".4.0" - ".3.6" -- matrix_name: "tests-all-encryption" - matrix_spec: - platform: - # OSes that support versions of MongoDB>=2.6 with SSL. - - awslinux - auth-ssl: "*" - encryption: "*" - display_name: "Encryption ${platform} ${auth-ssl} ${test_csfle} ${encryption}" - tasks: &encryption-server-versions - - ".latest" - - ".5.0" - - ".4.4" - - ".4.2" - - ".4.0" - - matrix_name: "tests-archlinux" matrix_spec: platform: @@ -2247,7 +2232,12 @@ buildvariants: ssl: "nossl" encryption: "*" display_name: "Encryption ${platform} ${auth} ${ssl} ${encryption}" - tasks: *encryption-server-versions + tasks: &encryption-server-versions + - ".latest" + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" # Test one server version (4.2) with zSeries, POWER8, and ARM. - matrix_name: "test-different-cpu-architectures" From c3655d1472c94bd44528ea87e9dd83472ec17725 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 10:17:26 -0700 Subject: [PATCH 12/21] remove amazon linux workaround --- .evergreen/run-tests.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 5aeec814e3..5c0c78f7cd 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -159,13 +159,8 @@ if [ -z $TEST_CSFLE ]; then echo "CSFLE not being tested" else git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git - if [[ "$(cat /etc/system-release)" == *"Amazon Linux"* ]]; then - MONGODL_ARGS="--target amazon2" - else - MONGODL_ARGS="" - fi $PYTHON drivers-evergreen-tools/.evergreen/mongodl.py --component csfle \ - --version latest --out ../csfle/ $MONGODL_ARGS + --version latest --out ../csfle/ export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH export PATH=../csfle/bin:$PATH From 5347fb9e55744ef13ecdacc0097bd5f9ccff44be Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 11:21:48 -0700 Subject: [PATCH 13/21] add fixes --- .evergreen/config.yml | 42 ++++++++++++++++++++++++++++++++++++++--- .evergreen/run-tests.sh | 3 +-- test/test_encryption.py | 3 ++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e299c40b05..df00f850f5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1235,6 +1235,32 @@ tasks: VERSION: "5.0" TOPOLOGY: "sharded_cluster" - func: "run tests" + - name: "test-6.0-standalone" + tags: [ "6.0", "standalone" ] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "6.0" + TOPOLOGY: "server" + - func: "run tests" + + - name: "test-6.0-replica_set" + tags: [ "6.0", "replica_set" ] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "6.0" + TOPOLOGY: "replica_set" + - func: "run tests" + + - name: "test-6.0-sharded_cluster" + tags: [ "6.0", "sharded_cluster" ] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "6.0" + TOPOLOGY: "sharded_cluster" + - func: "run tests" - name: "test-latest-standalone" tags: ["latest", "standalone"] @@ -2231,13 +2257,23 @@ buildvariants: auth: "auth" ssl: "nossl" encryption: "*" - display_name: "Encryption ${platform} ${auth} ${ssl} ${encryption}" + display_name: "${encryption} ${platform} ${auth} ${ssl}" tasks: &encryption-server-versions + - ".rapid" - ".latest" - ".5.0" - ".4.4" - ".4.2" - ".4.0" + rules: + - if: + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" # Test one server version (4.2) with zSeries, POWER8, and ARM. - matrix_name: "test-different-cpu-architectures" @@ -2310,7 +2346,7 @@ buildvariants: # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config # coverage: "*" encryption: "*" - display_name: "Encryption ${python-version} ${platform} ${auth-ssl} ${encryption}" + display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}" tasks: *encryption-server-versions - matrix_name: "tests-python-version-ubuntu18-without-c-extensions" @@ -2406,7 +2442,7 @@ buildvariants: python-version-windows: "*" auth-ssl: "*" encryption: "*" - display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl} ${encryption}" + display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}" tasks: *encryption-server-versions # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7. diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 5c0c78f7cd..96f42fa517 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -158,8 +158,7 @@ fi if [ -z $TEST_CSFLE ]; then echo "CSFLE not being tested" else - git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git - $PYTHON drivers-evergreen-tools/.evergreen/mongodl.py --component csfle \ + $PYTHON $DRIVERS_TOOLS/.evergreen/mongodl.py --component csfle \ --version latest --out ../csfle/ export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH diff --git a/test/test_encryption.py b/test/test_encryption.py index fb299aa768..632f4ed591 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -86,12 +86,13 @@ class TestAutoEncryptionOpts(PyMongoTestCase): @unittest.skipUnless(os.environ.get("TEST_CSFLE"), "csfle is not installed") def test_csfle(self): # Test that we can pick up csfle automatically - MongoClient( + client = MongoClient( auto_encryption_opts=AutoEncryptionOpts( KMS_PROVIDERS, "keyvault.datakeys", csfle_required=True ), connect=False, ) + self.addCleanup(client.close) @unittest.skipIf(_HAVE_PYMONGOCRYPT, "pymongocrypt is installed") def test_init_requires_pymongocrypt(self): From decd57604e580502f0bdc1e09f0539df19cb47ed Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 12:47:17 -0700 Subject: [PATCH 14/21] add 6.0 --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index df00f850f5..01ccc2fc93 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2261,6 +2261,7 @@ buildvariants: tasks: &encryption-server-versions - ".rapid" - ".latest" + - ".6.0" - ".5.0" - ".4.4" - ".4.2" From 4be7908219199d01e30f223bf2310b66eae1c821 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 12:53:23 -0700 Subject: [PATCH 15/21] should actually exclude everything --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 01ccc2fc93..5a16fed060 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2268,6 +2268,9 @@ buildvariants: - ".4.0" rules: - if: + platform: "*" + auth: "*" + ssl: "*" encryption: [ "encryption_with_csfle" ] then: remove_tasks: From 5690fdeda7683a5d472ebadf8402cc2f9878a128 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 12:56:47 -0700 Subject: [PATCH 16/21] add exclude rules to everything else --- .evergreen/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5a16fed060..b0338eae3a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2266,7 +2266,7 @@ buildvariants: - ".4.4" - ".4.2" - ".4.0" - rules: + rules: &encryption-exclude-rules - if: platform: "*" auth: "*" @@ -2352,6 +2352,8 @@ buildvariants: encryption: "*" display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}" tasks: *encryption-server-versions + rules: *encryption-exclude-rules + - matrix_name: "tests-python-version-ubuntu18-without-c-extensions" matrix_spec: @@ -2448,6 +2450,7 @@ buildvariants: encryption: "*" display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}" tasks: *encryption-server-versions + rules: *encryption-exclude-rules # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7. - matrix_name: "tests-storage-engines" From a1349f35382286b171b63df5ba71fd29358ceb61 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 13:25:04 -0700 Subject: [PATCH 17/21] maybe this will work --- .evergreen/config.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b0338eae3a..05edeabd98 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2352,7 +2352,18 @@ buildvariants: encryption: "*" display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}" tasks: *encryption-server-versions - rules: *encryption-exclude-rules + rules: + - if: + platform: "*" + auth: "*" + ssl: "*" + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" - matrix_name: "tests-python-version-ubuntu18-without-c-extensions" @@ -2450,7 +2461,18 @@ buildvariants: encryption: "*" display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}" tasks: *encryption-server-versions - rules: *encryption-exclude-rules + rules: + - if: + platform: "*" + auth: "*" + ssl: "*" + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7. - matrix_name: "tests-storage-engines" From bf58852781d26dd866ddf6e0b760a5f40925f577 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 13:43:11 -0700 Subject: [PATCH 18/21] fix axes --- .evergreen/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 05edeabd98..9653471c21 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2355,8 +2355,8 @@ buildvariants: rules: - if: platform: "*" - auth: "*" - ssl: "*" + python-version: "*" + auth-ssl: "*" encryption: [ "encryption_with_csfle" ] then: remove_tasks: @@ -2464,8 +2464,8 @@ buildvariants: rules: - if: platform: "*" - auth: "*" - ssl: "*" + python-version-windows: "*" + auth-ssl: "*" encryption: [ "encryption_with_csfle" ] then: remove_tasks: From 6a35554498a81a488b1fd09da165ec3b605a4f98 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 13:52:54 -0700 Subject: [PATCH 19/21] fix pre-commit --- test/test_encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_encryption.py b/test/test_encryption.py index 632f4ed591..2b4a1b8416 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -1767,7 +1767,7 @@ def test_case_8(self): class TestBypassSpawningMongocryptdProse(EncryptionIntegrationTest): @unittest.skipIf( os.environ.get("TEST_CSFLE"), - "this prose test does not work when CSFLE is " "on a system dynamic library search path.", + "this prose test does not work when CSFLE is on a system dynamic library search path.", ) def test_mongocryptd_bypass_spawn(self): # Lower the mongocryptd timeout to reduce the test run time. From d40fc0946c742fbc9c435606309590f13b8ad1c2 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 13:55:54 -0700 Subject: [PATCH 20/21] fix repetition of tasks --- .evergreen/config.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 94dd4c5578..cc92ee5860 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1262,33 +1262,6 @@ tasks: TOPOLOGY: "sharded_cluster" - func: "run tests" - - name: "test-6.0-standalone" - tags: ["6.0", "standalone"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "6.0" - TOPOLOGY: "server" - - func: "run tests" - - - name: "test-6.0-replica_set" - tags: ["6.0", "replica_set"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "6.0" - TOPOLOGY: "replica_set" - - func: "run tests" - - - name: "test-6.0-sharded_cluster" - tags: ["6.0", "sharded_cluster"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - VERSION: "6.0" - TOPOLOGY: "sharded_cluster" - - func: "run tests" - - name: "test-latest-standalone" tags: ["latest", "standalone"] commands: From 19fb58674183f2ff704ad3ea446f28fead4e0a63 Mon Sep 17 00:00:00 2001 From: julius Date: Thu, 26 May 2022 14:54:42 -0700 Subject: [PATCH 21/21] remove changes that shouldn't be there --- .evergreen/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index cc92ee5860..c12d4167b7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1236,7 +1236,7 @@ tasks: TOPOLOGY: "sharded_cluster" - func: "run tests" - name: "test-6.0-standalone" - tags: [ "6.0", "standalone" ] + tags: ["6.0", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: @@ -1245,7 +1245,7 @@ tasks: - func: "run tests" - name: "test-6.0-replica_set" - tags: [ "6.0", "replica_set" ] + tags: ["6.0", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: @@ -1254,7 +1254,7 @@ tasks: - func: "run tests" - name: "test-6.0-sharded_cluster" - tags: [ "6.0", "sharded_cluster" ] + tags: ["6.0", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: