diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3f8955f40e..c12d4167b7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -450,6 +450,9 @@ functions: export LIBMONGOCRYPT_URL="${libmongocrypt_url}" export TEST_ENCRYPTION=1 fi + if [ -n "${test_csfle}" ]; then + export TEST_CSFLE=1 + fi if [ -n "${test_pyopenssl}" ]; then export TEST_PYOPENSSL=1 fi @@ -1232,7 +1235,6 @@ tasks: VERSION: "5.0" TOPOLOGY: "sharded_cluster" - func: "run tests" - - name: "test-6.0-standalone" tags: ["6.0", "standalone"] commands: @@ -2161,6 +2163,14 @@ axes: variables: test_encryption: true batchtime: 10080 # 7 days + - id: "encryption_with_csfle" + display_name: "Encryption with CSFLE" + tags: ["encryption_tag", "csfle"] + variables: + test_encryption: true + test_csfle: true + batchtime: 10080 # 7 days + # Run pyopenssl tests? - id: pyopenssl @@ -2229,21 +2239,6 @@ buildvariants: - ".4.0" - ".3.6" -- matrix_name: "tests-all-encryption" - matrix_spec: - platform: - # OSes that support versions of MongoDB>=2.6 with SSL. - - awslinux - auth-ssl: "*" - encryption: "*" - display_name: "Encryption ${platform} ${auth-ssl}" - tasks: - - ".6.0" - - ".5.0" - - ".4.4" - - ".4.2" - - ".4.0" - - matrix_name: "tests-archlinux" matrix_spec: platform: @@ -2297,14 +2292,27 @@ buildvariants: auth: "auth" ssl: "nossl" encryption: "*" - display_name: "Encryption ${platform} ${auth} ${ssl}" + display_name: "${encryption} ${platform} ${auth} ${ssl}" tasks: &encryption-server-versions + - ".rapid" - ".latest" - ".6.0" - ".5.0" - ".4.4" - ".4.2" - ".4.0" + rules: &encryption-exclude-rules + - if: + platform: "*" + auth: "*" + ssl: "*" + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" # Test one server version with zSeries, POWER8, and ARM. - matrix_name: "test-different-cpu-architectures" @@ -2385,8 +2393,21 @@ buildvariants: # dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config # coverage: "*" encryption: "*" - display_name: "Encryption ${python-version} ${platform} ${auth-ssl}" + display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}" tasks: *encryption-server-versions + rules: + - if: + platform: "*" + python-version: "*" + auth-ssl: "*" + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" + - matrix_name: "tests-python-version-ubuntu18-without-c-extensions" matrix_spec: @@ -2481,8 +2502,20 @@ buildvariants: python-version-windows: "*" auth-ssl: "*" encryption: "*" - display_name: "Encryption ${platform} ${python-version-windows} ${auth-ssl}" + display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}" tasks: *encryption-server-versions + rules: + - if: + platform: "*" + python-version-windows: "*" + auth-ssl: "*" + encryption: [ "encryption_with_csfle" ] + then: + remove_tasks: + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" # Storage engine tests on Ubuntu 18.04 (x86_64) with Python 3.7. - matrix_name: "tests-storage-engines" diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4a48b4a33b..96f42fa517 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -11,6 +11,7 @@ set -o errexit # Exit the script with error if any of the commands fail # COVERAGE If non-empty, run the test suite with coverage. # TEST_ENCRYPTION If non-empty, install pymongocrypt. # LIBMONGOCRYPT_URL The URL to download libmongocrypt. +# TEST_CSFLE If non-empty, install CSFLE if [ -n "${SET_XTRACE_ON}" ]; then set -o xtrace @@ -27,6 +28,7 @@ COVERAGE=${COVERAGE:-} COMPRESSORS=${COMPRESSORS:-} MONGODB_API_VERSION=${MONGODB_API_VERSION:-} TEST_ENCRYPTION=${TEST_ENCRYPTION:-} +TEST_CSFLE=${TEST_CSFLE:-} LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-} DATA_LAKE=${DATA_LAKE:-} @@ -153,7 +155,16 @@ if [ -z "$DATA_LAKE" ]; then else TEST_ARGS="-s test.test_data_lake" fi - +if [ -z $TEST_CSFLE ]; then + echo "CSFLE not being tested" +else + $PYTHON $DRIVERS_TOOLS/.evergreen/mongodl.py --component csfle \ + --version latest --out ../csfle/ + export DYLD_FALLBACK_LIBRARY_PATH=../csfle/lib/:$DYLD_FALLBACK_LIBRARY_PATH + export LD_LIBRARY_PATH=../csfle/lib:$LD_LIBRARY_PATH + export PATH=../csfle/bin:$PATH + TEST_ARGS="-s test.test_encryption" +fi # Don't download unittest-xml-reporting from pypi, which often fails. if $PYTHON -c "import xmlrunner"; then # The xunit output dir must be a Python style absolute path. diff --git a/pymongo/encryption.py b/pymongo/encryption.py index 1e06f7062d..1a29131890 100644 --- a/pymongo/encryption.py +++ b/pymongo/encryption.py @@ -296,7 +296,14 @@ def _get_internal_client(encrypter, mongo_client): io_callbacks = _EncryptionIO(metadata_client, key_vault_coll, mongocryptd_client, opts) self._auto_encrypter = AutoEncrypter( - io_callbacks, MongoCryptOptions(opts._kms_providers, schema_map) + io_callbacks, + MongoCryptOptions( + opts._kms_providers, + schema_map, + csfle_path=opts._csfle_path, + csfle_required=opts._csfle_required, + bypass_encryption=opts._bypass_auto_encryption, + ), ) self._closed = False diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 2ac12bc4b4..0ce828ae4c 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -39,12 +39,14 @@ def __init__( key_vault_namespace: str, key_vault_client: Optional["MongoClient"] = None, schema_map: Optional[Mapping[str, Any]] = None, - bypass_auto_encryption: Optional[bool] = False, + bypass_auto_encryption: bool = False, mongocryptd_uri: str = "mongodb://localhost:27020", mongocryptd_bypass_spawn: bool = False, mongocryptd_spawn_path: str = "mongocryptd", mongocryptd_spawn_args: Optional[List[str]] = None, kms_tls_options: Optional[Mapping[str, Any]] = None, + csfle_path: Optional[str] = None, + csfle_required: bool = False, ) -> None: """Options to configure automatic client-side field level encryption. @@ -140,6 +142,12 @@ def __init__( Or to supply a client certificate:: kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}} + - `csfle_path` (optional): Override the path to load the CSFLE library. + - `csfle_required` (optional): If 'true', refuse to continue encryption without a CSFLE + library + + .. versionchanged:: 4.2 + Added `csfle_path` and `csfle_required` parameters .. versionchanged:: 4.0 Added the `kms_tls_options` parameter and the "kmip" KMS provider. @@ -152,7 +160,8 @@ def __init__( "install a compatible version with: " "python -m pip install 'pymongo[encryption]'" ) - + self._csfle_path = csfle_path + self._csfle_required = csfle_required self._kms_providers = kms_providers self._key_vault_namespace = key_vault_namespace self._key_vault_client = key_vault_client diff --git a/test/test_encryption.py b/test/test_encryption.py index 366c406b03..2b4a1b8416 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,6 +82,18 @@ def get_client_opts(client): class TestAutoEncryptionOpts(PyMongoTestCase): + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + @unittest.skipUnless(os.environ.get("TEST_CSFLE"), "csfle is not installed") + def test_csfle(self): + # Test that we can pick up csfle automatically + client = MongoClient( + auto_encryption_opts=AutoEncryptionOpts( + KMS_PROVIDERS, "keyvault.datakeys", csfle_required=True + ), + connect=False, + ) + self.addCleanup(client.close) + @unittest.skipIf(_HAVE_PYMONGOCRYPT, "pymongocrypt is installed") def test_init_requires_pymongocrypt(self): with self.assertRaises(ConfigurationError): @@ -1753,6 +1765,10 @@ def test_case_8(self): # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.rst#bypass-spawning-mongocryptd class TestBypassSpawningMongocryptdProse(EncryptionIntegrationTest): + @unittest.skipIf( + os.environ.get("TEST_CSFLE"), + "this prose test does not work when CSFLE is on a system dynamic library search path.", + ) def test_mongocryptd_bypass_spawn(self): # Lower the mongocryptd timeout to reduce the test run time. self._original_timeout = encryption._MONGOCRYPTD_TIMEOUT_MS