From ceec6de5a3422bf37c5fed025095330f034d09f4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 27 Sep 2022 15:28:34 -0500 Subject: [PATCH 01/18] PYTHON-3367 Add support for GCP attached service accounts when using GCP KMS --- .evergreen/config.yml | 91 ++++++++++++++++++++++++++ .evergreen/run-mongodb-fle-gcp-auto.sh | 40 +++++++++++ .evergreen/run-tests.sh | 5 +- test/test_encryption.py | 32 +++++++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 .evergreen/run-mongodb-fle-gcp-auto.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9d016f4d8a..7ec15b795f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1016,6 +1016,43 @@ task_groups: tasks: - ".serverless" + - name: testgcpkms_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: prepare resources + - func: fix absolute paths + - func: make files executable + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json + export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json + export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS + export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}" + export GCPKMS_MACHINETYPE="e2-standard-4" + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh + # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. + - command: expansions.update + params: + file: testgcpkms-expansions.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + tasks: + - testgcpkms-task + tasks: # Wildcard task. Do you need to find out what tools are available and where? # Throw it here, and execute this task on all buildvariants @@ -1838,6 +1875,51 @@ tasks: commands: - func: "download and merge coverage" + - name: "testgcpkms-task" + commands: + - command: shell.exec + type: setup + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + echo "Copying files ... begin" + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + tar czf /tmp/mongo-python-driver.tgz . + GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh + echo "Copying files ... end" + echo "Untarring file ... begin" + GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + echo "Untarring file ... end" + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + + - name: "testgcpkms-fail-task" + # testgcpkms-fail-task runs in a non-GCE environment. + # It is expected to fail to obtain GCE credentials. + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI='mongodb://localhost:27017' SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh axes: # Choice of distro @@ -2790,6 +2872,15 @@ buildvariants: tasks: - name: "load-balancer-test" +- name: testgcpkms-variant + display_name: "GCP KMS" + run_on: + - debian11-small + tasks: + - name: testgcpkms_task_group + batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - testgcpkms-fail-task + - name: Release display_name: Release batchtime: 20160 # 14 days diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh new file mode 100644 index 0000000000..3682cf04ac --- /dev/null +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -o xtrace +set -o errexit # Exit the script with error if any of the commands fail + +# Supported/used environment variables: +# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server +# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" +############################################ +# Main Program # +############################################ + +if [[ -z "$1" ]]; then + echo "usage: $0 " + exit 1 +fi +export MONGODB_URI="$1" + +if echo "$MONGODB_URI" | grep -q "@"; then + echo "MONGODB_URI unexpectedly contains user credentials in FLE GCP test!"; + exit 1 +fi +# Now we can safely enable xtrace +set -o xtrace + +# Install python3.7 with pip. +apt-get update +apt install python3.7 python3-pip -y + +authtest () { + echo "Running GCP Credential Acquisition Test with $PYTHON" + $PYTHON --version + $PYTHON -m pip install --upgrade wheel setuptools pip + cd src + $PYTHON -m pip install '.[encryption]' + $PYTHON test/test_encryption.py + cd - +} + +PYTHON="python3.7" authtest diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 9a0eb25e00..114334d2eb 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -140,7 +140,10 @@ if [ -n "$TEST_ENCRYPTION" ]; then export PYMONGOCRYPT_LIB # TODO: Test with 'pip install pymongocrypt' - git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git + git clone https://github.com/blink1073/libmongocrypt.git libmongocrypt_git + pushd libmongocrypt_git + git checkout PYTHON-3367 + popd python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt python -m pip install ./libmongocrypt_git/bindings/python python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" diff --git a/test/test_encryption.py b/test/test_encryption.py index 567d606893..75b6d1066a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2304,6 +2304,38 @@ def run_test(self, src_provider, dst_provider): self.assertEqual(decrypt_result2, "test") +class TestonDemandGCPCredentials(EncryptionIntegrationTest): + def setUp(self): + super(TestonDemandGCPCredentials, self).setUp() + self.master_key = { + "projectId": "devprod-drivers", + "location": "global", + "keyRing": "key-ring-csfle", + "keyName": "key-name-csfle", + } + + @unittest.skipIf(any(GCP_CREDS.values()), "GCP environment credentials are set") + def test_01_failure(self): + self.client_encryption = ClientEncryption( + kms_providers={"gcp": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=OPTS, + ) + with self.assertRaises(EncryptionError): + self.client_encryption.create_data_key("gcp", self.master_key) + + @unittest.skipUnless(any(GCP_CREDS.values()), "GCP environment credentials are not set") + def test_02_success(self): + self.client_encryption = ClientEncryption( + kms_providers={"gcp": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=OPTS, + ) + self.client_encryption.create_data_key("gcp", self.master_key) + + class TestQueryableEncryptionDocsExample(EncryptionIntegrationTest): # Queryable Encryption is not supported on Standalone topology. @client_context.require_no_standalone From 84cdc94ba6567b0a40642bc44e2ba4c5696c4460 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 27 Sep 2022 16:12:12 -0500 Subject: [PATCH 02/18] clean up test invocation --- .evergreen/config.yml | 4 ++-- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 +- test/test_encryption.py | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 7ec15b795f..6cf1662e37 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1906,7 +1906,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1919,7 +1919,7 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - MONGODB_URI='mongodb://localhost:27017' SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh + SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017 axes: # Choice of distro diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 3682cf04ac..87a8611294 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -33,7 +33,7 @@ authtest () { $PYTHON -m pip install --upgrade wheel setuptools pip cd src $PYTHON -m pip install '.[encryption]' - $PYTHON test/test_encryption.py + TEST_FLE_GCP_AUTO=1 $PYTHON test/test_encryption.py cd - } diff --git a/test/test_encryption.py b/test/test_encryption.py index 75b6d1066a..cb1d1458c4 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2314,8 +2314,10 @@ def setUp(self): "keyName": "key-name-csfle", } - @unittest.skipIf(any(GCP_CREDS.values()), "GCP environment credentials are set") + @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_01_failure(self): + if os.getenv("SUCCESS"): + self.skipTest("Expecting success") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, key_vault_namespace="keyvault.datakeys", @@ -2325,8 +2327,10 @@ def test_01_failure(self): with self.assertRaises(EncryptionError): self.client_encryption.create_data_key("gcp", self.master_key) - @unittest.skipUnless(any(GCP_CREDS.values()), "GCP environment credentials are not set") + @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_02_success(self): + if not os.getenv("SUCCESS"): + self.skipTest("Expecting failure") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, key_vault_namespace="keyvault.datakeys", From 2d4b129b8537ccae39e564a04e39f25e70388be2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 10:28:02 -0500 Subject: [PATCH 03/18] handle evergreen host --- .evergreen/run-mongodb-fle-gcp-auto.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 87a8611294..8352130431 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -23,9 +23,9 @@ fi # Now we can safely enable xtrace set -o xtrace -# Install python3.7 with pip. -apt-get update -apt install python3.7 python3-pip -y +# Install python3.7 with pip +apt-get update; true +apt install python3.7 python3-pip -y; true authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" From b7ed5cb6e33292ceb2ae9888666a192cc3098a60 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 10:34:32 -0500 Subject: [PATCH 04/18] more cleanup --- .evergreen/config.yml | 2 +- .evergreen/run-mongodb-fle-gcp-auto.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6cf1662e37..23264349a6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1893,7 +1893,7 @@ tasks: GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 8352130431..e8e7929164 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -23,9 +23,11 @@ fi # Now we can safely enable xtrace set -o xtrace +set +o errexit # Install python3.7 with pip -apt-get update; true -apt install python3.7 python3-pip -y; true +apt-get update +apt install python3.7 python3-pip -y +set -o errexit authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" From a77b772318f91a13c2760cc1bf4b213b61781db3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 11:02:09 -0500 Subject: [PATCH 05/18] try sudo pip --- .evergreen/run-mongodb-fle-gcp-auto.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index e8e7929164..f85576373f 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -23,11 +23,9 @@ fi # Now we can safely enable xtrace set -o xtrace -set +o errexit # Install python3.7 with pip -apt-get update -apt install python3.7 python3-pip -y -set -o errexit +sudo apt-get update +sudo apt install python3.7 python3-pip -y authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" From eee7984e0ffb4e24085a348f2cf5d3310dcc1409 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 11:25:57 -0500 Subject: [PATCH 06/18] use python3 --- .evergreen/run-mongodb-fle-gcp-auto.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index f85576373f..f2e81dad62 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -23,10 +23,6 @@ fi # Now we can safely enable xtrace set -o xtrace -# Install python3.7 with pip -sudo apt-get update -sudo apt install python3.7 python3-pip -y - authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" $PYTHON --version @@ -37,4 +33,4 @@ authtest () { cd - } -PYTHON="python3.7" authtest +PYTHON="python3" authtest From 1e6337eb93d4d5c7d977e09759e6c131701012ad Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 11:33:11 -0500 Subject: [PATCH 07/18] fix script --- .evergreen/run-mongodb-fle-gcp-auto.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index f2e81dad62..8c9a055522 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -27,10 +27,10 @@ authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip - cd src + pushd src $PYTHON -m pip install '.[encryption]' TEST_FLE_GCP_AUTO=1 $PYTHON test/test_encryption.py - cd - + popd } PYTHON="python3" authtest From e08290d4355e0c3dd77d413ca760fa8c0a6a20e8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 11:36:11 -0500 Subject: [PATCH 08/18] fix script --- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 8c9a055522..00b5151520 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -27,10 +27,8 @@ authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip - pushd src $PYTHON -m pip install '.[encryption]' TEST_FLE_GCP_AUTO=1 $PYTHON test/test_encryption.py - popd } PYTHON="python3" authtest From f45f0044f90e453c0992e8bfe5ce91aecde282ee Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 13:53:29 -0500 Subject: [PATCH 09/18] clean up --- .evergreen/run-mongodb-fle-gcp-auto.sh | 3 ++- test/test_encryption.py | 36 -------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 00b5151520..294d830277 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -28,7 +28,8 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install '.[encryption]' - TEST_FLE_GCP_AUTO=1 $PYTHON test/test_encryption.py + $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3367.zip + TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } PYTHON="python3" authtest diff --git a/test/test_encryption.py b/test/test_encryption.py index cb1d1458c4..567d606893 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2304,42 +2304,6 @@ def run_test(self, src_provider, dst_provider): self.assertEqual(decrypt_result2, "test") -class TestonDemandGCPCredentials(EncryptionIntegrationTest): - def setUp(self): - super(TestonDemandGCPCredentials, self).setUp() - self.master_key = { - "projectId": "devprod-drivers", - "location": "global", - "keyRing": "key-ring-csfle", - "keyName": "key-name-csfle", - } - - @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") - def test_01_failure(self): - if os.getenv("SUCCESS"): - self.skipTest("Expecting success") - self.client_encryption = ClientEncryption( - kms_providers={"gcp": {}}, - key_vault_namespace="keyvault.datakeys", - key_vault_client=client_context.client, - codec_options=OPTS, - ) - with self.assertRaises(EncryptionError): - self.client_encryption.create_data_key("gcp", self.master_key) - - @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") - def test_02_success(self): - if not os.getenv("SUCCESS"): - self.skipTest("Expecting failure") - self.client_encryption = ClientEncryption( - kms_providers={"gcp": {}}, - key_vault_namespace="keyvault.datakeys", - key_vault_client=client_context.client, - codec_options=OPTS, - ) - self.client_encryption.create_data_key("gcp", self.master_key) - - class TestQueryableEncryptionDocsExample(EncryptionIntegrationTest): # Queryable Encryption is not supported on Standalone topology. @client_context.require_no_standalone From 7e0ce9cb466012888b01740a32e92f43700e26e4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 13:53:41 -0500 Subject: [PATCH 10/18] add on demand creds file --- test/test_on_demand_csfle.py | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/test_on_demand_csfle.py diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py new file mode 100644 index 0000000000..eebc65d50a --- /dev/null +++ b/test/test_on_demand_csfle.py @@ -0,0 +1,67 @@ +# Copyright 2019-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test client side encryption with on demand credentials.""" +import os +import sys +import unittest + +sys.path[0:0] = [""] + +from test import IntegrationTest, client_context + +from bson.codec_options import CodecOptions +from pymongo.encryption import _HAVE_PYMONGOCRYPT, ClientEncryption, EncryptionError + + +class TestonDemandGCPCredentials(IntegrationTest): + @classmethod + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + @client_context.require_version_min(4, 2, -1) + def setUpClass(cls): + super(TestonDemandGCPCredentials, cls).setUpClass() + + def setUp(self): + super(TestonDemandGCPCredentials, self).setUp() + self.master_key = { + "projectId": "devprod-drivers", + "location": "global", + "keyRing": "key-ring-csfle", + "keyName": "key-name-csfle", + } + + @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") + def test_01_failure(self): + if os.getenv("SUCCESS"): + self.skipTest("Expecting success") + self.client_encryption = ClientEncryption( + kms_providers={"gcp": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=CodecOptions(), + ) + with self.assertRaises(EncryptionError): + self.client_encryption.create_data_key("gcp", self.master_key) + + @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") + def test_02_success(self): + if not os.getenv("SUCCESS"): + self.skipTest("Expecting failure") + self.client_encryption = ClientEncryption( + kms_providers={"gcp": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=CodecOptions(), + ) + self.client_encryption.create_data_key("gcp", self.master_key) From cbaa9e7669d9ad6275a2d883efcfc85a324a2fc6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 28 Sep 2022 14:14:07 -0500 Subject: [PATCH 11/18] use subdirectory --- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 294d830277..393774e3f1 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -28,7 +28,7 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install '.[encryption]' - $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3367.zip + $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3367.zip#subdirectory=bindings/python TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } From 53e68270fcc8e3efd566fa52f79201f904d9e19b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 6 Oct 2022 14:33:36 -0500 Subject: [PATCH 12/18] address regression and add trove classifier --- setup.py | 1 + test/test_client.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 524c1303e6..028a7db61f 100755 --- a/setup.py +++ b/setup.py @@ -330,6 +330,7 @@ def build_extension(self, ext): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database", diff --git a/test/test_client.py b/test/test_client.py index 7e7e14c0e5..5bb116dbda 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1641,7 +1641,8 @@ def server_description_count(): # If a bug like PYTHON-2433 is reintroduced then too many # ServerDescriptions will be kept alive and this test will fail: # AssertionError: 19 != 46 within 15 delta (27 difference) - self.assertAlmostEqual(initial_count, final_count, delta=15) + # On Python 3.11 we seem to get more of a delta. + self.assertAlmostEqual(initial_count, final_count, delta=20) @client_context.require_failCommand_fail_point def test_network_error_message(self): From fd389bf964dd021899e56e0faf3c0a679bc114d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 6 Oct 2022 14:34:47 -0500 Subject: [PATCH 13/18] Revert "address regression and add trove classifier" This reverts commit 53e68270fcc8e3efd566fa52f79201f904d9e19b. --- setup.py | 1 - test/test_client.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 028a7db61f..524c1303e6 100755 --- a/setup.py +++ b/setup.py @@ -330,7 +330,6 @@ def build_extension(self, ext): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database", diff --git a/test/test_client.py b/test/test_client.py index 5bb116dbda..7e7e14c0e5 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1641,8 +1641,7 @@ def server_description_count(): # If a bug like PYTHON-2433 is reintroduced then too many # ServerDescriptions will be kept alive and this test will fail: # AssertionError: 19 != 46 within 15 delta (27 difference) - # On Python 3.11 we seem to get more of a delta. - self.assertAlmostEqual(initial_count, final_count, delta=20) + self.assertAlmostEqual(initial_count, final_count, delta=15) @client_context.require_failCommand_fail_point def test_network_error_message(self): From 70fee001b0b5d54511db923f5d6c4e57c59fa479 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 6 Oct 2022 14:38:56 -0500 Subject: [PATCH 14/18] address review --- test/test_on_demand_csfle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index eebc65d50a..ae85566e5d 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -43,7 +43,7 @@ def setUp(self): @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_01_failure(self): - if os.getenv("SUCCESS"): + if os.getenv("SUCCESS", "").lower() == "true": self.skipTest("Expecting success") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, @@ -56,7 +56,7 @@ def test_01_failure(self): @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_02_success(self): - if not os.getenv("SUCCESS"): + if os.getenv("SUCCESS", "").lower() == "false": self.skipTest("Expecting failure") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, From 3e9da3859cd4762b8b8c57685c9f2da55254838d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Oct 2022 10:27:35 -0500 Subject: [PATCH 15/18] lint --- pymongo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/_version.py b/pymongo/_version.py index 7331d6ff25..78c325a23c 100644 --- a/pymongo/_version.py +++ b/pymongo/_version.py @@ -15,7 +15,7 @@ """Current version of PyMongo.""" from typing import Tuple, Union -version_tuple: Tuple[Union[int, str], ...] = (4, 4, 0, '.dev0') +version_tuple: Tuple[Union[int, str], ...] = (4, 4, 0, ".dev0") def get_version_string() -> str: From 3efa7ad3b7711003eb49075f08f33d1c0bcce8b5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 20 Oct 2022 15:22:07 -0500 Subject: [PATCH 16/18] address review --- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 +- .evergreen/run-tests.sh | 5 +---- test/test_on_demand_csfle.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 393774e3f1..81c4660275 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -28,7 +28,7 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install '.[encryption]' - $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3367.zip#subdirectory=bindings/python + $PYTHON -m pip install https://github.com/mongodb/libmongocrypt#subdirectory=bindings/python TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 114334d2eb..9a0eb25e00 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -140,10 +140,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then export PYMONGOCRYPT_LIB # TODO: Test with 'pip install pymongocrypt' - git clone https://github.com/blink1073/libmongocrypt.git libmongocrypt_git - pushd libmongocrypt_git - git checkout PYTHON-3367 - popd + git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt python -m pip install ./libmongocrypt_git/bindings/python python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index ae85566e5d..ce4bada741 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -1,4 +1,4 @@ -# Copyright 2019-present MongoDB, Inc. +# Copyright 2019-2022 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 7a237ed343da1beffe9c8f042da83b4207d8ab8a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 20 Oct 2022 16:50:25 -0500 Subject: [PATCH 17/18] address review --- test/test_on_demand_csfle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index ce4bada741..29e86da9bf 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -43,7 +43,7 @@ def setUp(self): @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_01_failure(self): - if os.getenv("SUCCESS", "").lower() == "true": + if os.environ["SUCCESS"].lower() == "true": self.skipTest("Expecting success") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, @@ -56,7 +56,7 @@ def test_01_failure(self): @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto") def test_02_success(self): - if os.getenv("SUCCESS", "").lower() == "false": + if os.environ["SUCCESS"].lower() == "false": self.skipTest("Expecting failure") self.client_encryption = ClientEncryption( kms_providers={"gcp": {}}, From 7b2734a2bf2db058cb4f62de1b63c777777f6ae5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 20 Oct 2022 17:45:24 -0500 Subject: [PATCH 18/18] update license date --- test/test_on_demand_csfle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index 29e86da9bf..408c942cc7 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -1,4 +1,4 @@ -# Copyright 2019-2022 MongoDB, Inc. +# Copyright 2022-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.