From f6f581b56ee9b5e2b44ecae837e0f02b3736ece7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 12:03:31 -0500 Subject: [PATCH 01/40] PYTHON-3396 Support the Azure VM-assigned Managed Identity for Automatic KMS Credentials --- .evergreen/config.yml | 90 ++++++++++++++++++++++++ .evergreen/run-mongodb-fle-azure-auto.sh | 35 +++++++++ test/test_on_demand_csfle.py | 42 +++++++++++ 3 files changed, 167 insertions(+) create mode 100644 .evergreen/run-mongodb-fle-azure-auto.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f0514681db..21fdd74693 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1072,6 +1072,45 @@ task_groups: tasks: - testgcpkms-task + - name: testazurekms_task_group + setup_group: + - command: shell.exec + params: + shell: bash + script: |- + set -o errexit + echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey + echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey + # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". + chmod 600 /tmp/testazurekms_privatekey + export AZUREKMS_CLIENTID=${testazurekms_clientid} + export AZUREKMS_TENANTID=${testazurekms_tenantid} + export AZUREKMS_SECRET=${testazurekms_secret} + export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey + export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + export AZUREKMS_SCOPE=${testazurekms_scope} + export AZUREKMS_VMNAME_PREFIX=CDRIVER + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh + - command: expansions.update + params: + file: testazurekms-expansions.yml + teardown_group: + - command: shell.exec + params: + shell: bash + script: |- + set -o errexit + DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - testazurekms-task + tasks: # Wildcard task. Do you need to find out what tools are available and where? # Throw it here, and execute this task on all buildvariants @@ -1940,6 +1979,57 @@ tasks: ${PREPARE_SHELL} SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017 + - name: testazurekms-task + commands: + - func: fetch source + - command: shell.exec + params: + shell: bash + script: |- + set -o errexit + echo "Copying files ... begin" + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + mkdir testazurekms + cp . testazurekms + tar czf testazurekms.tgz testazurekms/* + AZUREKMS_SRC="testazurekms.tgz" \ + AZUREKMS_DST="~/" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + echo "Copying files ... end" + echo "Untarring file ... begin" + AZUREKMS_CMD="tar xf testazurekms.tgz" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + echo "Untarring file ... end" + - command: shell.exec + type: test + params: + shell: bash + script: |- + set -o errexit + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' + SUCCESS=false + ./.evergreen/run-mongodb-fle-azure-auto.sh" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + - name: testazurekms-fail-task + commands: + - func: fetch source + - command: shell.exec + type: test + params: + shell: bash + script: |- + set -o errexit + LD_LIBRARY_PATH=./install \ + MONGODB_URI='mongodb://localhost:27017' \ + KEY_NAME='${testazurekms_keyname}' \ + KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ + SUCCESS=false + axes: # Choice of distro - id: platform diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh new file mode 100644 index 0000000000..72d8c7261d --- /dev/null +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o xtrace +set -o errexit # Exit the script with error if any of the commands fail + +# Supported/used environment variables: +# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server +# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" +############################################ +# Main Program # +############################################ + +if [[ -z "$1" ]]; then + echo "usage: $0 " + exit 1 +fi +export MONGODB_URI="$1" + +if echo "$MONGODB_URI" | grep -q "@"; then + echo "MONGODB_URI unexpectedly contains user credentials in FLE Azure test!"; + exit 1 +fi +# Now we can safely enable xtrace +set -o xtrace + +authtest () { + echo "Running Azure Credential Acquisition Test with $PYTHON" + $PYTHON --version + $PYTHON -m pip install --upgrade wheel setuptools pip + $PYTHON -m pip install '.[encryption]' + $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python + TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py +} + +PYTHON="python3" authtest diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index 408c942cc7..47e82ba009 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -65,3 +65,45 @@ def test_02_success(self): codec_options=CodecOptions(), ) self.client_encryption.create_data_key("gcp", self.master_key) + + +class TestonDemandAzureCredentials(IntegrationTest): + @classmethod + @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed") + @client_context.require_version_min(4, 2, -1) + def setUpClass(cls): + super(TestonDemandAzureCredentials, cls).setUpClass() + + def setUp(self): + super(TestonDemandAzureCredentials, self).setUp() + self.master_key = { + "projectId": "devprod-drivers", + "location": "global", + "keyRing": "key-ring-csfle", + "keyName": "key-name-csfle", + } + + @unittest.skipIf(not os.getenv("TEST_FLE_AZURE_AUTO"), "Not testing FLE Azure auto") + def test_01_failure(self): + if os.environ["SUCCESS"].lower() == "true": + self.skipTest("Expecting success") + self.client_encryption = ClientEncryption( + kms_providers={"azure": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=CodecOptions(), + ) + with self.assertRaises(EncryptionError): + self.client_encryption.create_data_key("azure", self.master_key) + + @unittest.skipIf(not os.getenv("TEST_FLE_AZURE_AUTO"), "Not testing FLE Azure auto") + def test_02_success(self): + if os.environ["SUCCESS"].lower() == "false": + self.skipTest("Expecting failure") + self.client_encryption = ClientEncryption( + kms_providers={"azure": {}}, + key_vault_namespace="keyvault.datakeys", + key_vault_client=client_context.client, + codec_options=CodecOptions(), + ) + self.client_encryption.create_data_key("azure", self.master_key) From dd5ed9c47365221e557ed4b41df7c874b3a920f8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 12:05:12 -0500 Subject: [PATCH 02/40] install from branch --- .evergreen/run-mongodb-fle-azure-auto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 72d8c7261d..1699b95178 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -28,7 +28,7 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install '.[encryption]' - $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python + $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py } From 1d7b46e25cd77fbc0aba6267407bc331b4ce49b7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 12:06:56 -0500 Subject: [PATCH 03/40] update master key --- test/test_on_demand_csfle.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index 47e82ba009..25f0127c4a 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -77,10 +77,8 @@ def setUpClass(cls): def setUp(self): super(TestonDemandAzureCredentials, self).setUp() self.master_key = { - "projectId": "devprod-drivers", - "location": "global", - "keyRing": "key-ring-csfle", - "keyName": "key-name-csfle", + "keyVaultEndpoint": "https://keyvault-drivers-2411.vault.azure.net/keys/", + "keyName": "KEY-NAME", } @unittest.skipIf(not os.getenv("TEST_FLE_AZURE_AUTO"), "Not testing FLE Azure auto") From 22ec3dee0a8da26cb9ef80e3f4bec0f64df6cae5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 12:10:00 -0500 Subject: [PATCH 04/40] add azure variant --- .evergreen/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 21fdd74693..0dcda3fbba 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -3016,6 +3016,14 @@ buildvariants: batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - testgcpkms-fail-task +- name: testazurekms-variant + display_name: "Azure KMS" + run_on: debian11-small + tasks: + - name: testazurekms_task_group + batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - testazurekms-fail-task + - name: Release display_name: Release batchtime: 20160 # 14 days From 105edd2d5eae697609be60bc3b0fbf0e45db8670 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 12:17:10 -0500 Subject: [PATCH 05/40] prepare resources --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0dcda3fbba..a48707d251 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1074,6 +1074,7 @@ task_groups: - name: testazurekms_task_group setup_group: + - func: prepare resources - command: shell.exec params: shell: bash @@ -1102,8 +1103,7 @@ task_groups: shell: bash script: |- set -o errexit - DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh setup_group_can_fail_task: true From 3604eeccec1b4b0bc43741d7ecbfef5f6912281e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 13:28:11 -0500 Subject: [PATCH 06/40] fetch source --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a48707d251..d3a6f2889e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1074,6 +1074,7 @@ task_groups: - name: testazurekms_task_group setup_group: + - func: fetch source - func: prepare resources - command: shell.exec params: @@ -1981,7 +1982,6 @@ tasks: - name: testazurekms-task commands: - - func: fetch source - command: shell.exec params: shell: bash From 1c4b26225aeee5033ccb9e22e6501ed9c96bf2d5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 13:36:32 -0500 Subject: [PATCH 07/40] fix env variable --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d3a6f2889e..5c263e0ccd 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2011,6 +2011,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh" \ From 3a6fa9247950efe170acda92a32d37a47a6072d7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 13:56:31 -0500 Subject: [PATCH 08/40] more task cleanup --- .evergreen/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5c263e0ccd..4f99614382 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1076,11 +1076,13 @@ task_groups: setup_group: - func: fetch source - func: prepare resources + - func: fix absolute paths + - func: make files executable - command: shell.exec params: shell: bash script: |- - set -o errexit + ${PREPARE_SHELL} echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". @@ -1103,7 +1105,7 @@ task_groups: params: shell: bash script: |- - set -o errexit + ${PREPARE_SHELL} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh From 7b91f1a8c278c805f00da701cd880e0d2624c027 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 15:06:07 -0500 Subject: [PATCH 09/40] debug --- .evergreen/config.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4f99614382..55ee7d3c63 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1095,7 +1095,34 @@ task_groups: export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey export AZUREKMS_SCOPE=${testazurekms_scope} - export AZUREKMS_VMNAME_PREFIX=CDRIVER + export AZUREKMS_VMNAME_PREFIX=PYTHON_DRIVER + if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then + echo "missing AZUREKMS_VMNAME_PREFIX" + fi + if [ -z "${AZUREKMS_CLIENTID:-}" ]; then + echo "missing AZUREKMS_CLIENTID" + fi + if [ -z "${AZUREKMS_TENANTID:-}" ]; then + echo "missing AZUREKMS_TENANTID" + fi + if [ -z "${AZUREKMS_SECRET:-}" ]; then + echo "missing AZUREKMS_SECRET" + fi + if [ -z "${AZUREKMS_DRIVERS_TOOLS:-}" ]; then + echo "missing AZUREKMS_DRIVERS_TOOLS" + fi + if [ -z "${AZUREKMS_RESOURCEGROUP:-}" ]; then + echo "missing AZUREKMS_RESOURCEGROUP" + fi + if [ -z "${AZUREKMS_PUBLICKEYPATH:-}" ]; then + echo "missing AZUREKMS_PUBLICKEYPATH" + fi + if [ -z "${AZUREKMS_PRIVATEKEYPATH:-}" ]; then + echo "missing AZUREKMS_PRIVATEKEYPATH" + fi + if [ -z "${AZUREKMS_SCOPE:-}" ]; then + echo "missing AZUREKMS_SCOPE" + fi $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: From de4f7f004b86fe7d1dd2e8ac8de88888f98b5e33 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 15:17:15 -0500 Subject: [PATCH 10/40] more debug --- .evergreen/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 55ee7d3c63..0f4b5d9936 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1096,6 +1096,8 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey export AZUREKMS_SCOPE=${testazurekms_scope} export AZUREKMS_VMNAME_PREFIX=PYTHON_DRIVER + echo "hello ${AZUREKMS_VMNAME_PREFIX}" + echo "hello ${AZUREKMS_VMNAME_PREFIX:-}" if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then echo "missing AZUREKMS_VMNAME_PREFIX" fi @@ -2054,11 +2056,10 @@ tasks: shell: bash script: |- set -o errexit - LD_LIBRARY_PATH=./install \ - MONGODB_URI='mongodb://localhost:27017' \ KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false + SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh \ + mongodb://localhost:27017 axes: # Choice of distro From c0835b67507a69ae5c25834ffadbab9a3c53f6c5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 15:47:01 -0500 Subject: [PATCH 11/40] more debug --- .evergreen/config.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0f4b5d9936..6a543ff76a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1087,17 +1087,17 @@ task_groups: echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". chmod 600 /tmp/testazurekms_privatekey - export AZUREKMS_CLIENTID=${testazurekms_clientid} - export AZUREKMS_TENANTID=${testazurekms_tenantid} - export AZUREKMS_SECRET=${testazurekms_secret} - export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - export AZUREKMS_SCOPE=${testazurekms_scope} - export AZUREKMS_VMNAME_PREFIX=PYTHON_DRIVER - echo "hello ${AZUREKMS_VMNAME_PREFIX}" - echo "hello ${AZUREKMS_VMNAME_PREFIX:-}" + export AZUREKMS_CLIENTID="${testazurekms_clientid}" + export AZUREKMS_TENANTID="${testazurekms_tenantid}" + export AZUREKMS_SECRET="${testazurekms_secret}" + export AZUREKMS_DRIVERS_TOOLS="$DRIVERS_TOOLS" + export AZUREKMS_RESOURCEGROUP="${testazurekms_resourcegroup}" + export AZUREKMS_PUBLICKEYPATH="/tmp/testazurekms_publickey" + export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" + export AZUREKMS_SCOPE="${testazurekms_scope}" + export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" + echo "hello ${AZUREKMS_VMNAME_PREFIX} there" + echo "hello ${AZUREKMS_VMNAME_PREFIX:-} there" if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then echo "missing AZUREKMS_VMNAME_PREFIX" fi @@ -2050,6 +2050,7 @@ tasks: - name: testazurekms-fail-task commands: - func: fetch source + - func: make files executable - command: shell.exec type: test params: From a980dd27ce1e158fe2d58eca09555d5acdf2ad3b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 16:02:57 -0500 Subject: [PATCH 12/40] more debug --- .evergreen/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6a543ff76a..a0185180ac 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1082,11 +1082,13 @@ task_groups: params: shell: bash script: |- + set -o errexit ${PREPARE_SHELL} echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". chmod 600 /tmp/testazurekms_privatekey + echo "Setting env variables..." export AZUREKMS_CLIENTID="${testazurekms_clientid}" export AZUREKMS_TENANTID="${testazurekms_tenantid}" export AZUREKMS_SECRET="${testazurekms_secret}" @@ -1096,6 +1098,7 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" export AZUREKMS_SCOPE="${testazurekms_scope}" export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" + echo "Set env variables..." echo "hello ${AZUREKMS_VMNAME_PREFIX} there" echo "hello ${AZUREKMS_VMNAME_PREFIX:-} there" if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then From 65a64d84f198a93d49bbc7e8f50fd3f01d131d10 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 17:22:32 -0500 Subject: [PATCH 13/40] more debug --- .evergreen/config.yml | 34 ++-------------------------------- check_stuff.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 32 deletions(-) create mode 100644 check_stuff.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a0185180ac..8b141809fb 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1088,7 +1088,6 @@ task_groups: echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". chmod 600 /tmp/testazurekms_privatekey - echo "Setting env variables..." export AZUREKMS_CLIENTID="${testazurekms_clientid}" export AZUREKMS_TENANTID="${testazurekms_tenantid}" export AZUREKMS_SECRET="${testazurekms_secret}" @@ -1098,36 +1097,7 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" export AZUREKMS_SCOPE="${testazurekms_scope}" export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" - echo "Set env variables..." - echo "hello ${AZUREKMS_VMNAME_PREFIX} there" - echo "hello ${AZUREKMS_VMNAME_PREFIX:-} there" - if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then - echo "missing AZUREKMS_VMNAME_PREFIX" - fi - if [ -z "${AZUREKMS_CLIENTID:-}" ]; then - echo "missing AZUREKMS_CLIENTID" - fi - if [ -z "${AZUREKMS_TENANTID:-}" ]; then - echo "missing AZUREKMS_TENANTID" - fi - if [ -z "${AZUREKMS_SECRET:-}" ]; then - echo "missing AZUREKMS_SECRET" - fi - if [ -z "${AZUREKMS_DRIVERS_TOOLS:-}" ]; then - echo "missing AZUREKMS_DRIVERS_TOOLS" - fi - if [ -z "${AZUREKMS_RESOURCEGROUP:-}" ]; then - echo "missing AZUREKMS_RESOURCEGROUP" - fi - if [ -z "${AZUREKMS_PUBLICKEYPATH:-}" ]; then - echo "missing AZUREKMS_PUBLICKEYPATH" - fi - if [ -z "${AZUREKMS_PRIVATEKEYPATH:-}" ]; then - echo "missing AZUREKMS_PRIVATEKEYPATH" - fi - if [ -z "${AZUREKMS_SCOPE:-}" ]; then - echo "missing AZUREKMS_SCOPE" - fi + src/check_stuff.sh $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: @@ -2062,7 +2032,7 @@ tasks: set -o errexit KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh \ + SUCCESS=false ./src/.evergreen/run-mongodb-fle-azure-auto.sh \ mongodb://localhost:27017 axes: diff --git a/check_stuff.sh b/check_stuff.sh new file mode 100644 index 0000000000..cbbe5d36eb --- /dev/null +++ b/check_stuff.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then + echo "missing AZUREKMS_VMNAME_PREFIX" + fi + if [ -z "${AZUREKMS_CLIENTID:-}" ]; then + echo "missing AZUREKMS_CLIENTID" + fi + if [ -z "${AZUREKMS_TENANTID:-}" ]; then + echo "missing AZUREKMS_TENANTID" + fi + if [ -z "${AZUREKMS_SECRET:-}" ]; then + echo "missing AZUREKMS_SECRET" + fi + if [ -z "${AZUREKMS_DRIVERS_TOOLS:-}" ]; then + echo "missing AZUREKMS_DRIVERS_TOOLS" + fi + if [ -z "${AZUREKMS_RESOURCEGROUP:-}" ]; then + echo "missing AZUREKMS_RESOURCEGROUP" + fi + if [ -z "${AZUREKMS_PUBLICKEYPATH:-}" ]; then + echo "missing AZUREKMS_PUBLICKEYPATH" + fi + if [ -z "${AZUREKMS_PRIVATEKEYPATH:-}" ]; then + echo "missing AZUREKMS_PRIVATEKEYPATH" + fi + if [ -z "${AZUREKMS_SCOPE:-}" ]; then + echo "missing AZUREKMS_SCOPE" + fi From d4c478bcb0f18290606225d96c88756be845ccf2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 17:44:50 -0500 Subject: [PATCH 14/40] more debug --- check_stuff.sh => .evergreen/check_stuff.sh | 0 .evergreen/config.yml | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) rename check_stuff.sh => .evergreen/check_stuff.sh (100%) diff --git a/check_stuff.sh b/.evergreen/check_stuff.sh similarity index 100% rename from check_stuff.sh rename to .evergreen/check_stuff.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8b141809fb..950aced6de 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1097,7 +1097,7 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" export AZUREKMS_SCOPE="${testazurekms_scope}" export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" - src/check_stuff.sh + ./src/.evergreen/check_stuff.sh $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: @@ -2030,9 +2030,10 @@ tasks: shell: bash script: |- set -o errexit + cd src KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false ./src/.evergreen/run-mongodb-fle-azure-auto.sh \ + SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh \ mongodb://localhost:27017 axes: From f1fc1bbe484a015489007a00e77b848995236093 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 17:53:54 -0500 Subject: [PATCH 15/40] more debug --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 950aced6de..d839f17293 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1097,6 +1097,7 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" export AZUREKMS_SCOPE="${testazurekms_scope}" export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" + echo "client id: ${testazurekms_clientid}" ./src/.evergreen/check_stuff.sh $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update From 5fcfe91dd903dc6a94232b60d517bb125d131759 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 17:56:12 -0500 Subject: [PATCH 16/40] more debug --- test/test_on_demand_csfle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index 25f0127c4a..a81e7a0965 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -105,3 +105,7 @@ def test_02_success(self): codec_options=CodecOptions(), ) self.client_encryption.create_data_key("azure", self.master_key) + + +if __name__ == "__main__": + unittest.main() From 42c2fb6b53d94be419074ec2316b5603c43755cb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 19:24:18 -0500 Subject: [PATCH 17/40] more debug --- .evergreen/check_stuff.sh | 29 ------------------------ .evergreen/config.yml | 2 -- .evergreen/run-mongodb-fle-azure-auto.sh | 2 +- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 +- 4 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 .evergreen/check_stuff.sh diff --git a/.evergreen/check_stuff.sh b/.evergreen/check_stuff.sh deleted file mode 100644 index cbbe5d36eb..0000000000 --- a/.evergreen/check_stuff.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -if [ -z "${AZUREKMS_VMNAME_PREFIX:-}" ]; then - echo "missing AZUREKMS_VMNAME_PREFIX" - fi - if [ -z "${AZUREKMS_CLIENTID:-}" ]; then - echo "missing AZUREKMS_CLIENTID" - fi - if [ -z "${AZUREKMS_TENANTID:-}" ]; then - echo "missing AZUREKMS_TENANTID" - fi - if [ -z "${AZUREKMS_SECRET:-}" ]; then - echo "missing AZUREKMS_SECRET" - fi - if [ -z "${AZUREKMS_DRIVERS_TOOLS:-}" ]; then - echo "missing AZUREKMS_DRIVERS_TOOLS" - fi - if [ -z "${AZUREKMS_RESOURCEGROUP:-}" ]; then - echo "missing AZUREKMS_RESOURCEGROUP" - fi - if [ -z "${AZUREKMS_PUBLICKEYPATH:-}" ]; then - echo "missing AZUREKMS_PUBLICKEYPATH" - fi - if [ -z "${AZUREKMS_PRIVATEKEYPATH:-}" ]; then - echo "missing AZUREKMS_PRIVATEKEYPATH" - fi - if [ -z "${AZUREKMS_SCOPE:-}" ]; then - echo "missing AZUREKMS_SCOPE" - fi diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d839f17293..029c9d6ccc 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1097,8 +1097,6 @@ task_groups: export AZUREKMS_PRIVATEKEYPATH="/tmp/testazurekms_privatekey" export AZUREKMS_SCOPE="${testazurekms_scope}" export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" - echo "client id: ${testazurekms_clientid}" - ./src/.evergreen/check_stuff.sh $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 1699b95178..655db16235 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -27,8 +27,8 @@ authtest () { echo "Running Azure Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip - $PYTHON -m pip install '.[encryption]' $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python + $PYTHON -m pip install '.[encryption]' TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py } diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 8b92551c10..447e5eacff 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -27,8 +27,8 @@ authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip - $PYTHON -m pip install '.[encryption]' $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python + $PYTHON -m pip install '.[encryption]' TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } From 71af809c4d8641f81bbd8dbeaa9ca7432dbdeeb6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 19:54:22 -0500 Subject: [PATCH 18/40] try again --- .evergreen/config.yml | 6 +++--- .evergreen/run-mongodb-fle-azure-auto.sh | 5 ++++- .evergreen/run-mongodb-fle-gcp-auto.sh | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 029c9d6ccc..32850f8d03 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1995,7 +1995,7 @@ tasks: export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey mkdir testazurekms - cp . testazurekms + cp -r . testazurekms tar czf testazurekms.tgz testazurekms/* AZUREKMS_SRC="testazurekms.tgz" \ AZUREKMS_DST="~/" \ @@ -3015,7 +3015,7 @@ buildvariants: - name: testgcpkms-variant display_name: "GCP KMS" run_on: - - debian11-small + - debian10-small tasks: - name: testgcpkms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -3023,7 +3023,7 @@ buildvariants: - name: testazurekms-variant display_name: "Azure KMS" - run_on: debian11-small + run_on: debian10-small tasks: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 655db16235..1e633d3954 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -28,7 +28,10 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python - $PYTHON -m pip install '.[encryption]' + curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz + mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all + export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib64/libmongocrypt.so + $PYTHON -m pip install '.' TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py } diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 447e5eacff..918ccc7345 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -28,7 +28,10 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python - $PYTHON -m pip install '.[encryption]' + curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz + mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all + export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib64/libmongocrypt.so + $PYTHON -m pip install '.' TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } From 8a8af23f2c79b02dfce1bf374b217b3e2f512e99 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 19:55:06 -0500 Subject: [PATCH 19/40] add verbosity --- test/test_on_demand_csfle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_on_demand_csfle.py b/test/test_on_demand_csfle.py index a81e7a0965..d5668199a3 100644 --- a/test/test_on_demand_csfle.py +++ b/test/test_on_demand_csfle.py @@ -108,4 +108,4 @@ def test_02_success(self): if __name__ == "__main__": - unittest.main() + unittest.main(verbosity=2) From 4b4f51bb37627b3b4c929f9a51a66b1c4fe9d19e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:01:29 -0500 Subject: [PATCH 20/40] fix path --- .evergreen/run-mongodb-fle-azure-auto.sh | 2 +- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 1e633d3954..6129be2d21 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -30,7 +30,7 @@ authtest () { $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib64/libmongocrypt.so + export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON -m pip install '.' TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py } diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 918ccc7345..1ab57a042d 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -30,7 +30,7 @@ authtest () { $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib64/libmongocrypt.so + export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON -m pip install '.' TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } From 1be977826c92c0b241930c303fb270547ba0b609 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:09:10 -0500 Subject: [PATCH 21/40] start servers on failing tests --- .evergreen/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 32850f8d03..1b5c312dd0 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1974,6 +1974,10 @@ tasks: # testgcpkms-fail-task runs in a non-GCE environment. # It is expected to fail to obtain GCE credentials. commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "server" - command: shell.exec type: test params: @@ -2023,6 +2027,10 @@ tasks: commands: - func: fetch source - func: make files executable + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "server" - command: shell.exec type: test params: From 2862a201a359773feb96e619992422fb18d1e539 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:13:50 -0500 Subject: [PATCH 22/40] more azure fixup --- .evergreen/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1b5c312dd0..291b49fac2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1998,9 +1998,9 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - mkdir testazurekms - cp -r . testazurekms - tar czf testazurekms.tgz testazurekms/* + mkdir /tmp/testazurekms + cp -r . /tmp/testazurekms + tar czf testazurekms.tgz /tmp/testazurekms/* AZUREKMS_SRC="testazurekms.tgz" \ AZUREKMS_DST="~/" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh From 3bbe1a71249c85e084044da74f2cecc9c6f2e9be Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:28:37 -0500 Subject: [PATCH 23/40] add prepare_shell --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 291b49fac2..1dcd3c8852 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1994,6 +1994,7 @@ tasks: shell: bash script: |- set -o errexit + ${PREPARE_SHELL} echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} From 2d50d1b91d650c3ee98abe230068d8d77728f16e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:42:59 -0500 Subject: [PATCH 24/40] more cleanup --- .evergreen/config.yml | 10 +++++----- .evergreen/run-mongodb-fle-azure-auto.sh | 7 ++----- .evergreen/run-mongodb-fle-gcp-auto.sh | 2 -- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1dcd3c8852..ae515b3deb 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2016,12 +2016,12 @@ tasks: shell: bash script: |- set -o errexit + ${PREPARE_SHELL} export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' - SUCCESS=false + AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' + SUCCESS=true ./.evergreen/run-mongodb-fle-azure-auto.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task @@ -2039,10 +2039,10 @@ tasks: script: |- set -o errexit cd src + MONGODB_URI="mongodb://localhost:27017" \ KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh \ - mongodb://localhost:27017 + SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh axes: # Choice of distro diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 6129be2d21..19eac0eae7 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -10,18 +10,15 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ -if [[ -z "$1" ]]; then - echo "usage: $0 " +if [[ -z "$MONGODB_URI" ]]; then + echo "Must define MONGODB_URI" exit 1 fi -export MONGODB_URI="$1" if echo "$MONGODB_URI" | grep -q "@"; then echo "MONGODB_URI unexpectedly contains user credentials in FLE Azure test!"; exit 1 fi -# Now we can safely enable xtrace -set -o xtrace authtest () { echo "Running Azure Credential Acquisition Test with $PYTHON" diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 1ab57a042d..1373527a21 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -20,8 +20,6 @@ if echo "$MONGODB_URI" | grep -q "@"; then echo "MONGODB_URI unexpectedly contains user credentials in FLE GCP test!"; exit 1 fi -# Now we can safely enable xtrace -set -o xtrace authtest () { echo "Running GCP Credential Acquisition Test with $PYTHON" From b6b5cdfc2ccadc560e2768b1decda540024ea59e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 20:54:41 -0500 Subject: [PATCH 25/40] more cleanup --- .evergreen/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ae515b3deb..819a7d109d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1995,13 +1995,14 @@ tasks: script: |- set -o errexit ${PREPARE_SHELL} + cd src echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey mkdir /tmp/testazurekms cp -r . /tmp/testazurekms - tar czf testazurekms.tgz /tmp/testazurekms/* + tar czfv testazurekms.tgz /tmp/testazurekms/* AZUREKMS_SRC="testazurekms.tgz" \ AZUREKMS_DST="~/" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh From 842225acd39fbbcbc1bd93fa17cca7bbddeb1caa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 21:09:25 -0500 Subject: [PATCH 26/40] try again --- .evergreen/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 819a7d109d..a60883072d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2000,15 +2000,13 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - mkdir /tmp/testazurekms - cp -r . /tmp/testazurekms - tar czfv testazurekms.tgz /tmp/testazurekms/* - AZUREKMS_SRC="testazurekms.tgz" \ + tar czf /tmp/mongo-python-driver.tgz . + AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" \ AZUREKMS_DST="~/" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf testazurekms.tgz" \ + AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" - command: shell.exec From 189ba0173fd089ca695a9f1e9d46401eb57c4b35 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 2 Nov 2022 21:28:07 -0500 Subject: [PATCH 27/40] fix syntax --- .evergreen/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a60883072d..2415db1da7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2019,9 +2019,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' - SUCCESS=true - ./.evergreen/run-mongodb-fle-azure-auto.sh" \ + AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true ./.evergreen/run-mongodb-fle-azure-auto.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task commands: From 235f0e4064ab9225a24757066871568e69570dd8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 3 Nov 2022 06:22:04 -0500 Subject: [PATCH 28/40] lint --- .evergreen/run-mongodb-fle-azure-auto.sh | 3 +-- .evergreen/run-mongodb-fle-gcp-auto.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index 19eac0eae7..fb28070c53 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -27,9 +27,8 @@ authtest () { $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON -m pip install '.' - TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py + PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py } PYTHON="python3" authtest diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index 1373527a21..4ed66ac8af 100644 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -28,9 +28,8 @@ authtest () { $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - export PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON -m pip install '.' - TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py + PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py } PYTHON="python3" authtest From 7aaed3c6f721466c77f6c07fbad289386c7445e8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 09:56:20 -0600 Subject: [PATCH 29/40] install pymongocrypt from master --- .evergreen/run-mongodb-fle-azure-auto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-azure-auto.sh index fb28070c53..cf2cccebed 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-azure-auto.sh @@ -24,7 +24,7 @@ authtest () { echo "Running Azure Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip - $PYTHON -m pip install https://github.com/blink1073/libmongocrypt/archive/refs/heads/PYTHON-3396.zip#subdirectory=bindings/python + $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all $PYTHON -m pip install '.' From 975a39fad76fdaeab1388ca78b683a62d7bcd99b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 14:55:37 -0600 Subject: [PATCH 30/40] address review --- .evergreen/config.yml | 11 +++--- ...-azure-auto.sh => run-mongodb-fle-auto.sh} | 6 ++-- .evergreen/run-mongodb-fle-gcp-auto.sh | 35 ------------------- 3 files changed, 9 insertions(+), 43 deletions(-) rename .evergreen/{run-mongodb-fle-azure-auto.sh => run-mongodb-fle-auto.sh} (88%) delete mode 100644 .evergreen/run-mongodb-fle-gcp-auto.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2415db1da7..0734779bc2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1080,8 +1080,9 @@ task_groups: - func: make files executable - command: shell.exec params: + silent: true shell: bash - script: |- + script: | set -o errexit ${PREPARE_SHELL} echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey @@ -1968,7 +1969,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh mongodb://localhost:27017" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1985,7 +1986,7 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017 + SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh mongodb://localhost:27017 - name: testazurekms-task commands: @@ -2019,7 +2020,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true ./.evergreen/run-mongodb-fle-azure-auto.sh" \ + AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task commands: @@ -2039,7 +2040,7 @@ tasks: MONGODB_URI="mongodb://localhost:27017" \ KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false ./.evergreen/run-mongodb-fle-azure-auto.sh + SUCCESS=false TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh axes: # Choice of distro diff --git a/.evergreen/run-mongodb-fle-azure-auto.sh b/.evergreen/run-mongodb-fle-auto.sh similarity index 88% rename from .evergreen/run-mongodb-fle-azure-auto.sh rename to .evergreen/run-mongodb-fle-auto.sh index cf2cccebed..572908e424 100644 --- a/.evergreen/run-mongodb-fle-azure-auto.sh +++ b/.evergreen/run-mongodb-fle-auto.sh @@ -16,19 +16,19 @@ if [[ -z "$MONGODB_URI" ]]; then fi if echo "$MONGODB_URI" | grep -q "@"; then - echo "MONGODB_URI unexpectedly contains user credentials in FLE Azure test!"; + echo "MONGODB_URI unexpectedly contains user credentials in FLE test!"; exit 1 fi authtest () { - echo "Running Azure Credential Acquisition Test with $PYTHON" + echo "Running Credential Acquisition Test with $PYTHON" $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all $PYTHON -m pip install '.' - PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so TEST_FLE_AZURE_AUTO=1 $PYTHON test/test_on_demand_csfle.py + PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON test/test_on_demand_csfle.py } PYTHON="python3" authtest diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh deleted file mode 100644 index 4ed66ac8af..0000000000 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit # Exit the script with error if any of the commands fail - -# Supported/used environment variables: -# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server -# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" -############################################ -# Main Program # -############################################ - -if [[ -z "$1" ]]; then - echo "usage: $0 " - exit 1 -fi -export MONGODB_URI="$1" - -if echo "$MONGODB_URI" | grep -q "@"; then - echo "MONGODB_URI unexpectedly contains user credentials in FLE GCP test!"; - exit 1 -fi - -authtest () { - echo "Running GCP Credential Acquisition Test with $PYTHON" - $PYTHON --version - $PYTHON -m pip install --upgrade wheel setuptools pip - $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python - curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz - mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - $PYTHON -m pip install '.' - PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py -} - -PYTHON="python3" authtest From 9a05d8aa82f3875a3452d66938b7ba9b85930cc3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 15:06:21 -0600 Subject: [PATCH 31/40] fix handling of mongodb uri --- .evergreen/run-mongodb-fle-auto.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/run-mongodb-fle-auto.sh b/.evergreen/run-mongodb-fle-auto.sh index 572908e424..710165e605 100644 --- a/.evergreen/run-mongodb-fle-auto.sh +++ b/.evergreen/run-mongodb-fle-auto.sh @@ -10,8 +10,10 @@ set -o errexit # Exit the script with error if any of the commands fail # Main Program # ############################################ -if [[ -z "$MONGODB_URI" ]]; then - echo "Must define MONGODB_URI" +MONGODB_URI=${MONGODB_URI:-mongodb://localhost:27017} + +if [[ -z "$SUCCESS" ]]; then + echo "Must define SUCCESS" exit 1 fi From abdd0b8779413e614e39411c0d4e7974ab1cac94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 15:15:46 -0600 Subject: [PATCH 32/40] try using run tests --- .evergreen/config.yml | 8 +++---- .evergreen/run-mongodb-fle-auto.sh | 36 ------------------------------ .evergreen/run-tests.sh | 14 ++++++++++++ 3 files changed, 18 insertions(+), 40 deletions(-) delete mode 100644 .evergreen/run-mongodb-fle-auto.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0734779bc2..ed91469f73 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1969,7 +1969,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh mongodb://localhost:27017" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1986,7 +1986,7 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh mongodb://localhost:27017 + SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh - name: testazurekms-task commands: @@ -2020,7 +2020,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh" \ + AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task commands: @@ -2040,7 +2040,7 @@ tasks: MONGODB_URI="mongodb://localhost:27017" \ KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-mongodb-fle-auto.sh + SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh axes: # Choice of distro diff --git a/.evergreen/run-mongodb-fle-auto.sh b/.evergreen/run-mongodb-fle-auto.sh deleted file mode 100644 index 710165e605..0000000000 --- a/.evergreen/run-mongodb-fle-auto.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit # Exit the script with error if any of the commands fail - -# Supported/used environment variables: -# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server -# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" -############################################ -# Main Program # -############################################ - -MONGODB_URI=${MONGODB_URI:-mongodb://localhost:27017} - -if [[ -z "$SUCCESS" ]]; then - echo "Must define SUCCESS" - exit 1 -fi - -if echo "$MONGODB_URI" | grep -q "@"; then - echo "MONGODB_URI unexpectedly contains user credentials in FLE test!"; - exit 1 -fi - -authtest () { - echo "Running Credential Acquisition Test with $PYTHON" - $PYTHON --version - $PYTHON -m pip install --upgrade wheel setuptools pip - $PYTHON -m pip install https://github.com/mongodb/libmongocrypt/archive/refs/heads/master.zip#subdirectory=bindings/python - curl -O https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz - mkdir libmongocrypt-all && tar xzf libmongocrypt-all.tar.gz -C libmongocrypt-all - $PYTHON -m pip install '.' - PYMONGOCRYPT_LIB=$(pwd)/libmongocrypt-all/debian10/nocrypto/lib/libmongocrypt.so $PYTHON test/test_on_demand_csfle.py -} - -PYTHON="python3" authtest diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index db20c9111e..ab75df1b5b 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -171,6 +171,20 @@ if [ -n "$TEST_ENCRYPTION" ]; then TEST_ARGS="-s test.test_encryption" fi +if [ -n "$TEST_FLE_AZURE_AUTO" || -n "$TEST_FLE_AZURE_AUTO" ]; then + if [[ -z "$SUCCESS" ]]; then + echo "Must define SUCCESS" + exit 1 + fi + + if echo "$MONGODB_URI" | grep -q "@"; then + echo "MONGODB_URI unexpectedly contains user credentials in FLE test!"; + exit 1 + fi + + TEST_ARGS="-s test.test_on_demand_csfle" +fi + if [ -n "$DATA_LAKE" ]; then TEST_ARGS="-s test.test_data_lake" fi From eba69bfeb3eade93eb65cba7376e24c245d2fcaa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 15:19:07 -0600 Subject: [PATCH 33/40] fix shell syntax --- .evergreen/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index ab75df1b5b..6dd19dc3ff 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -171,7 +171,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then TEST_ARGS="-s test.test_encryption" fi -if [ -n "$TEST_FLE_AZURE_AUTO" || -n "$TEST_FLE_AZURE_AUTO" ]; then +if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_AZURE_AUTO" ]; then if [[ -z "$SUCCESS" ]]; then echo "Must define SUCCESS" exit 1 From f446b6a2e7486178fbd623184b7eaf2bdf99345a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 15:55:55 -0600 Subject: [PATCH 34/40] fix handling of libmongocrypt_url --- .evergreen/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ed91469f73..978e82ffc3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2037,7 +2037,6 @@ tasks: script: |- set -o errexit cd src - MONGODB_URI="mongodb://localhost:27017" \ KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh @@ -3023,6 +3022,8 @@ buildvariants: display_name: "GCP KMS" run_on: - debian10-small + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz tasks: - name: testgcpkms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -3031,6 +3032,8 @@ buildvariants: - name: testazurekms-variant display_name: "Azure KMS" run_on: debian10-small + variables: + libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz tasks: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README From 6491ecf229284fa4ca354fd217f0d55421072713 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 16:15:18 -0600 Subject: [PATCH 35/40] fix handling of libmongocrypt_url --- .evergreen/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 978e82ffc3..5dc5ba21b9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1969,7 +1969,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 IBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1985,7 +1985,7 @@ tasks: working_dir: "src" shell: "bash" script: | - ${PREPARE_SHELL} + export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh - name: testazurekms-task @@ -2020,8 +2020,9 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh" \ + AZUREKMS_CMD="KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + - name: testazurekms-fail-task commands: - func: fetch source @@ -2036,10 +2037,13 @@ tasks: shell: bash script: |- set -o errexit + ${PREPARE_SHELL} cd src KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh + LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \ + SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 \ + ./.evergreen/run-tests.sh axes: # Choice of distro @@ -3022,8 +3026,6 @@ buildvariants: display_name: "GCP KMS" run_on: - debian10-small - variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz tasks: - name: testgcpkms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -3032,8 +3034,6 @@ buildvariants: - name: testazurekms-variant display_name: "Azure KMS" run_on: debian10-small - variables: - libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz tasks: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README From 92d996e7dbb17dbfd33925dd70dd786c60ba9f61 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 16:24:20 -0600 Subject: [PATCH 36/40] refactor --- .evergreen/config.yml | 3 ++- .evergreen/run-tests.sh | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5dc5ba21b9..31f806f738 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1969,7 +1969,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 IBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1985,6 +1985,7 @@ tasks: working_dir: "src" shell: "bash" script: | + ${PREPARE_SHELL} export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6dd19dc3ff..8abf0af762 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -101,7 +101,8 @@ if [ -n "$TEST_PYOPENSSL" ]; then python -m pip install --prefer-binary pyopenssl requests service_identity fi -if [ -n "$TEST_ENCRYPTION" ]; then +if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_AZURE_AUTO" ]; then + createvirtualenv $PYTHON venv-encryption trap "deactivate; rm -rf venv-encryption" EXIT HUP PYTHON=python @@ -146,7 +147,9 @@ if [ -n "$TEST_ENCRYPTION" ]; then python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" # PATH is updated by PREPARE_SHELL for access to mongocryptd. +fi +if [ -n "$TEST_ENCRYPTION" ]; then # Need aws dependency for On-Demand KMS Credentials. python -m pip install '.[aws]' From cbb6d67b30938c49c2c7caac5177423a1156546a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 16:43:30 -0600 Subject: [PATCH 37/40] more cleanup --- .evergreen/config.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 31f806f738..e591225427 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1969,7 +1969,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/run-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -1986,8 +1986,8 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz - SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh + export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1804-64/master/latest/libmongocrypt.tar.gz + SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/run-tests.sh - name: testazurekms-task commands: @@ -2021,7 +2021,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh" \ + AZUREKMS_CMD="KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/run-tests.sh" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task @@ -2040,10 +2040,11 @@ tasks: set -o errexit ${PREPARE_SHELL} cd src + PYTHON_BINARY= KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \ - SUCCESS=false TEST_ENCRYPTION=1 TEST_FLE_AZURE_AUTO=1 \ + LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/ubuntu1804-64/master/latest/libmongocrypt.tar.gz \ + SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ ./.evergreen/run-tests.sh axes: @@ -3026,7 +3027,7 @@ buildvariants: - name: testgcpkms-variant display_name: "GCP KMS" run_on: - - debian10-small + - ubuntu1804-test tasks: - name: testgcpkms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README @@ -3034,7 +3035,7 @@ buildvariants: - name: testazurekms-variant display_name: "Azure KMS" - run_on: debian10-small + run_on: ubuntu1804-test tasks: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README From 4ccf2a5bd80f3ba40bcab68de1afb06b8b64ef36 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 18:27:34 -0600 Subject: [PATCH 38/40] fix option --- .evergreen/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 8abf0af762..fed8c74f08 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -174,7 +174,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then TEST_ARGS="-s test.test_encryption" fi -if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_AZURE_AUTO" ]; then +if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then if [[ -z "$SUCCESS" ]]; then echo "Must define SUCCESS" exit 1 From 2d758af627f7ca132287579866697425dc2b268f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 7 Nov 2022 18:32:04 -0600 Subject: [PATCH 39/40] fix another option --- .evergreen/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index fed8c74f08..959ad901ad 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -101,7 +101,7 @@ if [ -n "$TEST_PYOPENSSL" ]; then python -m pip install --prefer-binary pyopenssl requests service_identity fi -if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_AZURE_AUTO" ]; then +if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then createvirtualenv $PYTHON venv-encryption trap "deactivate; rm -rf venv-encryption" EXIT HUP From 39898e60b7ec274670ffd4c3a9c00e25c732ea3f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 8 Nov 2022 09:11:44 -0600 Subject: [PATCH 40/40] restore chopping operator --- .evergreen/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e591225427..28e54e2ded 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1082,7 +1082,7 @@ task_groups: params: silent: true shell: bash - script: | + script: |- set -o errexit ${PREPARE_SHELL} echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey @@ -1109,8 +1109,8 @@ task_groups: script: |- ${PREPARE_SHELL} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: