PYTHON-3367 Add support for GCP attached service accounts when using GCP KMS

.evergreen/config.yml

@@ -1035,6 +1035,43 @@ task_groups:
     tasks:
       - ".serverless"
 
+  - name: testgcpkms_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - func: make files executable
+      - command: shell.exec
+        params:
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
+            export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
+            export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
+            export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
+            export GCPKMS_MACHINETYPE="e2-standard-4"
+            $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
+      # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions.
+      - command: expansions.update
+        params:
+          file: testgcpkms-expansions.yml
+    teardown_group:
+      - command: shell.exec
+        params:
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+            export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+            export GCPKMS_ZONE=${GCPKMS_ZONE}
+            export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+            $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
+    tasks:
+      - testgcpkms-task
+
 tasks:
     # Wildcard task. Do you need to find out what tools are available and where?
     # Throw it here, and execute this task on all buildvariants
@@ -1857,6 +1894,51 @@ tasks:
       commands:
         - func: "download and merge coverage"
 
+    - name: "testgcpkms-task"
+      commands:
+        - command: shell.exec
+          type: setup
+          params:
+            working_dir: "src"
+            shell: "bash"
+            script: |
+              ${PREPARE_SHELL}
+              echo "Copying files ... begin"
+              export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+              export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+              export GCPKMS_ZONE=${GCPKMS_ZONE}
+              export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+              tar czf /tmp/mongo-python-driver.tgz .
+              GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
+              echo "Copying files ... end"
+              echo "Untarring file ... begin"
+              GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+              echo "Untarring file ... end"
+        - command: shell.exec
+          type: test
+          params:
+            working_dir: "src"
+            shell: "bash"
+            script: |
+              ${PREPARE_SHELL}
+              export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+              export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+              export GCPKMS_ZONE=${GCPKMS_ZONE}
+              export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+              GCPKMS_CMD="SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+
+    - name: "testgcpkms-fail-task"
+      # testgcpkms-fail-task runs in a non-GCE environment.
+      # It is expected to fail to obtain GCE credentials.
+      commands:
+        - command: shell.exec
+          type: test
+          params:
+            working_dir: "src"
+            shell: "bash"
+            script: |
+              ${PREPARE_SHELL}
+              SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh mongodb://localhost:27017
 
 axes:
   # Choice of distro
@@ -2821,6 +2903,15 @@ buildvariants:
   tasks:
     - name: "load-balancer-test"
 
+- name: testgcpkms-variant
+  display_name: "GCP KMS"
+  run_on:
+    - debian11-small
+  tasks:
+    - name: testgcpkms_task_group
+      batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
+    - testgcpkms-fail-task
+
 - name: Release
   display_name: Release
   batchtime: 20160 # 14 days

.evergreen/run-mongodb-fle-gcp-auto.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# Supported/used environment variables:
+#       MONGODB_URI             Set the URI, including an optional username/password to use to connect to the server
+#       SUCCESS                 Whether the authentication is expected to succeed or fail.  One of "true" or "false"
+############################################
+#            Main Program                  #
+############################################
+
+if [[ -z "$1" ]]; then
+    echo "usage: $0 <MONGODB_URI>"
+    exit 1
+fi
+export MONGODB_URI="$1"
+
+if echo "$MONGODB_URI" | grep -q "@"; then
+  echo "MONGODB_URI unexpectedly contains user credentials in FLE GCP test!";
+  exit 1
+fi
+# Now we can safely enable xtrace
+set -o xtrace
+
+authtest () {
+    echo "Running GCP Credential Acquisition Test with $PYTHON"
+    $PYTHON --version
+    $PYTHON -m pip install --upgrade wheel setuptools pip
+    $PYTHON -m pip install '.[encryption]'
+    $PYTHON -m pip install https://github.com/mongodb/libmongocrypt#subdirectory=bindings/python
+    TEST_FLE_GCP_AUTO=1 $PYTHON test/test_on_demand_csfle.py
+}
+
+PYTHON="python3" authtest

pymongo/_version.py

@@ -15,7 +15,7 @@
 """Current version of PyMongo."""
 from typing import Tuple, Union
 
-version_tuple: Tuple[Union[int, str], ...] = (4, 4, 0, '.dev0')
+version_tuple: Tuple[Union[int, str], ...] = (4, 4, 0, ".dev0")
 
 
 def get_version_string() -> str:

test/test_on_demand_csfle.py

@@ -0,0 +1,67 @@
+# Copyright 2022-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test client side encryption with on demand credentials."""
+import os
+import sys
+import unittest
+
+sys.path[0:0] = [""]
+
+from test import IntegrationTest, client_context
+
+from bson.codec_options import CodecOptions
+from pymongo.encryption import _HAVE_PYMONGOCRYPT, ClientEncryption, EncryptionError
+
+
+class TestonDemandGCPCredentials(IntegrationTest):
+    @classmethod
+    @unittest.skipUnless(_HAVE_PYMONGOCRYPT, "pymongocrypt is not installed")
+    @client_context.require_version_min(4, 2, -1)
+    def setUpClass(cls):
+        super(TestonDemandGCPCredentials, cls).setUpClass()
+
+    def setUp(self):
+        super(TestonDemandGCPCredentials, self).setUp()
+        self.master_key = {
+            "projectId": "devprod-drivers",
+            "location": "global",
+            "keyRing": "key-ring-csfle",
+            "keyName": "key-name-csfle",
+        }
+
+    @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto")
+    def test_01_failure(self):
+        if os.environ["SUCCESS"].lower() == "true":
+            self.skipTest("Expecting success")
+        self.client_encryption = ClientEncryption(
+            kms_providers={"gcp": {}},
+            key_vault_namespace="keyvault.datakeys",
+            key_vault_client=client_context.client,
+            codec_options=CodecOptions(),
+        )
+        with self.assertRaises(EncryptionError):
+            self.client_encryption.create_data_key("gcp", self.master_key)
+
+    @unittest.skipIf(not os.getenv("TEST_FLE_GCP_AUTO"), "Not testing FLE GCP auto")
+    def test_02_success(self):
+        if os.environ["SUCCESS"].lower() == "false":
+            self.skipTest("Expecting failure")
+        self.client_encryption = ClientEncryption(
+            kms_providers={"gcp": {}},
+            key_vault_namespace="keyvault.datakeys",
+            key_vault_client=client_context.client,
+            codec_options=CodecOptions(),
+        )
+        self.client_encryption.create_data_key("gcp", self.master_key)