diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index df8f486d09..7538b5cbb2 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -1093,6 +1093,58 @@ tasks: params: file: src/results.json + - name: "test-gcpkms-task" + commands: + - func: "install dependencies" + # Upload node driver to a GCP instance + - command: subprocess.exec + type: setup + params: + binary: bash + add_expansions_to_env: true + env: + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + args: + - src/.evergreen/setup-gcp-testing.sh + # Run Mocha test over on GCE instance + - command: subprocess.exec + type: test + params: + working_dir: src + binary: bash + env: + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + GCPKMS_CMD: "env EXPECTED_GCPKMS_OUTCOME=success bash src/.evergreen/run-gcp-kms-tests.sh" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh + + + - name: "test-gcpkms-fail-task" + # test-gcpkms-fail-task runs in a non-GCE environment. + # It is expected to fail to obtain GCE credentials. + commands: + - func: "install dependencies" + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: server + AUTH: noauth + - command: subprocess.exec + type: test + params: + binary: bash + env: + EXPECTED_GCPKMS_OUTCOME: "failure" + args: + - src/.evergreen/run-gcp-kms-tests.sh + task_groups: - name: serverless_task_group setup_group_can_fail_task: true @@ -1101,7 +1153,7 @@ task_groups: - func: "fetch source" - command: shell.exec params: - shell: "bash" + shell: bash script: | ${PREPARE_SHELL} set +o xtrace @@ -1128,6 +1180,43 @@ task_groups: tasks: - ".serverless" + - name: test_gcpkms_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - command: subprocess.exec + params: + working_dir: "src" + binary: bash + add_expansions_to_env: true + env: + testgcpkms_key_file: ${gcpkms_key_file} + GCPKMS_SERVICEACCOUNT: ${gcpkms_service_account} + GCPKMS_DRIVERS_TOOLS: ${DRIVERS_TOOLS} + GCPKMS_MACHINETYPE: "e2-standard-4" + args: + - .evergreen/setup-gcp-instance.sh + - command: expansions.update + # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. + params: + file: src/testgcpkms-expansions.yml + + teardown_group: + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh + tasks: + - test-gcpkms-task + pre: - func: "fetch source" - func: "windows fix" diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f1be682b28..1ce1ba4245 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1031,6 +1031,51 @@ tasks: - command: perf.send params: file: src/results.json + - name: test-gcpkms-task + commands: + - func: install dependencies + - command: subprocess.exec + type: setup + params: + binary: bash + add_expansions_to_env: true + env: + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + args: + - src/.evergreen/setup-gcp-testing.sh + - command: subprocess.exec + type: test + params: + working_dir: src + binary: bash + env: + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + GCPKMS_CMD: env EXPECTED_GCPKMS_OUTCOME=success bash src/.evergreen/run-gcp-kms-tests.sh + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh + - name: test-gcpkms-fail-task + commands: + - func: install dependencies + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: server + AUTH: noauth + - command: subprocess.exec + type: test + params: + binary: bash + env: + EXPECTED_GCPKMS_OUTCOME: failure + args: + - src/.evergreen/run-gcp-kms-tests.sh - name: test-latest-server tags: - latest @@ -3002,6 +3047,40 @@ task_groups: bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh tasks: - .serverless + - name: test_gcpkms_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + setup_group: + - func: fetch source + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + env: + testgcpkms_key_file: ${gcpkms_key_file} + GCPKMS_SERVICEACCOUNT: ${gcpkms_service_account} + GCPKMS_DRIVERS_TOOLS: ${DRIVERS_TOOLS} + GCPKMS_MACHINETYPE: e2-standard-4 + args: + - .evergreen/setup-gcp-instance.sh + - command: expansions.update + params: + file: src/testgcpkms-expansions.yml + teardown_group: + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + GCPKMS_GCLOUD: ${GCPKMS_GCLOUD} + GCPKMS_PROJECT: ${GCPKMS_PROJECT} + GCPKMS_ZONE: ${GCPKMS_ZONE} + GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME} + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh + tasks: + - test-gcpkms-task pre: - func: fetch source - func: windows fix @@ -3461,6 +3540,12 @@ buildvariants: NODE_LTS_NAME: fermium tasks: - serverless_task_group + - name: rhel8-test-gcp-kms + display_name: GCP KMS Test + run_on: debian11-small + tasks: + - test_gcpkms_task_group + - test-gcpkms-fail-task - name: rhel8-no-auth-tests display_name: No Auth Tests run_on: rhel80-large diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 4ab3e7e919..0fce74c632 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -663,6 +663,13 @@ BUILD_VARIANTS.push({ tasks: ['serverless_task_group'] }); +BUILD_VARIANTS.push({ + name: 'rhel8-test-gcp-kms', + display_name: 'GCP KMS Test', + run_on: 'debian11-small', + tasks: ['test_gcpkms_task_group', 'test-gcpkms-fail-task'] +}); + BUILD_VARIANTS.push({ name: 'rhel8-no-auth-tests', display_name: 'No Auth Tests', diff --git a/.evergreen/run-gcp-kms-tests.sh b/.evergreen/run-gcp-kms-tests.sh new file mode 100644 index 0000000000..bfbaea7712 --- /dev/null +++ b/.evergreen/run-gcp-kms-tests.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash + +set -o errexit + +pushd "src" +PROJECT_DIRECTORY="$(pwd)" +export PROJECT_DIRECTORY +source ".evergreen/init-nvm.sh" + +set -o xtrace + +npm install 'mongodb-client-encryption@2.6.0-alpha.0' +npm install 'gcp-metadata' + +export MONGODB_URI="mongodb://localhost:27017" + +export EXPECTED_GCPKMS_OUTCOME=${EXPECTED_GCPKMS_OUTCOME:-omitted} +export TEST_CSFLE=true +export CSFLE_KMS_PROVIDERS='not json' + +npx mocha --config test/mocha_mongodb.json test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts diff --git a/.evergreen/setup-gcp-instance.sh b/.evergreen/setup-gcp-instance.sh new file mode 100644 index 0000000000..9f55332993 --- /dev/null +++ b/.evergreen/setup-gcp-instance.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +set -o errexit +if [ -z ${testgcpkms_key_file+omitted} ]; then echo "testgcpkms_key_file is unset" && exit 1; fi + +echo "${testgcpkms_key_file}" > ./testgcpkms_key_file.json +export GCPKMS_KEYFILE=./testgcpkms_key_file.json + +"$GCPKMS_DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh" diff --git a/.evergreen/setup-gcp-testing.sh b/.evergreen/setup-gcp-testing.sh new file mode 100644 index 0000000000..c93e62e6b0 --- /dev/null +++ b/.evergreen/setup-gcp-testing.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env bash + +# Assert required environment variables are present without printing them +if [ -z ${GCPKMS_GCLOUD+omitted} ]; then echo "GCPKMS_GCLOUD is unset" && exit 1; fi +if [ -z ${GCPKMS_PROJECT+omitted} ]; then echo "GCPKMS_PROJECT is unset" && exit 1; fi +if [ -z ${GCPKMS_ZONE+omitted} ]; then echo "GCPKMS_ZONE is unset" && exit 1; fi +if [ -z ${GCPKMS_INSTANCENAME+omitted} ]; then echo "GCPKMS_INSTANCENAME is unset" && exit 1; fi + +set -o errexit + +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" + +export GCPKMS_SRC=node-driver-source.tgz +export GCPKMS_DST=$GCPKMS_INSTANCENAME: + +# Box up the entire driver and it's node_modules +echo "compressing node driver source ... begin" +tar -czf $GCPKMS_SRC src +echo "compressing node driver source ... end" + +echo "copying node driver tar ... begin" +"${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/copy-file.sh" +echo "copying node driver tar ... end" + +echo "decompressing node driver tar on gcp ... begin" +export GCPKMS_CMD="tar -xzf $GCPKMS_SRC" +"${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh" +echo "decompressing node driver tar on gcp ... end" diff --git a/src/deps.ts b/src/deps.ts index bb675baa52..e3e931e5ef 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -223,17 +223,19 @@ export interface AutoEncryptionOptions { /** Configuration options that are used by specific KMS providers during key generation, encryption, and decryption. */ kmsProviders?: { /** Configuration options for using 'aws' as your KMS provider */ - aws?: { - /** The access key used for the AWS KMS provider */ - accessKeyId: string; - /** The secret access key used for the AWS KMS provider */ - secretAccessKey: string; - /** - * An optional AWS session token that will be used as the - * X-Amz-Security-Token header for AWS requests. - */ - sessionToken?: string; - }; + aws?: + | { + /** The access key used for the AWS KMS provider */ + accessKeyId: string; + /** The secret access key used for the AWS KMS provider */ + secretAccessKey: string; + /** + * An optional AWS session token that will be used as the + * X-Amz-Security-Token header for AWS requests. + */ + sessionToken?: string; + } + | Record; /** Configuration options for using 'local' as your KMS provider */ local?: { /** @@ -243,33 +245,48 @@ export interface AutoEncryptionOptions { key: Buffer | string; }; /** Configuration options for using 'azure' as your KMS provider */ - azure?: { - /** The tenant ID identifies the organization for the account */ - tenantId: string; - /** The client ID to authenticate a registered application */ - clientId: string; - /** The client secret to authenticate a registered application */ - clientSecret: string; - /** - * If present, a host with optional port. E.g. "example.com" or "example.com:443". - * This is optional, and only needed if customer is using a non-commercial Azure instance - * (e.g. a government or China account, which use different URLs). - * Defaults to "login.microsoftonline.com" - */ - identityPlatformEndpoint?: string | undefined; - }; + azure?: + | { + /** The tenant ID identifies the organization for the account */ + tenantId: string; + /** The client ID to authenticate a registered application */ + clientId: string; + /** The client secret to authenticate a registered application */ + clientSecret: string; + /** + * If present, a host with optional port. E.g. "example.com" or "example.com:443". + * This is optional, and only needed if customer is using a non-commercial Azure instance + * (e.g. a government or China account, which use different URLs). + * Defaults to "login.microsoftonline.com" + */ + identityPlatformEndpoint?: string | undefined; + } + | { + /** + * If present, an access token to authenticate with Azure. + */ + accessToken: string; + }; /** Configuration options for using 'gcp' as your KMS provider */ - gcp?: { - /** The service account email to authenticate */ - email: string; - /** A PKCS#8 encrypted key. This can either be a base64 string or a binary representation */ - privateKey: string | Buffer; - /** - * If present, a host with optional port. E.g. "example.com" or "example.com:443". - * Defaults to "oauth2.googleapis.com" - */ - endpoint?: string | undefined; - }; + gcp?: + | { + /** The service account email to authenticate */ + email: string; + /** A PKCS#8 encrypted key. This can either be a base64 string or a binary representation */ + privateKey: string | Buffer; + /** + * If present, a host with optional port. E.g. "example.com" or "example.com:443". + * Defaults to "oauth2.googleapis.com" + */ + endpoint?: string | undefined; + } + | { + /** + * If present, an access token to authenticate with GCP. + */ + accessToken: string; + } + | Record; /** * Configuration options for using 'kmip' as your KMS provider */ diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts new file mode 100644 index 0000000000..b3b3796e0f --- /dev/null +++ b/test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts @@ -0,0 +1,69 @@ +import { expect } from 'chai'; +import { env } from 'process'; + +import { Binary } from '../../mongodb'; + +const metadata: MongoDBMetadataUI = { + requires: { + clientSideEncryption: true + } +} as const; + +const dataKeyOptions = { + masterKey: { + projectId: 'devprod-drivers', + location: 'global', + keyRing: 'key-ring-csfle', + keyName: 'key-name-csfle' + } +}; + +describe('17. On-demand GCP Credentials', () => { + let clientEncryption: import('mongodb-client-encryption').ClientEncryption; + let keyVaultClient; + + beforeEach(async function () { + keyVaultClient = this.configuration.newClient(); + + const { ClientEncryption } = this.configuration.mongodbClientEncryption; + + if (typeof env.GCPKMS_GCLOUD === 'string') { + // If Google cloud env is present then EXPECTED_GCPKMS_OUTCOME MUST be set + expect(env.EXPECTED_GCPKMS_OUTCOME, `EXPECTED_GCPKMS_OUTCOME must be 'success' or 'failure'`) + .to.be.a('string') + .that.satisfies(s => s === 'success' || s === 'failure'); + } + + clientEncryption = new ClientEncryption(keyVaultClient, { + keyVaultClient, + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { gcp: {} } + }); + }); + + afterEach(async () => { + await keyVaultClient?.close(); + }); + + it('Case 1: Failure', metadata, async function () { + if (env.EXPECTED_GCPKMS_OUTCOME !== 'failure') { + this.skipReason = 'This test is supposed to run in the environment where failure is expected'; + this.skip(); + } + + const error = await clientEncryption.createDataKey('gcp', dataKeyOptions).catch(error => error); + // GaxiosError: Unsuccessful response status code. Request failed with status code 404 + expect(error).to.be.instanceOf(Error); + expect(error).property('code', '404'); + }); + + it('Case 2: Success', metadata, async function () { + if (env.EXPECTED_GCPKMS_OUTCOME !== 'success') { + this.skipReason = 'This test is supposed to run in the environment where success is expected'; + this.skip(); + } + + const dk = await clientEncryption.createDataKey('gcp', dataKeyOptions); + expect(dk).to.be.instanceOf(Binary); + }); +}); diff --git a/test/tools/runner/filters/client_encryption_filter.js b/test/tools/runner/filters/client_encryption_filter.js index 2ab66c75cb..563620d5d4 100644 --- a/test/tools/runner/filters/client_encryption_filter.js +++ b/test/tools/runner/filters/client_encryption_filter.js @@ -1,6 +1,7 @@ 'use strict'; const mongodb = require('../../../mongodb'); +const process = require('process'); /** * Filter for whether or not a test needs / doesn't need Client Side Encryption @@ -19,8 +20,10 @@ class ClientSideEncryptionFilter { let mongodbClientEncryption; try { mongodbClientEncryption = require('mongodb-client-encryption').extension(mongodb); - } catch (e) { - // Do Nothing + } catch (failedToGetFLELib) { + if (process.env.TEST_CSFLE) { + console.error({ failedToGetFLELib }); + } } this.enabled = !!(CSFLE_KMS_PROVIDERS && mongodbClientEncryption); diff --git a/test/types/encryption.test-d.ts b/test/types/encryption.test-d.ts new file mode 100644 index 0000000000..390b9307f2 --- /dev/null +++ b/test/types/encryption.test-d.ts @@ -0,0 +1,18 @@ +import { expectAssignable, expectNotAssignable } from 'tsd'; + +import type { AutoEncryptionOptions } from '../mongodb'; + +// Empty credentials support on each provider +expectAssignable({ + kmsProviders: { + gcp: {}, + aws: {} + } +}); + +// TODO(NODE-4537): Azure support +expectNotAssignable({ + kmsProviders: { + azure: {} + } +});