From eb2512976108a6385a4dc5c54b2728560d19d3d0 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 16 Mar 2022 17:42:50 +0100 Subject: [PATCH] feat(shell-api): add KMIP support MONGOSH-1013 Add support for passing through `tlsOptions` for FLE specifically. This was previously not possible, and is necessary for KMIP support, since KMIP uses TLS with a client key/certificate for authentication. --- packages/cli-repl/test/e2e-tls.spec.ts | 2 +- .../src/all-fle-types.ts | 1 + .../src/field-level-encryption.spec.ts | 62 ++++++++++++++++-- .../shell-api/src/field-level-encryption.ts | 2 + packages/shell-api/src/helpers.ts | 5 +- .../certificates/README.md | 0 .../certificates/ca-server.crl | 0 .../fixtures => testing}/certificates/ca.cnf | 0 .../fixtures => testing}/certificates/ca.crt | 0 .../fixtures => testing}/certificates/ca.db | 0 .../certificates/ca.db.attr | 0 .../fixtures => testing}/certificates/ca.key | 0 .../certificates/ca.serial | 0 .../certificates/client.bundle.encrypted.pem | 0 .../certificates/client.bundle.pem | 0 .../certificates/client.bundle.pfx | Bin .../certificates/client.encrypted.key | 0 .../certificates/client.key | 0 .../certificates/invalid-client.bundle.pem | 0 .../certificates/invalid-client.key | 0 .../certificates/non-ca.crt | 0 .../certificates/non-ca.key | 0 .../server-invalidhost.bundle.pem | 0 .../certificates/server-invalidhost.key | 0 .../certificates/server.bundle.pem | 0 .../certificates/server.key | 0 26 files changed, 64 insertions(+), 8 deletions(-) rename {packages/cli-repl/test/fixtures => testing}/certificates/README.md (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca-server.crl (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.cnf (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.crt (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.db (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.db.attr (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/ca.serial (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/client.bundle.encrypted.pem (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/client.bundle.pem (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/client.bundle.pfx (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/client.encrypted.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/client.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/invalid-client.bundle.pem (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/invalid-client.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/non-ca.crt (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/non-ca.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/server-invalidhost.bundle.pem (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/server-invalidhost.key (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/server.bundle.pem (100%) rename {packages/cli-repl/test/fixtures => testing}/certificates/server.key (100%) diff --git a/packages/cli-repl/test/e2e-tls.spec.ts b/packages/cli-repl/test/e2e-tls.spec.ts index e6d95f0cca..8ef456f6d1 100644 --- a/packages/cli-repl/test/e2e-tls.spec.ts +++ b/packages/cli-repl/test/e2e-tls.spec.ts @@ -8,7 +8,7 @@ import { promisify } from 'util'; import rimraf from 'rimraf'; function getCertPath(filename: string): string { - return path.join(__dirname, 'fixtures', 'certificates', filename); + return path.join(__dirname, '..', '..', '..', 'testing', 'certificates', filename); } const CA_CERT = getCertPath('ca.crt'); const NON_CA_CERT = getCertPath('non-ca.crt'); diff --git a/packages/service-provider-core/src/all-fle-types.ts b/packages/service-provider-core/src/all-fle-types.ts index 2f3df46159..0a744fd5ad 100644 --- a/packages/service-provider-core/src/all-fle-types.ts +++ b/packages/service-provider-core/src/all-fle-types.ts @@ -10,6 +10,7 @@ export type { ClientEncryptionEncryptCallback, ClientEncryptionEncryptOptions, ClientEncryptionOptions, + ClientEncryptionTlsOptions, KMSProviders } from 'mongodb-client-encryption'; diff --git a/packages/shell-api/src/field-level-encryption.spec.ts b/packages/shell-api/src/field-level-encryption.spec.ts index 25210f9e76..d2312bb779 100644 --- a/packages/shell-api/src/field-level-encryption.spec.ts +++ b/packages/shell-api/src/field-level-encryption.spec.ts @@ -1,7 +1,10 @@ import { MongoshInvalidInputError } from '@mongosh/errors'; -import { bson, ClientEncryption as FLEClientEncryption, ServiceProvider } from '@mongosh/service-provider-core'; +import { bson, ClientEncryption as FLEClientEncryption, ClientEncryptionTlsOptions, ServiceProvider } from '@mongosh/service-provider-core'; import { expect } from 'chai'; import { EventEmitter } from 'events'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { Duplex } from 'stream'; import sinon, { StubbedInstance, stubInterface } from 'ts-sinon'; import Database from './database'; import { signatures, toShellResult } from './decorators'; @@ -47,6 +50,10 @@ const ALGO = 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'; const RAW_CLIENT = { client: 1 } as any; +function getCertPath(filename: string): string { + return path.join(__dirname, '..', '..', '..', 'testing', 'certificates', filename); +} + describe('Field Level Encryption', () => { let sp: StubbedInstance; let mongo: Mongo; @@ -482,6 +489,17 @@ describe('Field Level Encryption', () => { connections = []; sinon.replace(require('tls'), 'connect', sinon.fake((options, onConnect) => { + if (options.host === 'kmip.example.com') { + // KMIP is not http(s)-based, we don't implement strong fakes for it + // and instead only verify that a connection has occurred. + connections.push({ options }); + process.nextTick(onConnect); + const conn = new Duplex({ + read() { setImmediate(() => this.destroy(new Error('mock connection broken'))); }, + write(chunk, enc, cb) { cb(); } + }); + return conn; + } if (!fakeAWSHandlers.some(handler => handler.host.test(options.host))) { throw new Error(`Unexpected TLS connection to ${options.host}`); } @@ -498,7 +516,7 @@ describe('Field Level Encryption', () => { sinon.restore(); }); - const kms: [keyof KMSProvider, KMSProvider[keyof KMSProvider]][] = [ + const kms: [keyof KMSProvider, KMSProvider[keyof KMSProvider] & { tlsOptions?: ClientEncryptionTlsOptions }][] = [ ['local', { key: new bson.Binary(Buffer.from('kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY', 'base64'), 0) }], @@ -528,15 +546,25 @@ lt6waE7I2uSPqIC20LcCIQDJQYIHQII+3YaPqyhGgqMexuuuGx+lDKD6/Fu/JwPb 5QIhAKthiYcYKlL9h8bjDsQhZDUACPasjzdsDEdq8inDyLOFAiEAmCr/tZwA3qeA ZoBzI10DGPIuoKXBd3nk/eBxPkaxlEECIQCNymjsoI7GldtujVnr1qT+3yedLfHK srDVjIT3LsvTqw==` + }], + ['kmip', { + endpoint: 'kmip.example.com:123', + tlsOptions: { + tlsCertificateKeyFile: getCertPath('client.bundle.encrypted.pem'), + tlsCertificateKeyFilePassword: 'p4ssw0rd', + tlsCAFile: getCertPath('ca.crt') + } }] ]; - for (const [ kmsName, kmsOptions ] of kms) { + for (const [ kmsName, kmsAndTlsOptions ] of kms) { // eslint-disable-next-line no-loop-func it(`provides ClientEncryption for kms=${kmsName}`, async() => { + const kmsOptions = { ...kmsAndTlsOptions, tlsOptions: undefined }; const mongo = new Mongo(instanceState, uri, { keyVaultNamespace: `${dbname}.__keyVault`, kmsProviders: { [kmsName]: kmsOptions } as any, - explicitEncryptionOnly: true + explicitEncryptionOnly: true, + tlsOptions: { [kmsName]: kmsAndTlsOptions.tlsOptions ?? undefined } }, serviceProvider); await mongo.connect(); instanceState.mongos.push(mongo); @@ -567,6 +595,28 @@ srDVjIT3LsvTqw==` keyName: 'foobar' }); break; + case 'kmip': + try { + await keyVault.createKey('kmip', undefined); + } catch (err) { + // See above, we don't attempt to successfully encrypt/decrypt + // when using KMIP + expect(err.message).to.include('KMS request failed'); + expect(connections).to.deep.equal([{ + options: { + host: 'kmip.example.com', + servername: 'kmip.example.com', + port: 123, + passphrase: 'p4ssw0rd', + ca: await fs.readFile(getCertPath('ca.crt')), + cert: await fs.readFile(getCertPath('client.bundle.encrypted.pem')), + key: await fs.readFile(getCertPath('client.bundle.encrypted.pem')) + } + }]); + return; + } + expect.fail('missed exception'); + break; default: throw new Error(`unreachable ${kmsName}`); } @@ -584,12 +634,12 @@ srDVjIT3LsvTqw==` expect(encrypted.sub_type).to.equal(6); // Encrypted expect(decrypted).to.deep.equal(plaintextValue); - if ((kmsOptions as any).sessionToken) { // as any -> NODE-3107 + if ('sessionToken' in kmsOptions) { expect( connections.map( conn => conn.requests.map( req => req.headers['x-amz-security-token'])).flat()) - .to.include((kmsOptions as any).sessionToken); + .to.include(kmsOptions.sessionToken); } }); } diff --git a/packages/shell-api/src/field-level-encryption.ts b/packages/shell-api/src/field-level-encryption.ts index ce2601d2ca..df20865150 100644 --- a/packages/shell-api/src/field-level-encryption.ts +++ b/packages/shell-api/src/field-level-encryption.ts @@ -12,6 +12,7 @@ import { ClientEncryptionDataKeyProvider, ClientEncryptionOptions, ClientEncryptionEncryptOptions, + ClientEncryptionTlsOptions, KMSProviders, ReplPlatform, AWSEncryptionKeyOptions, @@ -41,6 +42,7 @@ export interface ClientSideFieldLevelEncryptionOptions { schemaMap?: Document, bypassAutoEncryption?: boolean; explicitEncryptionOnly?: boolean; + tlsOptions?: { [k in keyof ClientSideFieldLevelEncryptionKmsProvider]?: ClientEncryptionTlsOptions }; } @shellApiClassDefault diff --git a/packages/shell-api/src/helpers.ts b/packages/shell-api/src/helpers.ts index 9ae4808cc5..9358eb7cd1 100644 --- a/packages/shell-api/src/helpers.ts +++ b/packages/shell-api/src/helpers.ts @@ -659,7 +659,7 @@ export function assertCLI(platform: ReplPlatform, features: string): void { export function processFLEOptions(fleOptions: ClientSideFieldLevelEncryptionOptions): AutoEncryptionOptions { assertKeysDefined(fleOptions, ['keyVaultNamespace', 'kmsProviders']); Object.keys(fleOptions).forEach(k => { - if (['keyVaultClient', 'keyVaultNamespace', 'kmsProviders', 'schemaMap', 'bypassAutoEncryption'].indexOf(k) === -1) { + if (['keyVaultClient', 'keyVaultNamespace', 'kmsProviders', 'schemaMap', 'bypassAutoEncryption', 'tlsOptions'].indexOf(k) === -1) { throw new MongoshInvalidInputError(`Unrecognized FLE Client Option ${k}`); } }); @@ -691,6 +691,9 @@ export function processFLEOptions(fleOptions: ClientSideFieldLevelEncryptionOpti if (fleOptions.bypassAutoEncryption !== undefined) { autoEncryption.bypassAutoEncryption = fleOptions.bypassAutoEncryption; } + if (fleOptions.tlsOptions !== undefined) { + autoEncryption.tlsOptions = fleOptions.tlsOptions; + } return autoEncryption; } diff --git a/packages/cli-repl/test/fixtures/certificates/README.md b/testing/certificates/README.md similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/README.md rename to testing/certificates/README.md diff --git a/packages/cli-repl/test/fixtures/certificates/ca-server.crl b/testing/certificates/ca-server.crl similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca-server.crl rename to testing/certificates/ca-server.crl diff --git a/packages/cli-repl/test/fixtures/certificates/ca.cnf b/testing/certificates/ca.cnf similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.cnf rename to testing/certificates/ca.cnf diff --git a/packages/cli-repl/test/fixtures/certificates/ca.crt b/testing/certificates/ca.crt similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.crt rename to testing/certificates/ca.crt diff --git a/packages/cli-repl/test/fixtures/certificates/ca.db b/testing/certificates/ca.db similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.db rename to testing/certificates/ca.db diff --git a/packages/cli-repl/test/fixtures/certificates/ca.db.attr b/testing/certificates/ca.db.attr similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.db.attr rename to testing/certificates/ca.db.attr diff --git a/packages/cli-repl/test/fixtures/certificates/ca.key b/testing/certificates/ca.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.key rename to testing/certificates/ca.key diff --git a/packages/cli-repl/test/fixtures/certificates/ca.serial b/testing/certificates/ca.serial similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/ca.serial rename to testing/certificates/ca.serial diff --git a/packages/cli-repl/test/fixtures/certificates/client.bundle.encrypted.pem b/testing/certificates/client.bundle.encrypted.pem similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/client.bundle.encrypted.pem rename to testing/certificates/client.bundle.encrypted.pem diff --git a/packages/cli-repl/test/fixtures/certificates/client.bundle.pem b/testing/certificates/client.bundle.pem similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/client.bundle.pem rename to testing/certificates/client.bundle.pem diff --git a/packages/cli-repl/test/fixtures/certificates/client.bundle.pfx b/testing/certificates/client.bundle.pfx similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/client.bundle.pfx rename to testing/certificates/client.bundle.pfx diff --git a/packages/cli-repl/test/fixtures/certificates/client.encrypted.key b/testing/certificates/client.encrypted.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/client.encrypted.key rename to testing/certificates/client.encrypted.key diff --git a/packages/cli-repl/test/fixtures/certificates/client.key b/testing/certificates/client.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/client.key rename to testing/certificates/client.key diff --git a/packages/cli-repl/test/fixtures/certificates/invalid-client.bundle.pem b/testing/certificates/invalid-client.bundle.pem similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/invalid-client.bundle.pem rename to testing/certificates/invalid-client.bundle.pem diff --git a/packages/cli-repl/test/fixtures/certificates/invalid-client.key b/testing/certificates/invalid-client.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/invalid-client.key rename to testing/certificates/invalid-client.key diff --git a/packages/cli-repl/test/fixtures/certificates/non-ca.crt b/testing/certificates/non-ca.crt similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/non-ca.crt rename to testing/certificates/non-ca.crt diff --git a/packages/cli-repl/test/fixtures/certificates/non-ca.key b/testing/certificates/non-ca.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/non-ca.key rename to testing/certificates/non-ca.key diff --git a/packages/cli-repl/test/fixtures/certificates/server-invalidhost.bundle.pem b/testing/certificates/server-invalidhost.bundle.pem similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/server-invalidhost.bundle.pem rename to testing/certificates/server-invalidhost.bundle.pem diff --git a/packages/cli-repl/test/fixtures/certificates/server-invalidhost.key b/testing/certificates/server-invalidhost.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/server-invalidhost.key rename to testing/certificates/server-invalidhost.key diff --git a/packages/cli-repl/test/fixtures/certificates/server.bundle.pem b/testing/certificates/server.bundle.pem similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/server.bundle.pem rename to testing/certificates/server.bundle.pem diff --git a/packages/cli-repl/test/fixtures/certificates/server.key b/testing/certificates/server.key similarity index 100% rename from packages/cli-repl/test/fixtures/certificates/server.key rename to testing/certificates/server.key