From e3dd6150b7c6525cf38b8809e3d94d649f314c7a Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Fri, 6 May 2022 11:03:22 +0200 Subject: [PATCH 1/6] feat(shell-api): ensure collection.drop() always drops FLE2 collections MONGOSH-1208 --- packages/shell-api/src/collection.ts | 30 +++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/shell-api/src/collection.ts b/packages/shell-api/src/collection.ts index 9853b32877..fd77498b65 100644 --- a/packages/shell-api/src/collection.ts +++ b/packages/shell-api/src/collection.ts @@ -1330,6 +1330,17 @@ export default class Collection extends ShellApiWithMongoClass { return this._getSingleStorageStatValue('totalSize'); } + /** + * Check if a collection is listed in the Mongo() encryptedFieldsMap. + * + * @return {Promise} returns Promise + */ + _isCollectionInEncryptedFieldsMap() { + // @ts-expect-error waiting for driver release + const encryptedFieldsMap = this._mongo._fleOptions?.encryptedFieldsMap; + return encryptedFieldsMap && encryptedFieldsMap[this._name]; + } + /** * Drop a collection. * @@ -1340,11 +1351,28 @@ export default class Collection extends ShellApiWithMongoClass { async drop(options: DropCollectionOptions = {}): Promise { this._emitCollectionApiCall('drop'); + let encryptedFieldsOptions = {}; + + if (!this._isCollectionInEncryptedFieldsMap() && !options.encryptedFields) { + const collectionInfos = await this._mongo._serviceProvider.listCollections( + this._database._name, + { + name: this._name + }, + await this._database._baseOptions() + ); + + if (collectionInfos) { + const encryptedFields: Document | undefined = collectionInfos[0].options.encryptedFields; + encryptedFieldsOptions = { encryptedFields }; + } + } + try { return await this._mongo._serviceProvider.dropCollection( this._database._name, this._name, - { ...await this._database._baseOptions(), ...options } + { ...await this._database._baseOptions(), ...options, ...encryptedFieldsOptions } ); } catch (error: any) { if (error?.codeName === 'NamespaceNotFound') { From 3f1c8bea65142594bf21a0f505ffe7ae8485deb9 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Mon, 9 May 2022 13:36:09 +0200 Subject: [PATCH 2/6] test: add e2e tests for fle2 --- packages/cli-repl/test/e2e-fle.spec.ts | 51 ++++++++++++++++- packages/shell-api/src/collection.spec.ts | 67 +++++++++++++++++++++++ packages/shell-api/src/collection.ts | 22 +++----- 3 files changed, 124 insertions(+), 16 deletions(-) diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index 2b1a53d860..297e13ec9d 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -8,6 +8,7 @@ import { once } from 'events'; import { serialize } from 'v8'; import { inspect } from 'util'; import path from 'path'; +import stripAnsi from 'strip-ansi'; describe('FLE tests', () => { const testServer = startTestServer('shared'); @@ -204,10 +205,9 @@ describe('FLE tests', () => { await shell.executeLine(`autoMongo = Mongo(${uri}, { \ keyVaultNamespace: '${dbname}.keyVault', \ kmsProviders: { local }, \ - schemaMap: schemaMap \ + schemaMap \ });`); - await shell.executeLine(`bypassMongo = Mongo(${uri}, { \ keyVaultNamespace: '${dbname}.keyVault', \ kmsProviders: { local }, \ @@ -237,6 +237,53 @@ describe('FLE tests', () => { expect(plainMongoResult).to.not.include("phoneNumber: '+12874627836445'"); }); + it('works when a encryptedFieldsMap option has been passed', async() => { + const shell = TestShell.start({ + args: ['--nodb'] + }); + const uri = JSON.stringify(await testServer.connectionString()); + + await shell.waitForPrompt(); + + await shell.executeLine('local = { key: BinData(0, "kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY") }'); + + await shell.executeLine(`keyMongo = Mongo(${uri}, { \ + keyVaultNamespace: '${dbname}.keyVault', \ + kmsProviders: { local } \ + });`); + + await shell.executeLine('keyVault = keyMongo.getKeyVault();'); + await shell.executeLine('keyId = keyVault.createKey("local");'); + + await shell.executeLine(`encryptedFieldsMap = { \ + '${dbname}.collfle2': { \ + fields: [{ path: 'phoneNumber', keyId, bsonType: 'string' }] \ + } \ + };`); + + await shell.executeLine(`autoMongo = Mongo(${uri}, { \ + keyVaultNamespace: '${dbname}.keyVault', \ + kmsProviders: { local }, \ + encryptedFieldsMap \ + });`); + + await shell.executeLine(`plainMongo = Mongo(${uri});`); + + await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.insertOne({ \ + phoneNumber: '+12874627836445' \ + });`); + + const autoMongoResult = await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.find()`); + expect(autoMongoResult).to.include("phoneNumber: '+12874627836445'"); + + const plainMongoResult = await shell.executeLine(`plainMongo.getDB('${dbname}').collfle2.find()`); + expect(plainMongoResult).to.include('phoneNumber: Binary(Buffer.from'); + expect(plainMongoResult).to.not.include("phoneNumber: '+12874627836445'"); + + const encryptedFieldsMap = await shell.executeLine('autoMongo._fleOptions.encryptedFieldsMap'); + expect(stripAnsi(encryptedFieldsMap).replace(/\s+/g, ' ')).to.include(`{ '${dbname}.collfle2': { fields: [ { path: 'phoneNumber'`); + }); + it('performs KeyVault data key management as expected', async() => { const shell = TestShell.start({ args: [await testServer.connectionString()] diff --git a/packages/shell-api/src/collection.spec.ts b/packages/shell-api/src/collection.spec.ts index c1041d00a8..dd61db8f6d 100644 --- a/packages/shell-api/src/collection.spec.ts +++ b/packages/shell-api/src/collection.spec.ts @@ -1243,6 +1243,7 @@ describe('Collection', () => { }); it('passes through options', async() => { + serviceProvider.listCollections.resolves([{}]); serviceProvider.dropCollection.resolves(); await collection.drop({ promoteValues: false }); expect(serviceProvider.dropCollection).to.have.been.calledWith( @@ -1873,6 +1874,72 @@ describe('Collection', () => { }); }); }); + describe('fle2', () => { + let mongo1: Mongo; + let mongo2: Mongo; + let serviceProvider: StubbedInstance; + let database: Database; + let bus: StubbedInstance; + let instanceState: ShellInstanceState; + let collection: Collection; + let keyId: any[] +; + beforeEach(() => { + bus = stubInterface(); + serviceProvider = stubInterface(); + serviceProvider.runCommand.resolves({ ok: 1 }); + serviceProvider.runCommandWithCheck.resolves({ ok: 1 }); + serviceProvider.initialDb = 'test'; + serviceProvider.bsonLibrary = bson; + instanceState = new ShellInstanceState(serviceProvider, bus); + keyId = [ { $binary: { base64: 'oh3caogGQ4Sf34ugKnZ7Xw==', subType: '04' } } ]; + mongo1 = new Mongo( + instanceState, + undefined, + { + keyVaultNamespace: 'db1.keyvault', + kmsProviders: { local: { key: 'A'.repeat(128) } }, + encryptedFieldsMap: { + 'db1.collfle2': { + fields: [{ path: 'phoneNumber', keyId, bsonType: 'string' }], + } + } + }, + undefined, + serviceProvider + ); + database = new Database(mongo1, 'db1'); + collection = new Collection(mongo1, database, 'collfle2'); + mongo2 = new Mongo( + instanceState, + undefined, + undefined, + undefined, + serviceProvider + ); + }); + + describe('drop', () => { + it('does not pass encryptedFields through options when collection is in encryptedFieldsMap', async() => { + serviceProvider.dropCollection.resolves(); + await collection.drop(); + expect(serviceProvider.dropCollection).to.have.been.calledWith( + 'db1', 'collfle2', {} + ); + }); + + it('passes encryptedFields through options when collection is not in encryptedFieldsMap', async() => { + serviceProvider.listCollections.resolves([{ + options: { encryptedFields: { fields: [ { path: 'phoneNumber', keyId, bsonType: 'string' } ] } } + }]); + serviceProvider.dropCollection.resolves(); + await mongo2.getDB('db1').getCollection('collfle2').drop(); + expect(serviceProvider.dropCollection).to.have.been.calledWith( + 'db1', 'collfle2', { encryptedFields: { fields: [ { path: 'phoneNumber', keyId, bsonType: 'string' } ] } } + ); + }); + }); + }); describe('with session', () => { let serviceProvider: StubbedInstance; let collection: Collection; diff --git a/packages/shell-api/src/collection.ts b/packages/shell-api/src/collection.ts index fd77498b65..2fe31cbeb5 100644 --- a/packages/shell-api/src/collection.ts +++ b/packages/shell-api/src/collection.ts @@ -1330,17 +1330,6 @@ export default class Collection extends ShellApiWithMongoClass { return this._getSingleStorageStatValue('totalSize'); } - /** - * Check if a collection is listed in the Mongo() encryptedFieldsMap. - * - * @return {Promise} returns Promise - */ - _isCollectionInEncryptedFieldsMap() { - // @ts-expect-error waiting for driver release - const encryptedFieldsMap = this._mongo._fleOptions?.encryptedFieldsMap; - return encryptedFieldsMap && encryptedFieldsMap[this._name]; - } - /** * Drop a collection. * @@ -1353,7 +1342,11 @@ export default class Collection extends ShellApiWithMongoClass { let encryptedFieldsOptions = {}; - if (!this._isCollectionInEncryptedFieldsMap() && !options.encryptedFields) { + // @ts-expect-error waiting for driver release + const encryptedFieldsMap = this._mongo._fleOptions?.encryptedFieldsMap; + const encryptedFields: Document | undefined = encryptedFieldsMap?.[`${this._database._name}.${ this._name}`]; + + if (!encryptedFields && !options.encryptedFields) { const collectionInfos = await this._mongo._serviceProvider.listCollections( this._database._name, { @@ -1362,8 +1355,9 @@ export default class Collection extends ShellApiWithMongoClass { await this._database._baseOptions() ); - if (collectionInfos) { - const encryptedFields: Document | undefined = collectionInfos[0].options.encryptedFields; + const encryptedFields: Document | undefined = collectionInfos?.[0]?.options?.encryptedFields; + + if (encryptedFields) { encryptedFieldsOptions = { encryptedFields }; } } From 28b7d75d116c0f979c8b70183300262c5c45dca1 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Mon, 9 May 2022 14:07:17 +0200 Subject: [PATCH 3/6] fix: ignore ts error for CommandOperationOptions --- packages/shell-api/src/collection.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shell-api/src/collection.ts b/packages/shell-api/src/collection.ts index 2fe31cbeb5..c31ced9097 100644 --- a/packages/shell-api/src/collection.ts +++ b/packages/shell-api/src/collection.ts @@ -1346,6 +1346,7 @@ export default class Collection extends ShellApiWithMongoClass { const encryptedFieldsMap = this._mongo._fleOptions?.encryptedFieldsMap; const encryptedFields: Document | undefined = encryptedFieldsMap?.[`${this._database._name}.${ this._name}`]; + // @ts-expect-error waiting for driver release if (!encryptedFields && !options.encryptedFields) { const collectionInfos = await this._mongo._serviceProvider.listCollections( this._database._name, From 3d388f74a13de5011f3f48572d9421a98e794408 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Mon, 9 May 2022 17:49:10 +0200 Subject: [PATCH 4/6] test: update e2e test to check dropping collection functionality --- packages/cli-repl/test/e2e-fle.spec.ts | 90 +++++++++++++++----------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index 297e13ec9d..d7dbc7f8c3 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -8,10 +8,9 @@ import { once } from 'events'; import { serialize } from 'v8'; import { inspect } from 'util'; import path from 'path'; -import stripAnsi from 'strip-ansi'; describe('FLE tests', () => { - const testServer = startTestServer('shared'); + const testServer = startTestServer('not-shared', '--replicaset', '--nodes', '1'); skipIfServerVersion(testServer, '< 4.2'); // FLE only available on 4.2+ skipIfCommunityServer(testServer); // FLE is enterprise-only useBinaryPath(testServer); // Get mongocryptd in the PATH for this test @@ -237,51 +236,70 @@ describe('FLE tests', () => { expect(plainMongoResult).to.not.include("phoneNumber: '+12874627836445'"); }); - it('works when a encryptedFieldsMap option has been passed', async() => { - const shell = TestShell.start({ - args: ['--nodb'] - }); - const uri = JSON.stringify(await testServer.connectionString()); + context('6.0+', () => { + skipIfServerVersion(testServer, '< 6.0'); // FLE2 only available on 6.0+ - await shell.waitForPrompt(); + it('drops fle2 collection with all helper collections when encryptedFields in collections listd', async() => { + const shell = TestShell.start({ + args: ['--nodb'], + env: { + ...process.env, + MONGOSH_FLE2_SUPPORT: 'true' + }, + }); + const uri = JSON.stringify(await testServer.connectionString()); - await shell.executeLine('local = { key: BinData(0, "kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY") }'); + await shell.waitForPrompt(); - await shell.executeLine(`keyMongo = Mongo(${uri}, { \ - keyVaultNamespace: '${dbname}.keyVault', \ - kmsProviders: { local } \ - });`); + await shell.executeLine('local = { key: BinData(0, "kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY") }'); - await shell.executeLine('keyVault = keyMongo.getKeyVault();'); - await shell.executeLine('keyId = keyVault.createKey("local");'); + await shell.executeLine(`keyMongo = Mongo(${uri}, { \ + keyVaultNamespace: '${dbname}.keyVault', \ + kmsProviders: { local } \ + });`); - await shell.executeLine(`encryptedFieldsMap = { \ - '${dbname}.collfle2': { \ - fields: [{ path: 'phoneNumber', keyId, bsonType: 'string' }] \ - } \ - };`); + await shell.executeLine('keyVault = keyMongo.getKeyVault();'); + await shell.executeLine('keyId = keyVault.createKey("local");'); - await shell.executeLine(`autoMongo = Mongo(${uri}, { \ - keyVaultNamespace: '${dbname}.keyVault', \ - kmsProviders: { local }, \ - encryptedFieldsMap \ - });`); + await shell.executeLine(`encryptedFieldsMap = { \ + '${dbname}.collfle2': { \ + fields: [{ path: 'phoneNumber', keyId, bsonType: 'string' }] \ + } \ + };`); - await shell.executeLine(`plainMongo = Mongo(${uri});`); + await shell.executeLine(`autoMongo = Mongo(${uri}, { \ + keyVaultNamespace: '${dbname}.keyVault', \ + kmsProviders: { local }, \ + encryptedFieldsMap \ + });`); - await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.insertOne({ \ - phoneNumber: '+12874627836445' \ - });`); + // Drivers will create the auxilliary FLE2 collections only when explicitly creating collections + // via the createCollection() command. + await shell.executeLine(`autoMongo.getDB('${dbname}').createCollection('collfle2');`); + await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.insertOne({ \ + phoneNumber: '+12874627836445' \ + });`); - const autoMongoResult = await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.find()`); - expect(autoMongoResult).to.include("phoneNumber: '+12874627836445'"); + const autoMongoResult = await shell.executeLine(`autoMongo.getDB('${dbname}').collfle2.find()`); + expect(autoMongoResult).to.include("phoneNumber: '+12874627836445'"); - const plainMongoResult = await shell.executeLine(`plainMongo.getDB('${dbname}').collfle2.find()`); - expect(plainMongoResult).to.include('phoneNumber: Binary(Buffer.from'); - expect(plainMongoResult).to.not.include("phoneNumber: '+12874627836445'"); + await shell.executeLine(`plainMongo = Mongo(${uri});`); + let collections = await shell.executeLine(`plainMongo.getDB('${dbname}').getCollectionNames()`); + + expect(collections).to.include('enxcol_.collfle2.ecc'); + expect(collections).to.include('enxcol_.collfle2.esc'); + expect(collections).to.include('enxcol_.collfle2.ecoc'); + expect(collections).to.include('collfle2'); - const encryptedFieldsMap = await shell.executeLine('autoMongo._fleOptions.encryptedFieldsMap'); - expect(stripAnsi(encryptedFieldsMap).replace(/\s+/g, ' ')).to.include(`{ '${dbname}.collfle2': { fields: [ { path: 'phoneNumber'`); + await shell.executeLine(`plainMongo.getDB('${dbname}').collfle2.drop();`); + + collections = await shell.executeLine(`plainMongo.getDB('${dbname}').getCollectionNames()`); + + expect(collections).to.not.include('enxcol_.collfle2.ecc'); + expect(collections).to.not.include('enxcol_.collfle2.esc'); + expect(collections).to.not.include('enxcol_.collfle2.ecoc'); + expect(collections).to.not.include('collfle2'); + }); }); it('performs KeyVault data key management as expected', async() => { From 397bc0d368d330cbdfb954134fca237c2841061d Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Mon, 9 May 2022 17:52:35 +0200 Subject: [PATCH 5/6] test: update name --- packages/cli-repl/test/e2e-fle.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index d7dbc7f8c3..d7929a8724 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -239,7 +239,7 @@ describe('FLE tests', () => { context('6.0+', () => { skipIfServerVersion(testServer, '< 6.0'); // FLE2 only available on 6.0+ - it('drops fle2 collection with all helper collections when encryptedFields in collections listd', async() => { + it('drops fle2 collection with all helper collections when encryptedFields options are in listCollections', async() => { const shell = TestShell.start({ args: ['--nodb'], env: { From de41d450746ac18203e402ece2a9612b8bc17d97 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Tue, 10 May 2022 14:11:29 +0200 Subject: [PATCH 6/6] test: check that fle2 data encrypted on plain mongo --- packages/cli-repl/test/e2e-fle.spec.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cli-repl/test/e2e-fle.spec.ts b/packages/cli-repl/test/e2e-fle.spec.ts index d7929a8724..bc0a425bdb 100644 --- a/packages/cli-repl/test/e2e-fle.spec.ts +++ b/packages/cli-repl/test/e2e-fle.spec.ts @@ -284,6 +284,11 @@ describe('FLE tests', () => { expect(autoMongoResult).to.include("phoneNumber: '+12874627836445'"); await shell.executeLine(`plainMongo = Mongo(${uri});`); + + const plainMongoResult = await shell.executeLine(`plainMongo.getDB('${dbname}').collfle2.find()`); + expect(plainMongoResult).to.include('phoneNumber: Binary(Buffer.from'); + expect(plainMongoResult).to.not.include("phoneNumber: '+12874627836445'"); + let collections = await shell.executeLine(`plainMongo.getDB('${dbname}').getCollectionNames()`); expect(collections).to.include('enxcol_.collfle2.ecc');