From a0a1344ca6a96c07b8ead99eb73d3e3412c6a198 Mon Sep 17 00:00:00 2001 From: Aleh Zasypkin Date: Mon, 20 Jul 2020 12:30:50 +0200 Subject: [PATCH] Add support for the encryption key rotation to the encrypted saved objects plugin. --- .../encrypted_saved_objects/server/config.ts | 38 +++- .../crypto/encrypted_saved_objects_service.ts | 108 ++++++++-- .../crypto/encryption_key_rotation_service.ts | 184 ++++++++++++++++++ .../server/crypto/index.ts | 1 + .../encrypted_saved_objects/server/plugin.ts | 40 +++- .../server/routes/index.ts | 27 +++ .../server/routes/key_rotation.ts | 41 ++++ 7 files changed, 400 insertions(+), 39 deletions(-) create mode 100644 x-pack/plugins/encrypted_saved_objects/server/crypto/encryption_key_rotation_service.ts create mode 100644 x-pack/plugins/encrypted_saved_objects/server/routes/index.ts create mode 100644 x-pack/plugins/encrypted_saved_objects/server/routes/key_rotation.ts diff --git a/x-pack/plugins/encrypted_saved_objects/server/config.ts b/x-pack/plugins/encrypted_saved_objects/server/config.ts index 9c751a9c67f523f..0aee76e7aa305be 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/config.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/config.ts @@ -7,17 +7,34 @@ import crypto from 'crypto'; import { map } from 'rxjs/operators'; import { schema, TypeOf } from '@kbn/config-schema'; +import { UnwrapObservable } from '@kbn/utility-types'; import { PluginInitializerContext } from 'src/core/server'; -export const ConfigSchema = schema.object({ - enabled: schema.boolean({ defaultValue: true }), - encryptionKey: schema.conditional( - schema.contextRef('dist'), - true, - schema.maybe(schema.string({ minLength: 32 })), - schema.string({ minLength: 32, defaultValue: 'a'.repeat(32) }) - ), -}); +export type ConfigType = UnwrapObservable>; + +export const ConfigSchema = schema.object( + { + enabled: schema.boolean({ defaultValue: true }), + encryptionKey: schema.conditional( + schema.contextRef('dist'), + true, + schema.maybe(schema.string({ minLength: 32 })), + schema.string({ minLength: 32, defaultValue: 'a'.repeat(32) }) + ), + keyRotation: schema.object({ + decryptionOnlyKeys: schema.arrayOf(schema.string({ minLength: 32 }), { defaultValue: [] }), + batchSize: schema.number({ min: 1, defaultValue: 100 }), + }), + }, + { + validate(value) { + const decryptionOnlyKeys = value.keyRotation?.decryptionOnlyKeys ?? []; + if (value.encryptionKey && decryptionOnlyKeys.includes(value.encryptionKey)) { + return '`keyRotation.decryptionOnlyKeys` cannot contain primary encryption key specified in `encryptionKey`.'; + } + }, + } +); export function createConfig$(context: PluginInitializerContext) { return context.config.create>().pipe( @@ -37,7 +54,8 @@ export function createConfig$(context: PluginInitializerContext) { } return { - config: { ...config, encryptionKey }, + ...config, + encryptionKey, usingEphemeralEncryptionKey, }; }) diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts index 99361107047c27f..474b029e09cd772 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts @@ -52,6 +52,16 @@ interface CommonParameters { user?: AuthenticatedUser; } +/** + * Describes parameters for the decrypt methods. + */ +interface DecryptParameters extends CommonParameters { + /** + * Indicates whether decryption should only be performed using secondary decryption-only keys. + */ + useDecryptionOnlyKeys?: boolean; +} + /** * Utility function that gives array representation of the saved object descriptor respecting * optional `namespace` property. @@ -80,15 +90,21 @@ export class EncryptedSavedObjectsService { > = new Map(); /** - * @param crypto nodeCrypto instance. + * @param cryptos nodeCrypto instances used for encryption and/or decryption. The first crypto + * in the list is considered as primary and it's the only one that is used for both encryption and + * decryption, the rest are decryption-only cryptos. * @param logger Ordinary logger instance. * @param audit Audit logger instance. */ constructor( - private readonly crypto: Readonly, + private readonly cryptos: Readonly, private readonly logger: Logger, private readonly audit: EncryptedSavedObjectsAuditLogger - ) {} + ) { + if (cryptos.length === 0 || !cryptos[0]) { + throw new Error('The list of cryptos should include at least one item.'); + } + } /** * Registers saved object type as the one that contains attributes that should be encrypted. @@ -136,7 +152,7 @@ export class EncryptedSavedObjectsService { descriptor: SavedObjectDescriptor, attributes: T, originalAttributes?: T, - params?: CommonParameters + params?: DecryptParameters ) { const typeDefinition = this.typeDefinitions.get(descriptor.type); if (typeDefinition === undefined) { @@ -174,7 +190,7 @@ export class EncryptedSavedObjectsService { Object.fromEntries( Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) ) as T, - { user: params?.user } + params ); } catch (err) { decryptionError = err; @@ -261,13 +277,14 @@ export class EncryptedSavedObjectsService { attributes: T, params?: CommonParameters ): Promise { + const encrypter = this.getEncrypter(); const iterator = this.attributesToEncryptIterator(descriptor, attributes, params); let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; try { - iteratorResult = iterator.next(await this.crypto.encrypt(attributeValue, encryptionAAD)); + iteratorResult = iterator.next(await encrypter.encrypt(attributeValue, encryptionAAD)); } catch (err) { iterator.throw!(err); } @@ -290,13 +307,14 @@ export class EncryptedSavedObjectsService { attributes: T, params?: CommonParameters ): T { + const encrypter = this.getEncrypter(); const iterator = this.attributesToEncryptIterator(descriptor, attributes, params); let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; try { - iteratorResult = iterator.next(this.crypto.encryptSync(attributeValue, encryptionAAD)); + iteratorResult = iterator.next(encrypter.encryptSync(attributeValue, encryptionAAD)); } catch (err) { iterator.throw!(err); } @@ -318,19 +336,31 @@ export class EncryptedSavedObjectsService { public async decryptAttributes>( descriptor: SavedObjectDescriptor, attributes: T, - params?: CommonParameters + params?: DecryptParameters ): Promise { + const decrypters = this.getDecrypters(params?.useDecryptionOnlyKeys); const iterator = this.attributesToDecryptIterator(descriptor, attributes, params); let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - try { - iteratorResult = iterator.next( - (await this.crypto.decrypt(attributeValue, encryptionAAD)) as string - ); - } catch (err) { - iterator.throw!(err); + + let decryptionError; + for (const decrypter of decrypters) { + try { + iteratorResult = iterator.next(await decrypter.decrypt(attributeValue, encryptionAAD)); + decryptionError = undefined; + break; + } catch (err) { + // Remember the error thrown when we tried to decrypt with the primary key. + if (!decryptionError) { + decryptionError = err; + } + } + } + + if (decryptionError) { + iterator.throw!(decryptionError); } } @@ -350,17 +380,31 @@ export class EncryptedSavedObjectsService { public decryptAttributesSync>( descriptor: SavedObjectDescriptor, attributes: T, - params?: CommonParameters + params?: DecryptParameters ): T { + const decrypters = this.getDecrypters(params?.useDecryptionOnlyKeys); const iterator = this.attributesToDecryptIterator(descriptor, attributes, params); let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - try { - iteratorResult = iterator.next(this.crypto.decryptSync(attributeValue, encryptionAAD)); - } catch (err) { - iterator.throw!(err); + + let decryptionError; + for (const decrypter of decrypters) { + try { + iteratorResult = iterator.next(decrypter.decryptSync(attributeValue, encryptionAAD)); + decryptionError = undefined; + break; + } catch (err) { + // Remember the error thrown when we tried to decrypt with the primary key. + if (!decryptionError) { + decryptionError = err; + } + } + } + + if (decryptionError) { + iterator.throw!(decryptionError); } } @@ -464,4 +508,30 @@ export class EncryptedSavedObjectsService { return stringify([...descriptorToArray(descriptor), attributesAAD]); } + + /** + * Returns NodeCrypto instance used for encryption (the primary crypto). + */ + private getEncrypter() { + return this.cryptos[0]; + } + + /** + * Returns list of NodeCrypto instances used for decryption. + * @param useDecryptionOnlyKeys Specifies whether returned decrypters should include only those + * that are using decryption only keys (the secondary cryptos). + */ + private getDecrypters(useDecryptionOnlyKeys?: boolean) { + if (!useDecryptionOnlyKeys) { + return this.cryptos; + } + + if (this.cryptos.length === 1) { + throw new Error( + `"useDecryptionOnlyKeys" cannot be set when decryption only keys aren't configured.` + ); + } + + return this.cryptos.slice(1); + } } diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encryption_key_rotation_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encryption_key_rotation_service.ts new file mode 100644 index 000000000000000..edc2a5a92b2a214 --- /dev/null +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encryption_key_rotation_service.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ISavedObjectTypeRegistry, + KibanaRequest, + Logger, + SavedObject, + StartServicesAccessor, +} from 'src/core/server'; +import { AuthenticatedUser, SecurityPluginSetup } from '../../../security/server'; +import { getDescriptorNamespace } from '../saved_objects/get_descriptor_namespace'; +import { EncryptedSavedObjectsService } from './encrypted_saved_objects_service'; + +interface EncryptionKeyRotationServiceOptions { + logger: Logger; + service: PublicMethodsOf; + getStartServices: StartServicesAccessor; + security?: SecurityPluginSetup; + batchSize: number; +} + +interface EncryptionKeyRotationResult { + /** + * The total number of the Saved Objects encrypted by the Encrypted Saved Objects plugin. + */ + total: number; + + /** + * The number of the Saved Objects that were still encrypted with one of the secondary encryption + * keys and were successfully re-encrypted with the primary key. + */ + rotated: number; + + /** + * The number of the Saved Objects that were still encrypted with one of the secondary encryption + * keys that we failed to re-encrypt with the primary key. + */ + failed: number; +} + +/** + * Service that deals with encryption key rotation matters. + */ +export class EncryptionKeyRotationService { + constructor(private readonly options: EncryptionKeyRotationServiceOptions) {} + + public async rotate(request: KibanaRequest): Promise { + this.options.logger.debug('Starting encryption key rotation.'); + + const [{ savedObjects }] = await this.options.getStartServices(); + const typeRegistry = savedObjects.getTypeRegistry(); + + const registeredSavedObjectTypes = []; + const registeredHiddenSavedObjectTypes = []; + for (const type of typeRegistry.getAllTypes()) { + if (this.options.service.isRegistered(type.name)) { + registeredSavedObjectTypes.push(type.name); + + if (type.hidden) { + registeredHiddenSavedObjectTypes.push(type.name); + } + } + } + + if (registeredSavedObjectTypes.length === 0) { + this.options.logger.debug( + 'There are no registered encrypted Saved Object types, encryption key rotation is not needed.' + ); + return { total: 0, rotated: 0, failed: 0 }; + } + + const user = this.options.security?.authc.getCurrentUser(request) ?? undefined; + const retrieveClient = savedObjects.getScopedClient(request, { + includedHiddenTypes: registeredHiddenSavedObjectTypes, + excludedWrappers: ['encryptedSavedObjects'], + }); + const updateClient = savedObjects.getScopedClient(request, { + includedHiddenTypes: registeredHiddenSavedObjectTypes, + }); + + let currentPage = 1; + const firstPageResponse = await retrieveClient.find({ + type: registeredSavedObjectTypes, + perPage: this.options.batchSize, + page: currentPage, + namespaces: ['*'], + }); + const total = firstPageResponse.total; + const totalPages = Math.ceil(total / this.options.batchSize); + + let failed = 0; + let rotated = 0; + let savedObjectsToDecrypt = firstPageResponse.saved_objects; + while (savedObjectsToDecrypt.length > 0) { + let savedObjectsToEncrypt; + try { + // While we're decrypting Saved Objects from the current page (CPU bound task) we can try to + // fetch the next Saved Objects page, if any (Network-bound task). + const decryptAndFetchResult = await Promise.all([ + this.decryptSavedObjectsAttributes(savedObjectsToDecrypt, typeRegistry, user), + // If we fail to fetch next page of the Saved Objects we should terminate rotation and return + // results marking all not yet processed Saved Objects as failed. + ++currentPage <= totalPages + ? retrieveClient + .find({ + type: registeredSavedObjectTypes, + perPage: this.options.batchSize, + page: currentPage, + namespaces: ['*'], + }) + .then((nextPageResponse) => nextPageResponse.saved_objects) + : Promise.resolve([]), + ]); + + savedObjectsToEncrypt = decryptAndFetchResult[0]; + savedObjectsToDecrypt = decryptAndFetchResult[1]; + } catch (err) { + this.options.logger.error( + `Failed to fetch saved objects page "${currentPage}": ${err.message}` + ); + return { total, rotated, failed: total - rotated }; + } + + if (savedObjectsToEncrypt.length > 0) { + try { + const succeeded = ( + await updateClient.bulkUpdate(savedObjectsToEncrypt) + ).saved_objects.filter((savedObject) => !savedObject.error).length; + + rotated += succeeded; + failed += savedObjectsToEncrypt.length - succeeded; + } catch (err) { + this.options.logger.error(`Failed to update saved objects: ${err.message}`); + failed += savedObjectsToEncrypt.length; + } + } + } + + this.options.logger.debug( + `Encryption key rotation is completed. ${rotated} objects out ouf ${total} were successfully re-encrypted with the primary encryption key and ${failed} objects failed.` + ); + return { total, rotated, failed }; + } + + /** + * Takes a list of Saved Objects and tries to decrypt their attributes with the secondary encryption + * keys, silently skipping those that cannot be decrypted. + * @param savedObjects Saved Objects to decrypt attributes for. + * @param typeRegistry Saved Objects type registry. + * @param user The user that initiated decryption. + */ + private async decryptSavedObjectsAttributes( + savedObjects: SavedObject[], + typeRegistry: ISavedObjectTypeRegistry, + user?: AuthenticatedUser + ) { + const decryptedSavedObjects = []; + for (const savedObject of savedObjects) { + const namespace = getDescriptorNamespace( + typeRegistry, + savedObject.type, + savedObject.namespaces?.[0] + ); + try { + decryptedSavedObjects.push({ + ...savedObject, + attributes: await this.options.service.decryptAttributes( + { type: savedObject.type, id: savedObject.id, namespace }, + savedObject.attributes as Record, + { useDecryptionOnlyKeys: true, user } + ), + }); + } catch { + // Just skip object if we couldn't decrypt it with the decryption-only keys. + } + } + + return decryptedSavedObjects; + } +} diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts index 75445bd24eba848..ff5e5fdc010596b 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts @@ -12,3 +12,4 @@ export { } from './encrypted_saved_objects_service'; export { EncryptionError } from './encryption_error'; export { EncryptedSavedObjectAttributesDefinition } from './encrypted_saved_object_type_definition'; +export { EncryptionKeyRotationService } from './encryption_key_rotation_service'; diff --git a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts index 69777798ddf1920..5b86468de16c6d1 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts @@ -13,10 +13,12 @@ import { EncryptedSavedObjectsService, EncryptedSavedObjectTypeRegistration, EncryptionError, + EncryptionKeyRotationService, } from './crypto'; import { EncryptedSavedObjectsAuditLogger } from './audit'; import { setupSavedObjects, ClientInstanciator } from './saved_objects'; import { getCreateMigration, CreateEncryptedSavedObjectsMigrationFn } from './create_migration'; +import { defineRoutes } from './routes'; export interface PluginsSetup { security?: SecurityPluginSetup; @@ -48,18 +50,21 @@ export class Plugin { core: CoreSetup, deps: PluginsSetup ): Promise { - const { - config: { encryptionKey }, - usingEphemeralEncryptionKey, - } = await createConfig$(this.initializerContext).pipe(first()).toPromise(); - - const crypto = nodeCrypto({ encryptionKey }); - + const config = await createConfig$(this.initializerContext).pipe(first()).toPromise(); const auditLogger = new EncryptedSavedObjectsAuditLogger( deps.security?.audit.getLogger('encryptedSavedObjects') ); + + // This list of cryptos consists of the primary crypto that is used for both encryption and + // decryption, and the optional secondary cryptos that are used for decryption only. + const cryptos = [ + nodeCrypto({ encryptionKey: config.encryptionKey }), + ...config.keyRotation.decryptionOnlyKeys.map((decryptionKey) => + nodeCrypto({ encryptionKey: decryptionKey }) + ), + ]; const service = Object.freeze( - new EncryptedSavedObjectsService(crypto, this.logger, auditLogger) + new EncryptedSavedObjectsService(cryptos, this.logger, auditLogger) ); this.savedObjectsSetup = setupSavedObjects({ @@ -69,15 +74,30 @@ export class Plugin { getStartServices: core.getStartServices, }); + defineRoutes({ + router: core.http.createRouter(), + logger: this.initializerContext.logger.get('routes'), + encryptionKeyRotationService: Object.freeze( + new EncryptionKeyRotationService({ + logger: this.logger.get('key-rotation-service'), + service, + getStartServices: core.getStartServices, + security: deps.security, + batchSize: config.keyRotation.batchSize, + }) + ), + config, + }); + return { registerType: (typeRegistration: EncryptedSavedObjectTypeRegistration) => service.registerType(typeRegistration), - usingEphemeralEncryptionKey, + usingEphemeralEncryptionKey: config.usingEphemeralEncryptionKey, createMigration: getCreateMigration( service, (typeRegistration: EncryptedSavedObjectTypeRegistration) => { const serviceForMigration = new EncryptedSavedObjectsService( - crypto, + cryptos, this.logger, auditLogger ); diff --git a/x-pack/plugins/encrypted_saved_objects/server/routes/index.ts b/x-pack/plugins/encrypted_saved_objects/server/routes/index.ts new file mode 100644 index 000000000000000..a4135b136094c58 --- /dev/null +++ b/x-pack/plugins/encrypted_saved_objects/server/routes/index.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, Logger } from '../../../../../src/core/server'; +import { ConfigType } from '../config'; +import { EncryptionKeyRotationService } from '../crypto'; + +import { defineKeyRotationRoutes } from './key_rotation'; + +/** + * Describes parameters used to define HTTP routes. + */ +export interface RouteDefinitionParams { + router: IRouter; + logger: Logger; + config: ConfigType; + encryptionKeyRotationService: PublicMethodsOf; +} + +export function defineRoutes(params: RouteDefinitionParams) { + if (params.config.keyRotation.decryptionOnlyKeys.length > 0) { + defineKeyRotationRoutes(params); + } +} diff --git a/x-pack/plugins/encrypted_saved_objects/server/routes/key_rotation.ts b/x-pack/plugins/encrypted_saved_objects/server/routes/key_rotation.ts new file mode 100644 index 000000000000000..6975eed28df4ccc --- /dev/null +++ b/x-pack/plugins/encrypted_saved_objects/server/routes/key_rotation.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; +import { RouteDefinitionParams } from '.'; + +/** + * Defines routes that are used for encryption key rotation. + */ +export function defineKeyRotationRoutes({ + encryptionKeyRotationService, + router, + logger, +}: RouteDefinitionParams) { + router.post( + { + path: '/api/encrypted_saved_objects/rotate_key', + validate: { + query: schema.object({ + conflicts: schema.oneOf([schema.literal('abort'), schema.literal('proceed')], { + defaultValue: 'proceed', + }), + }), + }, + options: { + tags: ['access:rotateEncryptionKey'], + }, + }, + async (context, request, response) => { + try { + return response.ok({ body: await encryptionKeyRotationService.rotate(request) }); + } catch (err) { + logger.error(err); + return response.internalError(); + } + } + ); +}