From ce24d1a3fff7f38a8132615b11928ec4417566b4 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 24 Oct 2023 13:22:50 +0100 Subject: [PATCH] [Fleet] Output Secrets Backend (#169221) ## Summary Had to recreate this after pinging the whole of Kibana accidentally on the last one :D Part of #157458 Adds the ability to sepcify secrets in outputs. Currently the following secrets are supported: - kafka output SSL key - kafka output password - logstash output SSL key The behaviour is as follows: - on create, secrets are created and the plain string is replaced with a secret reference on the output saved object - on update, if a secret is updated, the old secret is deleted and a new one is created, the new secret ref is added to the output - on delete, all secrets are deleted - behaviour is behind a feature flag as flee tserver does not support these yet Secrets are only enabled if a fleet server of 8.10.0 or greater is connected. Integration tests added for all scenarios. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../current_mappings.json | 26 ++ .../group2/check_registered_types.test.ts | 2 +- .../fleet/common/experimental_features.ts | 1 + .../plugins/fleet/common/openapi/bundled.json | 29 ++ .../plugins/fleet/common/openapi/bundled.yaml | 18 + .../schemas/output_create_request_kafka.yaml | 11 + .../output_create_request_logstash.yaml | 8 + .../fleet/common/types/models/output.ts | 23 ++ .../fleet/common/types/models/secret.ts | 4 + .../fleet/server/routes/output/handler.ts | 23 +- .../fleet/server/saved_objects/index.ts | 22 ++ .../agent_policies/full_agent_policy.ts | 14 +- .../fleet/server/services/agents/versions.ts | 5 +- .../fleet/server/services/output.test.ts | 14 +- .../plugins/fleet/server/services/output.ts | 56 +++- .../plugins/fleet/server/services/secrets.ts | 192 ++++++++++- .../plugins/fleet/server/services/settings.ts | 14 + .../fleet/server/types/models/output.ts | 36 +- .../fleet/server/types/so_attributes.ts | 12 + .../apis/outputs/crud.ts | 310 ++++++++++++++++++ .../apis/policy_secrets.ts | 55 ++++ .../test/fleet_api_integration/config.base.ts | 2 +- 22 files changed, 852 insertions(+), 25 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 7fc18bb58a00b5..dea741b930a7d3 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1816,6 +1816,32 @@ }, "channel_buffer_size": { "type": "integer" + }, + "secrets": { + "dynamic": false, + "properties": { + "password": { + "dynamic": false, + "properties": { + "id": { + "type": "keyword" + } + } + }, + "ssl": { + "dynamic": false, + "properties": { + "key": { + "dynamic": false, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } + } } } }, diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index c3c631591e7eec..6de7f6e90de7ce 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0", + "ingest-outputs": "3982d6296373111467e839a0768d3e1c4d0ebc61", "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 5925949d61737b..89a5282199b753 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -23,6 +23,7 @@ export const allowedExperimentalValues = Object.freeze>( agentTamperProtectionEnabled: true, secretsStorage: true, kafkaOutput: true, + outputSecretsStorage: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index e260764d57de85..5953bd1fdc2871 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -8121,6 +8121,22 @@ }, "required_acks": { "type": "number" + }, + "secrets": { + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "ssl": { + "type": "object", + "properties": { + "key": { + "type": "string" + } + } + } + } } }, "required": [ @@ -8216,6 +8232,19 @@ "type": "boolean" } } + }, + "secrets": { + "type": "object", + "properties": { + "ssl": { + "type": "object", + "properties": { + "key": { + "type": "string" + } + } + } + } } }, "required": [ diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 9ed380d26726b1..c348cec266c124 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5240,6 +5240,16 @@ components: type: number required_acks: type: number + secrets: + type: object + properties: + password: + type: string + ssl: + type: object + properties: + key: + type: string required: - name - type @@ -5304,6 +5314,14 @@ components: type: number loadbalance: type: boolean + secrets: + type: object + properties: + ssl: + type: object + properties: + key: + type: string required: - name - hosts diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml index fa76c2301ed943..0a50abfb03d88e 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml @@ -122,6 +122,17 @@ properties: type: number required_acks: type: number + secrets: + type: object + properties: + password: + type: string + ssl: + type: object + properties: + key: + type: string + required: - name - type diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml index 9618694490004e..a7142c967c4dcf 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml @@ -54,6 +54,14 @@ properties: type: number loadbalance: type: boolean + secrets: + type: object + properties: + ssl: + type: object + properties: + key: + type: string required: - name - hosts diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index a537e0ab0233b4..5bed131be8d56b 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -43,6 +43,15 @@ interface NewBaseOutput { proxy_id?: string | null; shipper?: ShipperOutput | null; allow_edit?: string[]; + secrets?: { + ssl?: { + key?: + | string + | { + id: string; + }; + }; + }; } export interface NewElasticsearchOutput extends NewBaseOutput { @@ -112,4 +121,18 @@ export interface KafkaOutput extends NewBaseOutput { timeout?: number; broker_timeout?: number; required_acks?: ValueOf; + secrets?: { + password?: + | string + | { + id: string; + }; + ssl?: { + key?: + | string + | { + id: string; + }; + }; + }; } diff --git a/x-pack/plugins/fleet/common/types/models/secret.ts b/x-pack/plugins/fleet/common/types/models/secret.ts index cf95d88b82e2bd..74341275859f26 100644 --- a/x-pack/plugins/fleet/common/types/models/secret.ts +++ b/x-pack/plugins/fleet/common/types/models/secret.ts @@ -21,6 +21,10 @@ export interface SecretPath { path: string; value: PackagePolicyConfigRecordEntry; } +export interface OutputSecretPath { + path: string; + value: string | { id: string }; +} // this is used in the top level secret_refs array on package and agent policies export interface PolicySecretReference { id: string; diff --git a/x-pack/plugins/fleet/server/routes/output/handler.ts b/x-pack/plugins/fleet/server/routes/output/handler.ts index d90ed9ddf11def..475e7e96255042 100644 --- a/x-pack/plugins/fleet/server/routes/output/handler.ts +++ b/x-pack/plugins/fleet/server/routes/output/handler.ts @@ -8,6 +8,10 @@ import type { RequestHandler } from '@kbn/core/server'; import type { TypeOf } from '@kbn/config-schema'; +import Boom from '@hapi/boom'; + +import { outputType } from '../../../common/constants'; + import type { DeleteOutputRequestSchema, GetOneOutputRequestSchema, @@ -18,6 +22,7 @@ import type { DeleteOutputResponse, GetOneOutputResponse, GetOutputsResponse, + Output, PostLogstashApiKeyResponse, } from '../../../common/types'; import { outputService } from '../../services/output'; @@ -25,6 +30,15 @@ import { defaultFleetErrorHandler, FleetUnauthorizedError } from '../../errors'; import { agentPolicyService } from '../../services'; import { generateLogstashApiKey, canCreateLogstashApiKey } from '../../services/api_keys'; +function ensureNoDuplicateSecrets(output: Partial) { + if (output.type === outputType.Kafka && output?.password && output?.secrets?.password) { + throw Boom.badRequest('Cannot specify both password and secrets.password'); + } + if (output.ssl?.key && output.secrets?.ssl?.key) { + throw Boom.badRequest('Cannot specify both ssl.key and secrets.ssl.key'); + } +} + export const getOutputsHandler: RequestHandler = async (context, request, response) => { const soClient = (await context.core).savedObjects.client; try { @@ -74,8 +88,10 @@ export const putOutputHandler: RequestHandler< const coreContext = await context.core; const soClient = coreContext.savedObjects.client; const esClient = coreContext.elasticsearch.client.asInternalUser; + const outputUpdate = request.body; + ensureNoDuplicateSecrets(outputUpdate); try { - await outputService.update(soClient, esClient, request.params.outputId, request.body); + await outputService.update(soClient, esClient, request.params.outputId, outputUpdate); const output = await outputService.get(soClient, request.params.outputId); if (output.is_default || output.is_default_monitoring) { await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); @@ -108,8 +124,9 @@ export const postOutputHandler: RequestHandler< const soClient = coreContext.savedObjects.client; const esClient = coreContext.elasticsearch.client.asInternalUser; try { - const { id, ...data } = request.body; - const output = await outputService.create(soClient, esClient, data, { id }); + const { id, ...newOutput } = request.body; + ensureNoDuplicateSecrets(newOutput); + const output = await outputService.create(soClient, esClient, newOutput, { id }); if (output.is_default || output.is_default_monitoring) { await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); } diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 8791d9871982b5..7458fad2459b6b 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -247,6 +247,28 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ broker_buffer_size: { type: 'integer' }, required_acks: { type: 'integer' }, channel_buffer_size: { type: 'integer' }, + secrets: { + dynamic: false, + properties: { + password: { + dynamic: false, + properties: { + id: { type: 'keyword' }, + }, + }, + ssl: { + dynamic: false, + properties: { + key: { + dynamic: false, + properties: { + id: { type: 'keyword' }, + }, + }, + }, + }, + }, + }, }, }, modelVersions: { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index d15e7adb9bd7fc..ffa681fdf0d408 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -31,6 +31,8 @@ import { getPackageInfo } from '../epm/packages'; import { pkgToPkgKey, splitPkgKey } from '../epm/registry'; import { appContextService } from '../app_context'; +import { getOutputSecretReferences } from '../secrets'; + import { getMonitoringPermissions } from './monitoring_permissions'; import { storedPackagePoliciesToAgentInputs } from '.'; import { @@ -110,6 +112,10 @@ export async function getFullAgentPolicy( return acc; }, {} as NonNullable['features']); + const outputSecretReferences = outputs.flatMap((output) => getOutputSecretReferences(output)); + const packagePolicySecretReferences = (agentPolicy?.package_policies || []).flatMap( + (policy) => policy.secret_references || [] + ); const defaultMonitoringConfig: FullAgentPolicyMonitoring = { enabled: false, logs: false, @@ -152,9 +158,7 @@ export async function getFullAgentPolicy( }, {}), }, inputs, - secret_references: (agentPolicy?.package_policies || []).flatMap( - (policy) => policy.secret_references || [] - ), + secret_references: [...outputSecretReferences, ...packagePolicySecretReferences], revision: agentPolicy.revision, agent: { download: { @@ -305,7 +309,8 @@ export function transformOutputToFullPolicyOutput( standalone = false ): FullAgentPolicyOutput { // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper } = output; + const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper, secrets } = + output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; @@ -432,6 +437,7 @@ export function transformOutputToFullPolicyOutput( ...(!isShipperDisabled ? generalShipperData : {}), ...(ca_sha256 ? { ca_sha256 } : {}), ...(ssl ? { ssl } : {}), + ...(secrets ? { secrets } : {}), ...(ca_trusted_fingerprint ? { 'ssl.ca_trusted_fingerprint': ca_trusted_fingerprint } : {}), }; diff --git a/x-pack/plugins/fleet/server/services/agents/versions.ts b/x-pack/plugins/fleet/server/services/agents/versions.ts index fda703056daa5d..7a1b82bb723594 100644 --- a/x-pack/plugins/fleet/server/services/agents/versions.ts +++ b/x-pack/plugins/fleet/server/services/agents/versions.ts @@ -75,9 +75,8 @@ export const getAvailableVersions = async ({ return availableVersions; } catch (e) { - if (e.code === 'ENOENT' && !config?.internal?.onlyAllowAgentUpgradeToKnownVersions) { - // If the file does not exist, return the current version - return [kibanaVersion]; + if (e.code === 'ENOENT') { + return config?.internal?.onlyAllowAgentUpgradeToKnownVersions ? [] : [kibanaVersion]; } throw e; } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 12abc7c0bec706..744be302bddc04 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -9,8 +9,9 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/serv import { securityMock } from '@kbn/security-plugin/server/mocks'; -import type { OutputSOAttributes } from '../types'; +import type { Logger } from '@kbn/logging'; +import type { OutputSOAttributes } from '../types'; import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { outputService, outputIdToUuid } from './output'; @@ -28,6 +29,17 @@ mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ ...securityMock.createSetup(), })); +mockedAppContextService.getLogger.mockImplementation(() => { + return { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as unknown as Logger; +}); + +mockedAppContextService.getExperimentalFeatures.mockReturnValue({}); + const mockedAgentPolicyService = agentPolicyService as jest.Mocked; const CLOUD_ID = diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 15b256fc84207a..efbc35806e925b 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -25,6 +25,7 @@ import type { OutputSOAttributes, AgentPolicy, OutputSoKafkaAttributes, + PolicySecretReference, } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, @@ -53,6 +54,12 @@ import { agentPolicyService } from './agent_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; import { auditLoggingService } from './audit_logging'; +import { + deleteOutputSecrets, + deleteSecrets, + extractAndUpdateOutputSecrets, + extractAndWriteOutputSecrets, +} from './secrets'; type Nullable = { [P in keyof T]: T[P] | null }; @@ -103,6 +110,12 @@ function outputSavedObjectToOutput(so: SavedObject): Output }; } +function isOutputSecretsEnabled() { + const { outputSecretsStorage } = appContextService.getExperimentalFeatures(); + + return !!outputSecretsStorage; +} + async function getAgentPoliciesPerOutput( soClient: SavedObjectsClientContract, outputId?: string, @@ -407,7 +420,7 @@ class OutputService { output: NewOutput, options?: { id?: string; fromPreconfiguration?: boolean; overwrite?: boolean } ): Promise { - const data: OutputSOAttributes = { ...omit(output, 'ssl') }; + const data: OutputSOAttributes = { ...omit(output, ['ssl', 'secrets']) }; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { @@ -527,6 +540,15 @@ class OutputService { const id = options?.id ? outputIdToUuid(options.id) : SavedObjectsUtils.generateId(); + if (isOutputSecretsEnabled()) { + const { output: outputWithSecrets } = await extractAndWriteOutputSecrets({ + output, + esClient, + }); + + if (outputWithSecrets.secrets) data.secrets = outputWithSecrets.secrets; + } + auditLoggingService.writeCustomSoAuditLog({ action: 'create', id, @@ -668,7 +690,14 @@ class OutputService { savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, }); - return this.encryptedSoClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); + const soDeleteResult = this.encryptedSoClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); + + await deleteOutputSecrets({ + esClient: appContextService.getInternalUserESClient(), + output: originalOutput, + }); + + return soDeleteResult; } public async update( @@ -680,6 +709,7 @@ class OutputService { fromPreconfiguration: false, } ) { + let secretsToDelete: PolicySecretReference[] = []; const originalOutput = await this.get(soClient, id); this._validateFieldsAreEditable(originalOutput, data, id, fromPreconfiguration); @@ -692,7 +722,17 @@ class OutputService { ); } - const updateData: Nullable> = { ...omit(data, 'ssl') }; + const updateData: Nullable> = { ...omit(data, ['ssl', 'secrets']) }; + if (isOutputSecretsEnabled()) { + const secretsRes = await extractAndUpdateOutputSecrets({ + oldOutput: originalOutput, + outputUpdate: data, + esClient, + }); + + updateData.secrets = secretsRes.outputUpdate.secrets; + secretsToDelete = secretsRes.secretsToDelete; + } const mergedType = data.type ?? originalOutput.type; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); await validateTypeChanges( @@ -881,6 +921,16 @@ class OutputService { if (outputSO.error) { throw new Error(outputSO.error.message); } + + if (secretsToDelete.length) { + try { + await deleteSecrets({ esClient, ids: secretsToDelete.map((s) => s.id) }); + } catch (err) { + appContextService + .getLogger() + .warn(`Error cleaning up secrets for output ${id}: ${err.message}`); + } + } } } diff --git a/x-pack/plugins/fleet/server/services/secrets.ts b/x-pack/plugins/fleet/server/services/secrets.ts index 886e7d7243172a..fcb6a0fdc2fe6c 100644 --- a/x-pack/plugins/fleet/server/services/secrets.ts +++ b/x-pack/plugins/fleet/server/services/secrets.ts @@ -10,9 +10,16 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/ import { keyBy } from 'lodash'; import { set } from '@kbn/safer-lodash-set'; +import type { KafkaOutput, Output, OutputSecretPath } from '../../common/types'; + import { packageHasNoPolicyTemplates } from '../../common/services/policy_template'; -import type { NewPackagePolicy, RegistryStream, UpdatePackagePolicy } from '../../common'; +import type { + NewOutput, + NewPackagePolicy, + RegistryStream, + UpdatePackagePolicy, +} from '../../common'; import { SO_SEARCH_LIMIT } from '../../common'; import { @@ -117,7 +124,7 @@ export async function deleteSecretsIfNotReferenced(opts: { return; } try { - await _deleteSecrets({ + await deleteSecrets({ esClient, ids: secretsToDelete, }); @@ -166,7 +173,7 @@ export async function findPackagePoliciesUsingSecrets(opts: { return res; } -export async function _deleteSecrets(opts: { +export async function deleteSecrets(opts: { esClient: ElasticsearchClient; ids: string[]; }): Promise { @@ -237,6 +244,105 @@ export async function extractAndWriteSecrets(opts: { }; } +export async function extractAndWriteOutputSecrets(opts: { + output: NewOutput; + esClient: ElasticsearchClient; +}): Promise<{ output: NewOutput; secretReferences: PolicySecretReference[] }> { + const { output, esClient } = opts; + + const secretPaths = getOutputSecretPaths(output.type, output).filter( + (path) => typeof path.value === 'string' + ); + + if (secretPaths.length === 0) { + return { output, secretReferences: [] }; + } + + const secrets = await createSecrets({ + esClient, + values: secretPaths.map(({ value }) => value as string), + }); + + const outputWithSecretRefs = JSON.parse(JSON.stringify(output)); + secretPaths.forEach((secretPath, i) => { + set(outputWithSecretRefs, secretPath.path, { id: secrets[i].id }); + }); + + return { + output: outputWithSecretRefs, + secretReferences: secrets.map(({ id }) => ({ id })), + }; +} + +function getOutputSecretPaths( + outputType: NewOutput['type'], + output: NewOutput | Partial +): OutputSecretPath[] { + const outputSecretPaths: OutputSecretPath[] = []; + + if ((outputType === 'kafka' || outputType === 'logstash') && output.secrets?.ssl?.key) { + outputSecretPaths.push({ + path: 'secrets.ssl.key', + value: output.secrets.ssl.key, + }); + } + + if (outputType === 'kafka') { + const kafkaOutput = output as KafkaOutput; + if (kafkaOutput?.secrets?.password) { + outputSecretPaths.push({ + path: 'secrets.password', + value: kafkaOutput.secrets.password, + }); + } + } + + return outputSecretPaths; +} + +export async function deleteOutputSecrets(opts: { + output: Output; + esClient: ElasticsearchClient; +}): Promise { + const { output, esClient } = opts; + + const outputType = output.type; + const outputSecretPaths = getOutputSecretPaths(outputType, output); + + if (outputSecretPaths.length === 0) { + return Promise.resolve(); + } + + const secretIds = outputSecretPaths.map(({ value }) => (value as { id: string }).id); + + try { + return deleteSecrets({ esClient, ids: secretIds }); + } catch (err) { + appContextService.getLogger().warn(`Error deleting secrets: ${err}`); + } +} + +export function getOutputSecretReferences(output: Output): PolicySecretReference[] { + const outputSecretPaths: PolicySecretReference[] = []; + + if ( + (output.type === 'kafka' || output.type === 'logstash') && + typeof output.secrets?.ssl?.key === 'object' + ) { + outputSecretPaths.push({ + id: output.secrets.ssl.key.id, + }); + } + + if (output.type === 'kafka' && typeof output?.secrets?.password === 'object') { + outputSecretPaths.push({ + id: output.secrets.password.id, + }); + } + + return outputSecretPaths; +} + export async function extractAndUpdateSecrets(opts: { oldPackagePolicy: PackagePolicy; packagePolicyUpdate: UpdatePackagePolicy; @@ -289,6 +395,52 @@ export async function extractAndUpdateSecrets(opts: { secretsToDelete, }; } +export async function extractAndUpdateOutputSecrets(opts: { + oldOutput: Output; + outputUpdate: Partial; + esClient: ElasticsearchClient; +}): Promise<{ + outputUpdate: Partial; + secretReferences: PolicySecretReference[]; + secretsToDelete: PolicySecretReference[]; +}> { + const { oldOutput, outputUpdate, esClient } = opts; + const outputType = outputUpdate.type || oldOutput.type; + const oldSecretPaths = getOutputSecretPaths(outputType, oldOutput); + const updatedSecretPaths = getOutputSecretPaths(outputType, outputUpdate); + + if (!oldSecretPaths.length && !updatedSecretPaths.length) { + return { outputUpdate, secretReferences: [], secretsToDelete: [] }; + } + + const { toCreate, toDelete, noChange } = diffOutputSecretPaths( + oldSecretPaths, + updatedSecretPaths + ); + + const createdSecrets = await createSecrets({ + esClient, + values: toCreate.map((secretPath) => secretPath.value as string), + }); + + const outputWithSecretRefs = JSON.parse(JSON.stringify(outputUpdate)); + toCreate.forEach((secretPath, i) => { + set(outputWithSecretRefs, secretPath.path, { id: createdSecrets[i].id }); + }); + + const secretReferences = [ + ...noChange.map((secretPath) => ({ id: (secretPath.value as { id: string }).id })), + ...createdSecrets.map(({ id }) => ({ id })), + ]; + + return { + outputUpdate: outputWithSecretRefs, + secretReferences, + secretsToDelete: toDelete.map((secretPath) => ({ + id: (secretPath.value as { id: string }).id, + })), + }; +} function isSecretVar(varDef: RegistryVarsEntry) { return varDef.secret === true; @@ -340,6 +492,36 @@ export function diffSecretPaths( return { toCreate: [...toCreate, ...remainingNewPaths], toDelete, noChange }; } +export function diffOutputSecretPaths( + oldPaths: OutputSecretPath[], + newPaths: OutputSecretPath[] +): { toCreate: OutputSecretPath[]; toDelete: OutputSecretPath[]; noChange: OutputSecretPath[] } { + const toCreate: OutputSecretPath[] = []; + const toDelete: OutputSecretPath[] = []; + const noChange: OutputSecretPath[] = []; + const newPathsByPath = keyBy(newPaths, 'path'); + + for (const oldPath of oldPaths) { + if (!newPathsByPath[oldPath.path]) { + toDelete.push(oldPath); + } + + const newPath = newPathsByPath[oldPath.path]; + if (newPath && newPath.value) { + const newValue = newPath.value; + if (typeof newValue === 'string') toCreate.push(newPath); + toDelete.push(oldPath); + } else { + noChange.push(newPath); + } + delete newPathsByPath[oldPath.path]; + } + + const remainingNewPaths = Object.values(newPathsByPath); + + return { toCreate: [...toCreate, ...remainingNewPaths], toDelete, noChange }; +} + // Given a package policy and a package, // returns an array of lodash style paths to all secrets and their current values export function getPolicySecretPaths( @@ -381,9 +563,9 @@ export async function isSecretStorageEnabled( // now check the flag in settings to see if the fleet server requirement has already been met // once the requirement has been met, secrets are always on - const settings = await settingsService.getSettings(soClient); + const settings = await settingsService.getSettingsOrUndefined(soClient); - if (settings.secret_storage_requirements_met) { + if (settings && settings.secret_storage_requirements_met) { logger.debug('Secrets storage already met, turned on is settings'); return true; } diff --git a/x-pack/plugins/fleet/server/services/settings.ts b/x-pack/plugins/fleet/server/services/settings.ts index 2b5d228ffc97e1..6e248d7817a5fc 100644 --- a/x-pack/plugins/fleet/server/services/settings.ts +++ b/x-pack/plugins/fleet/server/services/settings.ts @@ -41,6 +41,20 @@ export async function getSettings(soClient: SavedObjectsClientContract): Promise }; } +export async function getSettingsOrUndefined( + soClient: SavedObjectsClientContract +): Promise { + try { + return await getSettings(soClient); + } catch (e) { + if (e.isBoom && e.output.statusCode === 404) { + return undefined; + } + + throw e; + } +} + export async function settingsSetup(soClient: SavedObjectsClientContract) { try { await getSettings(soClient); diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index 845361116d7329..5bf4d22d8f1883 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -49,6 +49,13 @@ export const validateKafkaHost = (input: string): string | undefined => { return undefined; }; +const secretRefSchema = schema.oneOf([ + schema.object({ + id: schema.string(), + }), + schema.string(), +]); + /** * Base schemas */ @@ -124,6 +131,11 @@ export const LogstashSchema = { ...BaseSchema, type: schema.literal(outputType.Logstash), hosts: schema.arrayOf(schema.string({ validate: validateLogstashHost }), { minSize: 1 }), + secrets: schema.maybe( + schema.object({ + ssl: schema.maybe(schema.object({ key: schema.maybe(secretRefSchema) })), + }) + ), }; const LogstashUpdateSchema = { @@ -132,6 +144,11 @@ const LogstashUpdateSchema = { hosts: schema.maybe( schema.arrayOf(schema.string({ validate: validateLogstashHost }), { minSize: 1 }) ), + secrets: schema.maybe( + schema.object({ + ssl: schema.maybe(schema.object({ key: schema.maybe(secretRefSchema) })), + }) + ), }; /** @@ -200,10 +217,15 @@ export const KafkaSchema = { schema.never() ), password: schema.conditional( - schema.siblingRef('username'), - schema.string(), - schema.string(), - schema.never() + schema.siblingRef('secrets.password'), + secretRefSchema, + schema.never(), + schema.conditional( + schema.siblingRef('username'), + schema.string(), + schema.string(), + schema.never() + ) ), sasl: schema.maybe( schema.object({ @@ -237,6 +259,12 @@ export const KafkaSchema = { required_acks: schema.maybe( schema.oneOf([schema.literal(1), schema.literal(0), schema.literal(-1)]) ), + secrets: schema.maybe( + schema.object({ + password: schema.maybe(secretRefSchema), + ssl: schema.maybe(schema.object({ key: secretRefSchema })), + }) + ), }; const KafkaUpdateSchema = { diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index f6e7506d81ecf8..00bd1b3a67d575 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -148,10 +148,16 @@ interface OutputSoBaseAttributes { interface OutputSoElasticsearchAttributes extends OutputSoBaseAttributes { type: OutputType['Elasticsearch']; + secrets?: {}; } interface OutputSoLogstashAttributes extends OutputSoBaseAttributes { type: OutputType['Logstash']; + secrets?: { + ssl?: { + key?: { id: string }; + }; + }; } export interface OutputSoKafkaAttributes extends OutputSoBaseAttributes { @@ -193,6 +199,12 @@ export interface OutputSoKafkaAttributes extends OutputSoBaseAttributes { timeout?: number; broker_timeout?: number; required_acks?: ValueOf; + secrets?: { + password?: { id: string }; + ssl?: { + key?: { id: string }; + }; + }; } export type OutputSOAttributes = diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 3cc860433eb69e..686d29c9cdf221 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { GLOBAL_SETTINGS_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; @@ -15,9 +16,57 @@ export default function (providerContext: FtrProviderContext) { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); + const es = getService('es'); let pkgVersion: string; + const getSecrets = async (ids?: string[]) => { + const query = ids ? { terms: { _id: ids } } : { match_all: {} }; + return es.search({ + index: '.fleet-secrets', + body: { + query, + }, + }); + }; + + const getSecretById = (id: string) => { + return es.get({ + index: '.fleet-secrets', + id, + }); + }; + + const deleteAllSecrets = async () => { + try { + await es.deleteByQuery({ + index: '.fleet-secrets', + body: { + query: { + match_all: {}, + }, + }, + }); + } catch (err) { + // index doesn't exist + } + }; + + const enableSecrets = async () => { + try { + await kibanaServer.savedObjects.update({ + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + id: 'fleet-default-settings', + attributes: { + secret_storage_requirements_met: true, + }, + overwrite: false, + }); + } catch (e) { + throw e; + } + }; + describe('fleet_outputs_crud', async function () { skipIfNoDockerRegistry(providerContext); before(async () => { @@ -32,6 +81,7 @@ export default function (providerContext: FtrProviderContext) { let fleetServerPolicyWithCustomOutputId: string; before(async function () { + await enableSecrets(); // we must first force install the fleet_server package to override package verification error on policy create // https://github.com/elastic/kibana/issues/137450 const getPkRes = await supertest @@ -40,6 +90,8 @@ export default function (providerContext: FtrProviderContext) { .expect(200); pkgVersion = getPkRes.body.item.version; + await deleteAllSecrets(); + await supertest .post(`/api/fleet/epm/packages/fleet_server/${pkgVersion}`) .set('kbn-xsrf', 'xxxx') @@ -422,6 +474,84 @@ export default function (providerContext: FtrProviderContext) { const newOutput = outputs.filter((o: any) => o.id === defaultOutputId); expect(newOutput[0].shipper).to.equal(null); }); + + it('should allow secrets to be updated + delete unused secret', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'ssl', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + ssl: { + certificate: 'CERTIFICATE', + certificate_authorities: ['CA1', 'CA2'], + }, + secrets: { + ssl: { + key: 'KEY', + }, + }, + }) + .expect(200); + + const outputId = res.body.item.id; + const secretId = res.body.item.secrets.ssl.key.id; + const secret = await getSecretById(secretId); + // @ts-ignore _source unknown type + expect(secret._source.value).to.equal('KEY'); + + const updateRes = await supertest + .put(`/api/fleet/outputs/${outputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'ssl', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + ssl: { + certificate: 'CERTIFICATE', + certificate_authorities: ['CA1', 'CA2'], + }, + secrets: { + ssl: { + key: 'NEW_KEY', + }, + }, + }) + .expect(200); + + const updatedSecretId = updateRes.body.item.secrets.ssl.key.id; + + expect(updatedSecretId).not.to.equal(secretId); + + const updatedSecret = await getSecretById(updatedSecretId); + + // @ts-ignore _source unknown type + expect(updatedSecret._source.value).to.equal('NEW_KEY'); + + try { + await getSecretById(secretId); + expect().fail('Secret should have been deleted'); + } catch (e) { + // not found + } + }); }); describe('POST /outputs', () => { @@ -876,6 +1006,142 @@ export default function (providerContext: FtrProviderContext) { queue_flush_timeout: null, }); }); + + it('should not allow ssl.key and secrets.ssl.key to be set for logstash output ', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Logstash Output', + type: 'logstash', + hosts: ['test.fr:443'], + ssl: { + certificate: 'CERTIFICATE', + key: 'KEY', + certificate_authorities: ['CA1', 'CA2'], + }, + config_yaml: 'shipper: {}', + secrets: { ssl: { key: 'KEY' } }, + }) + .expect(400); + + expect(res.body.message).to.equal('Cannot specify both ssl.key and secrets.ssl.key'); + }); + + it('should not allow password and secrets.password to be set for kafka output ', async function () { + await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'user_pass', + username: 'user', + password: 'pass', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + secrets: { password: 'pass' }, + }) + .expect(400); + }); + + it('should not allow ssl.key and secrets.ssl.key to be set for kafka output ', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'ssl', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + ssl: { + certificate: 'CERTIFICATE', + key: 'KEY', + certificate_authorities: ['CA1', 'CA2'], + }, + secrets: { + ssl: { + key: 'KEY', + }, + }, + }) + .expect(400); + + expect(res.body.message).to.equal('Cannot specify both ssl.key and secrets.ssl.key'); + }); + + it('should create ssl.key secret correctly', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'ssl', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + ssl: { + certificate: 'CERTIFICATE', + certificate_authorities: ['CA1', 'CA2'], + }, + secrets: { + ssl: { + key: 'KEY', + }, + }, + }) + .expect(200); + + const secretId = res.body.item.secrets.ssl.key.id; + const searchRes = await getSecrets([secretId]); + // @ts-ignore _source unknown type + expect(searchRes.hits.hits[0]._source.value).to.equal('KEY'); + }); + + it('should create ssl.password secret correctly', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Password Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'user_pass', + username: 'user', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + secrets: { password: 'pass' }, + }); + + const secretId = res.body.item.secrets.password.id; + const searchRes = await getSecrets([secretId]); + // @ts-ignore _source unknown type + expect(searchRes.hits.hits[0]._source.value).to.equal('pass'); + }); }); describe('DELETE /outputs/{outputId}', () => { @@ -949,6 +1215,50 @@ export default function (providerContext: FtrProviderContext) { expect(deleteResponse.id).to.eql(outputId); }); + + it('should delete secrets when deleting an output', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'ssl', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + ssl: { + certificate: 'CERTIFICATE', + certificate_authorities: ['CA1', 'CA2'], + }, + secrets: { + ssl: { + key: 'KEY', + }, + }, + }) + .expect(200); + + const outputWithSecretsId = res.body.item.id; + const secretId = res.body.item.secrets.ssl.key.id; + + await supertest + .delete(`/api/fleet/outputs/${outputWithSecretsId}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + try { + await getSecretById(secretId); + expect().fail('Secret should have been deleted'); + } catch (e) { + // not found + } + }); }); describe('Kafka output', () => { diff --git a/x-pack/test/fleet_api_integration/apis/policy_secrets.ts b/x-pack/test/fleet_api_integration/apis/policy_secrets.ts index d7d139c2ca884c..6197738eda4e8f 100644 --- a/x-pack/test/fleet_api_integration/apis/policy_secrets.ts +++ b/x-pack/test/fleet_api_integration/apis/policy_secrets.ts @@ -92,6 +92,29 @@ export default function (providerContext: FtrProviderContext) { return agentPolicyId; }; + const createOutputWithSecret = async () => { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Kafka Output With Password Secret', + type: 'kafka', + hosts: ['test.fr:2000'], + auth_type: 'user_pass', + username: 'user', + topics: [{ topic: 'topic1' }], + config_yaml: 'shipper: {}', + shipper: { + disk_queue_enabled: true, + disk_queue_path: 'path/to/disk/queue', + disk_queue_encryption_enabled: true, + }, + secrets: { password: 'pass' }, + }); + + return res.body.item; + }; + const createPolicyWithSecrets = async () => { return supertest .post(`/api/fleet/package_policies`) @@ -437,6 +460,37 @@ export default function (providerContext: FtrProviderContext) { expect(packagePolicy.inputs[0].streams[0].vars.stream_var_secret.value.id).eql(streamVarId); }); + it('Should return output secrets if policy uses output with secrets', async () => { + const outputWithSecret = await createOutputWithSecret(); + + const { body: agentPolicyResponse } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `Test policy ${uuidv4()}`, + namespace: 'default', + data_output_id: outputWithSecret.id, + monitoring_output_id: outputWithSecret.id, + }) + .expect(200); + + const fullAgentPolicy = await getFullAgentPolicyById(agentPolicyResponse.item.id); + + const passwordSecretId = outputWithSecret!.secrets?.password?.id; + + expect(fullAgentPolicy.secret_references).to.eql([{ id: passwordSecretId }]); + + const output = Object.entries(fullAgentPolicy.outputs)[0][1]; + // @ts-expect-error + expect(output.secrets.password.id).to.eql(passwordSecretId); + + // delete output with secret + await supertest + .delete(`/api/fleet/outputs/${outputWithSecret.id}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + }); + it('should have correctly created the secrets', async () => { const searchRes = await getSecrets([packageVarId, inputVarId, streamVarId]); @@ -465,6 +519,7 @@ export default function (providerContext: FtrProviderContext) { it('should allow secret values to be updated (single policy update API)', async () => { const updatedPolicy = createdPolicyToUpdatePolicy(createdPackagePolicy); updatedPolicy.vars.package_var_secret.value = 'new_package_secret_val'; + const updateRes = await supertest .put(`/api/fleet/package_policies/${createdPackagePolicyId}`) .set('kbn-xsrf', 'xxxx') diff --git a/x-pack/test/fleet_api_integration/config.base.ts b/x-pack/test/fleet_api_integration/config.base.ts index 3e4b35988efba5..9649747d966327 100644 --- a/x-pack/test/fleet_api_integration/config.base.ts +++ b/x-pack/test/fleet_api_integration/config.base.ts @@ -71,7 +71,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { )}`, `--xpack.securitySolution.enableExperimental=${JSON.stringify(['endpointRbacEnabled'])}`, `--xpack.fleet.enableExperimental=${JSON.stringify([ - 'secretsStorage', + 'outputSecretsStorage', 'agentTamperProtectionEnabled', ])}`, `--logging.loggers=${JSON.stringify([