From dac8a8567a7168a061b6d1dc6c27478a709e4724 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 25 Oct 2023 15:35:26 +0100 Subject: [PATCH 01/18] add node-argon2 --- package.json | 1 + yarn.lock | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/package.json b/package.json index e5213d00d20d46..1f8327e000e808 100644 --- a/package.json +++ b/package.json @@ -851,6 +851,7 @@ "ansi-regex": "^5.0.1", "antlr4ts": "^0.5.0-alpha.3", "archiver": "^5.3.1", + "argon2": "0.31.1", "async": "^3.2.3", "aws4": "^1.12.0", "axios": "^1.4.0", diff --git a/yarn.lock b/yarn.lock index 28df3eac7c3f25..b29c6c3de4c0e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6392,6 +6392,21 @@ resolved "https://registry.yarnpkg.com/@mapbox/mapbox-gl-supported/-/mapbox-gl-supported-2.0.1.tgz#c15367178d8bfe4765e6b47b542fe821ce259c7b" integrity sha512-HP6XvfNIzfoMVfyGjBckjiAOQK9WfX0ywdLubuPMPv+Vqf5fj0uCbgBQYpiqcWZT6cbyyRnTSXDheT1ugvF6UQ== +"@mapbox/node-pre-gyp@^1.0.11": + version "1.0.11" + resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa" + integrity sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ== + dependencies: + detect-libc "^2.0.0" + https-proxy-agent "^5.0.0" + make-dir "^3.1.0" + node-fetch "^2.6.7" + nopt "^5.0.0" + npmlog "^5.0.1" + rimraf "^3.0.2" + semver "^7.3.5" + tar "^6.1.11" + "@mapbox/point-geometry@0.1.0", "@mapbox/point-geometry@^0.1.0", "@mapbox/point-geometry@~0.1.0": version "0.1.0" resolved "https://registry.yarnpkg.com/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz#8a83f9335c7860effa2eeeca254332aa0aeed8f2" @@ -7040,6 +7055,11 @@ node-addon-api "^3.2.1" node-gyp-build "^4.3.0" +"@phc/format@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@phc/format/-/format-1.0.0.tgz#b5627003b3216dc4362125b13f48a4daa76680e4" + integrity sha512-m7X9U6BG2+J+R1lSOdCiITLLrxm+cWlNI3HUFA92oLO77ObGNzaKdh8pMLqdZcshtkKuV84olNNXDfMc4FezBQ== + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" @@ -11031,6 +11051,15 @@ arg@^5.0.1, arg@^5.0.2: resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== +argon2@0.31.1: + version "0.31.1" + resolved "https://registry.yarnpkg.com/argon2/-/argon2-0.31.1.tgz#c8560bc76b12681afea13e28f3417aaa4b84c466" + integrity sha512-ik2xnJrLXazya7m4Nz1XfBSRjXj8Koq8qF9PsQC8059p20ifWc9zx/hgU3ItZh/3TnwXkv0RbhvjodPkmFf0bg== + dependencies: + "@mapbox/node-pre-gyp" "^1.0.11" + "@phc/format" "^1.0.0" + node-addon-api "^7.0.0" + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -22657,6 +22686,11 @@ node-addon-api@^6.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== +node-addon-api@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-7.0.0.tgz#8136add2f510997b3b94814f4af1cce0b0e3962e" + integrity sha512-vgbBJTS4m5/KkE16t5Ly0WW9hz46swAstv0hYYwMtbG7AznRhNyfLRe8HZAiWIpcHzoO7HxhLuBQj9rJ/Ho0ZA== + node-cache@^5.1.0: version "5.1.2" resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d" From d70b9dd3037ac08ab0621557b4dcfaf14838bb7c Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 31 Oct 2023 17:03:07 +0000 Subject: [PATCH 02/18] add hash to saved object mapping --- x-pack/plugins/fleet/server/saved_objects/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 7458fad2459b6b..de5b6c1a881b6b 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -254,6 +254,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ dynamic: false, properties: { id: { type: 'keyword' }, + hash: { type: 'keyword', index: false }, }, }, ssl: { @@ -263,6 +264,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ dynamic: false, properties: { id: { type: 'keyword' }, + hash: { type: 'keyword', index: false }, }, }, }, From cf99bed4534b95198bd8ceeaff58f7e858243127 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 31 Oct 2023 17:03:24 +0000 Subject: [PATCH 03/18] add hash to output type --- .../fleet/common/types/models/output.ts | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 5bed131be8d56b..1813da29261cbd 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -23,7 +23,12 @@ export type KafkaPartitionType = typeof kafkaPartitionType; export type KafkaTopicWhenType = typeof kafkaTopicWhenType; export type KafkaAcknowledgeReliabilityLevel = typeof kafkaAcknowledgeReliabilityLevel; export type KafkaVerificationMode = typeof kafkaVerificationModes; - +export type OutputSecret = + | string + | { + id: string; + hash?: string; + }; interface NewBaseOutput { is_default: boolean; is_default_monitoring: boolean; @@ -45,11 +50,7 @@ interface NewBaseOutput { allow_edit?: string[]; secrets?: { ssl?: { - key?: - | string - | { - id: string; - }; + key?: OutputSecret; }; }; } @@ -122,17 +123,9 @@ export interface KafkaOutput extends NewBaseOutput { broker_timeout?: number; required_acks?: ValueOf; secrets?: { - password?: - | string - | { - id: string; - }; + password?: OutputSecret; ssl?: { - key?: - | string - | { - id: string; - }; + key?: OutputSecret; }; }; } From 38d4e4a82d0af65f31b96b4e6b90f290d1a88e73 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 31 Oct 2023 17:05:06 +0000 Subject: [PATCH 04/18] use argon2 to diff preconfigured secrets --- .../plugins/fleet/server/services/output.ts | 14 +- .../services/preconfiguration/outputs.ts | 147 +++++++++++++++--- .../plugins/fleet/server/services/secrets.ts | 23 ++- 3 files changed, 158 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index efbc35806e925b..286e3ae0277b64 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -418,7 +418,12 @@ class OutputService { soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, output: NewOutput, - options?: { id?: string; fromPreconfiguration?: boolean; overwrite?: boolean } + options?: { + id?: string; + fromPreconfiguration?: boolean; + overwrite?: boolean; + secretHashes?: Record; + } ): Promise { const data: OutputSOAttributes = { ...omit(output, ['ssl', 'secrets']) }; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); @@ -544,6 +549,7 @@ class OutputService { const { output: outputWithSecrets } = await extractAndWriteOutputSecrets({ output, esClient, + secretHashes: output.is_preconfigured ? options?.secretHashes : undefined, }); if (outputWithSecrets.secrets) data.secrets = outputWithSecrets.secrets; @@ -705,7 +711,10 @@ class OutputService { esClient: ElasticsearchClient, id: string, data: Partial, - { fromPreconfiguration = false }: { fromPreconfiguration: boolean } = { + { + fromPreconfiguration = false, + secretHashes, + }: { fromPreconfiguration: boolean; secretHashes?: Record } = { fromPreconfiguration: false, } ) { @@ -728,6 +737,7 @@ class OutputService { oldOutput: originalOutput, outputUpdate: data, esClient, + secretHashes: data.is_preconfigured ? secretHashes : undefined, }); updateData.secrets = secretsRes.outputUpdate.secrets; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 5bc7c452b481e0..c499082666df6b 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -8,8 +8,16 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { isEqual } from 'lodash'; import { safeDump } from 'js-yaml'; +import argon2 from 'argon2'; -import type { PreconfiguredOutput, Output, NewOutput } from '../../../common/types'; +import type { + PreconfiguredOutput, + Output, + NewOutput, + OutputSecret, + KafkaOutput, + NewLogstashOutput, +} from '../../../common/types'; import { normalizeHostsForAgents } from '../../../common/services'; import type { FleetConfigType } from '../../config'; import { DEFAULT_OUTPUT_ID, DEFAULT_OUTPUT } from '../../constants'; @@ -99,25 +107,70 @@ export async function createOrUpdatePreconfiguredOutputs( } const isUpdateWithNewData = - existingOutput && isPreconfiguredOutputDifferentFromCurrent(existingOutput, data); - - if (isCreate) { - logger.debug(`Creating output ${output.id}`); - await outputService.create(soClient, esClient, data, { id, fromPreconfiguration: true }); - } else if (isUpdateWithNewData) { - logger.debug(`Updating output ${output.id}`); - await outputService.update(soClient, esClient, id, data, { fromPreconfiguration: true }); - // Bump revision of all policies using that output - if (outputData.is_default || outputData.is_default_monitoring) { - await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); - } else { - await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id); + existingOutput && (await isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)); + + if (isCreate || isUpdateWithNewData) { + const secretHashes = await hashSecrets(output); + + if (isCreate) { + logger.debug(`Creating output ${output.id}`); + await outputService.create(soClient, esClient, data, { + id, + fromPreconfiguration: true, + secretHashes, + }); + } else if (isUpdateWithNewData) { + logger.debug(`Updating output ${output.id}`); + await outputService.update(soClient, esClient, id, data, { + fromPreconfiguration: true, + secretHashes, + }); + // Bump revision of all policies using that output + if (outputData.is_default || outputData.is_default_monitoring) { + await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); + } else { + await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id); + } } } }) ); } +async function hashSecrets(output: PreconfiguredOutput) { + if (output.type === 'kafka') { + const kafkaOutput = output as KafkaOutput; + if (typeof kafkaOutput.secrets?.password === 'string') { + const password = await argon2.hash(kafkaOutput.secrets?.password); + return { + password, + }; + } + if (typeof kafkaOutput.secrets?.ssl?.key === 'string') { + const key = await argon2.hash(kafkaOutput.secrets?.ssl?.key); + return { + ssl: { + key, + }, + }; + } + } + if (output.type === 'logstash') { + const logstashOutput = output as NewLogstashOutput; + + if (typeof logstashOutput.secrets?.ssl?.key === 'string') { + const key = await argon2.hash(logstashOutput.secrets?.ssl?.key); + return { + ssl: { + key, + }, + }; + } + } + + return undefined; +} + export async function cleanPreconfiguredOutputs( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, @@ -165,15 +218,56 @@ export async function cleanPreconfiguredOutputs( } } -function isPreconfiguredOutputDifferentFromCurrent( +const hasHash = (secret?: OutputSecret): secret is { id: string; hash: string } => { + return !!secret && typeof secret !== 'string' && !!secret.hash; +}; + +async function isSecretDifferent( + preconfiguredValue: OutputSecret | undefined, + existingSecret: OutputSecret | undefined +): Promise { + if (!existingSecret && preconfiguredValue) { + return true; + } + + if (!preconfiguredValue && existingSecret) { + return true; + } + + if (!preconfiguredValue && !existingSecret) { + return false; + } + + if (hasHash(existingSecret) && typeof preconfiguredValue === 'string') { + // verifying the has tells us if the value has changed + const hashIsVerified = await argon2.verify(existingSecret.hash, preconfiguredValue!); + + return !hashIsVerified; + } else { + // if there is no hash then the safest thing to do is assume the value has changed + return true; + } +} + +async function isPreconfiguredOutputDifferentFromCurrent( existingOutput: Output, preconfiguredOutput: Partial -): boolean { - const kafkaFieldsAreDifferent = (): boolean => { +): Promise { + const kafkaFieldsAreDifferent = async (): Promise => { if (existingOutput.type !== 'kafka' || preconfiguredOutput.type !== 'kafka') { return false; } + const passwordHashIsDifferent = await isSecretDifferent( + preconfiguredOutput.secrets?.password, + existingOutput.secrets?.password + ); + + const sslKeyHashIsDifferent = await isSecretDifferent( + preconfiguredOutput.secrets?.ssl?.key, + existingOutput.secrets?.ssl?.key + ); + return ( isDifferent(existingOutput.client_id, preconfiguredOutput.client_id) || isDifferent(existingOutput.version, preconfiguredOutput.version) || @@ -193,8 +287,22 @@ function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.headers, preconfiguredOutput.headers) || isDifferent(existingOutput.timeout, preconfiguredOutput.timeout) || isDifferent(existingOutput.broker_timeout, preconfiguredOutput.broker_timeout) || - isDifferent(existingOutput.required_acks, preconfiguredOutput.required_acks) + isDifferent(existingOutput.required_acks, preconfiguredOutput.required_acks) || + passwordHashIsDifferent || + sslKeyHashIsDifferent + ); + }; + + const logstashFieldsAreDifferent = async (): Promise => { + if (existingOutput.type !== 'logstash' || preconfiguredOutput.type !== 'logstash') { + return false; + } + const sslKeyHashIsDifferent = await isSecretDifferent( + preconfiguredOutput.secrets?.ssl?.key, + existingOutput.secrets?.ssl?.key ); + + return sslKeyHashIsDifferent; }; return ( @@ -221,6 +329,7 @@ function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.config_yaml, preconfiguredOutput.config_yaml) || isDifferent(existingOutput.proxy_id, preconfiguredOutput.proxy_id) || isDifferent(existingOutput.allow_edit ?? [], preconfiguredOutput.allow_edit ?? []) || - kafkaFieldsAreDifferent() + (await kafkaFieldsAreDifferent()) || + (await logstashFieldsAreDifferent()) ); } diff --git a/x-pack/plugins/fleet/server/services/secrets.ts b/x-pack/plugins/fleet/server/services/secrets.ts index 36a88b4a7a4c1a..baabdf55e793d2 100644 --- a/x-pack/plugins/fleet/server/services/secrets.ts +++ b/x-pack/plugins/fleet/server/services/secrets.ts @@ -7,7 +7,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; -import { keyBy } from 'lodash'; +import { get, keyBy } from 'lodash'; import { set } from '@kbn/safer-lodash-set'; import type { KafkaOutput, Output, OutputSecretPath } from '../../common/types'; @@ -247,8 +247,9 @@ export async function extractAndWriteSecrets(opts: { export async function extractAndWriteOutputSecrets(opts: { output: NewOutput; esClient: ElasticsearchClient; + secretHashes?: Record; }): Promise<{ output: NewOutput; secretReferences: PolicySecretReference[] }> { - const { output, esClient } = opts; + const { output, esClient, secretHashes = {} } = opts; const secretPaths = getOutputSecretPaths(output.type, output).filter( (path) => typeof path.value === 'string' @@ -265,7 +266,12 @@ export async function extractAndWriteOutputSecrets(opts: { const outputWithSecretRefs = JSON.parse(JSON.stringify(output)); secretPaths.forEach((secretPath, i) => { - set(outputWithSecretRefs, secretPath.path, { id: secrets[i].id }); + const pathWithoutPrefix = secretPath.path.replace('secrets.', ''); + const maybeHash = get(secretHashes, pathWithoutPrefix); + set(outputWithSecretRefs, secretPath.path, { + id: secrets[i].id, + ...(typeof maybeHash === 'string' && { hash: maybeHash }), + }); }); return { @@ -399,12 +405,13 @@ export async function extractAndUpdateOutputSecrets(opts: { oldOutput: Output; outputUpdate: Partial; esClient: ElasticsearchClient; + secretHashes?: Record; }): Promise<{ outputUpdate: Partial; secretReferences: PolicySecretReference[]; secretsToDelete: PolicySecretReference[]; }> { - const { oldOutput, outputUpdate, esClient } = opts; + const { oldOutput, outputUpdate, esClient, secretHashes } = opts; const outputType = outputUpdate.type || oldOutput.type; const oldSecretPaths = getOutputSecretPaths(outputType, oldOutput); const updatedSecretPaths = getOutputSecretPaths(outputType, outputUpdate); @@ -425,7 +432,13 @@ export async function extractAndUpdateOutputSecrets(opts: { const outputWithSecretRefs = JSON.parse(JSON.stringify(outputUpdate)); toCreate.forEach((secretPath, i) => { - set(outputWithSecretRefs, secretPath.path, { id: createdSecrets[i].id }); + const pathWithoutPrefix = secretPath.path.replace('secrets.', ''); + const maybeHash = get(secretHashes, pathWithoutPrefix); + + set(outputWithSecretRefs, secretPath.path, { + id: createdSecrets[i].id, + ...(typeof maybeHash === 'string' && { hash: maybeHash }), + }); }); const secretReferences = [ From f119d4386910b3491cb84e8944081a39024ca6e8 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 31 Oct 2023 17:35:17 +0000 Subject: [PATCH 05/18] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../kbn-check-mappings-update-cli/current_mappings.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index dea741b930a7d3..7f9fab872022f3 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1825,6 +1825,10 @@ "properties": { "id": { "type": "keyword" + }, + "hash": { + "type": "keyword", + "index": false } } }, @@ -1836,6 +1840,10 @@ "properties": { "id": { "type": "keyword" + }, + "hash": { + "type": "keyword", + "index": false } } } From 9e3d6966116e8c047ce20da82ce985fda05ab5d7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:06:52 +0000 Subject: [PATCH 06/18] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 6de7f6e90de7ce..b49c98db4dcb49 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "3982d6296373111467e839a0768d3e1c4d0ebc61", + "ingest-outputs": "1214b813a0f4772fb8ad824aa136dfc44ee42fcf", "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", From 32d3d0831b67e436a9775f1e46b3d923dd5759a3 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:34:07 +0000 Subject: [PATCH 07/18] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index dd736740a9377d..5aab1412e92731 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "8546f1123ec30dcbd6f238f72729c5f1656a4d9b", + "ingest-outputs": "9ddce96e9af2ea96a1e01483a13714654105c9a7", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", From b8b5b7caf8568fada9ac65580ebd174d8963db64 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Wed, 15 Nov 2023 17:21:43 +0000 Subject: [PATCH 08/18] Configure argon2 with recommended params --- .../server/services/preconfiguration/outputs.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index c499082666df6b..01053b913c8123 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -137,17 +137,26 @@ export async function createOrUpdatePreconfiguredOutputs( ); } +async function hash(str) { + argon2.hash(str, { + type: argon2.argon2id, + memoryCost: 19456, + timeCost: 2, + parallelism: 1, + }); +} + async function hashSecrets(output: PreconfiguredOutput) { if (output.type === 'kafka') { const kafkaOutput = output as KafkaOutput; if (typeof kafkaOutput.secrets?.password === 'string') { - const password = await argon2.hash(kafkaOutput.secrets?.password); + const password = await hash(kafkaOutput.secrets?.password); return { password, }; } if (typeof kafkaOutput.secrets?.ssl?.key === 'string') { - const key = await argon2.hash(kafkaOutput.secrets?.ssl?.key); + const key = await hash(kafkaOutput.secrets?.ssl?.key); return { ssl: { key, @@ -159,7 +168,7 @@ async function hashSecrets(output: PreconfiguredOutput) { const logstashOutput = output as NewLogstashOutput; if (typeof logstashOutput.secrets?.ssl?.key === 'string') { - const key = await argon2.hash(logstashOutput.secrets?.ssl?.key); + const key = await hash(logstashOutput.secrets?.ssl?.key); return { ssl: { key, From 85aaaf2abc94b4b30652036ad99a280605c2e333 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Wed, 15 Nov 2023 17:21:59 +0000 Subject: [PATCH 09/18] Add and fix preconfigured output tests --- .../services/preconfiguration/outputs.test.ts | 126 ++++++++++++++++-- 1 file changed, 114 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index 1023e1bdb7b567..e5d307c06f53ef 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -74,8 +74,24 @@ describe('output preconfiguration', () => { hosts: ['kafka.co:80'], is_preconfigured: true, }, + { + id: 'existing-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Secret Output 1', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }, ]; }); + spyAgentPolicyServicBumpAllAgentPoliciesForOutput.mockClear(); }); it('should generate a preconfigured output if elasticsearch.hosts is set in the config', async () => { @@ -104,7 +120,7 @@ describe('output preconfiguration', () => { `); }); - it('should create preconfigured output that does not exists', async () => { + it('should create preconfigured output that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ @@ -123,7 +139,7 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should create preconfigured kafka output that does not exists', async () => { + it('should create preconfigured kafka output that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ @@ -142,7 +158,7 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should create a preconfigured output with ca_trusted_fingerprint that does not exists', async () => { + it('should create a preconfigured output with ca_trusted_fingerprint that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ @@ -170,7 +186,7 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should create preconfigured logstash output that does not exist', async () => { + it('should create a preconfigured logstash output that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ @@ -190,7 +206,43 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should set default hosts if hosts is not set output that does not exists', async () => { + it('should create a preconfigured output with secrets that does not exist', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'non-existing-output-with-secrets-1', + name: 'Secret Output 2', + type: 'elasticsearch', + is_default: false, + is_default_monitoring: false, + hosts: ['http://test.fr'], + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }, + ]); + + expect(mockedOutputService.create).toBeCalled(); + expect(mockedOutputService.create).toBeCalledWith( + expect.anything(), + expect.anything(), + expect.objectContaining({ + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }), + expect.anything() + ); + expect(mockedOutputService.update).not.toBeCalled(); + // expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + }); + + it('should set default hosts if hosts is not set output that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ @@ -268,6 +320,30 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); + it('should update ouput if secret hash has changed', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Secret Output 1', + type: 'elasticsearch', + secrets: { + ssl: { + key: 'secretKey2', // field that changed + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + }); + it('should update output if preconfigured kafka output exists and changed', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -279,16 +355,16 @@ describe('output preconfiguration', () => { is_default_monitoring: false, name: 'Kafka Output 1', type: 'kafka', - hosts: ['kafka.co:8080'], + hosts: ['kafka.co:80'], }, ]); expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).toBeCalled(); - expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should not update output if preconfigured output exists and did not changed', async () => { + it('should not update output if preconfigured output exists and did not change', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); @@ -299,13 +375,13 @@ describe('output preconfiguration', () => { is_default_monitoring: false, name: 'Output 1', type: 'elasticsearch', - hosts: ['http://newhostichanged.co:9201'], // field that changed + hosts: ['http://es.co:80'], }, ]); expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).toBeCalled(); - expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); it('should not update output if preconfigured kafka output exists and did not change', async () => { @@ -328,6 +404,32 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); + it('should not update a preconfigured output with secrets if it exists and did not change', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Secret Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_preconfigured: true, + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + }); + const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [ { name: 'no changes', From 49fdb0316495b0755e94ca2b7ac755dad81b5d15 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Thu, 16 Nov 2023 10:11:04 +0000 Subject: [PATCH 10/18] Add type to hash param --- .../plugins/fleet/server/services/preconfiguration/outputs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 01053b913c8123..1918595ba5808b 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -137,7 +137,7 @@ export async function createOrUpdatePreconfiguredOutputs( ); } -async function hash(str) { +async function hash(str: string) { argon2.hash(str, { type: argon2.argon2id, memoryCost: 19456, From 3429e0f00f3d3716b34fc87bca687cad162e4fee Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Thu, 16 Nov 2023 12:14:13 +0000 Subject: [PATCH 11/18] Fix typo --- .../plugins/fleet/server/services/preconfiguration/outputs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 1918595ba5808b..3a9662354d14f9 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -138,7 +138,7 @@ export async function createOrUpdatePreconfiguredOutputs( } async function hash(str: string) { - argon2.hash(str, { + return argon2.hash(str, { type: argon2.argon2id, memoryCost: 19456, timeCost: 2, From 279f3f7ef269bde6d70d08ef3b8ef9b25c9dfebe Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Tue, 21 Nov 2023 11:19:20 +0000 Subject: [PATCH 12/18] Fix and add unit tests --- .../services/preconfiguration/outputs.test.ts | 173 ++++++++++++++---- .../services/preconfiguration/outputs.ts | 2 +- 2 files changed, 137 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index e5d307c06f53ef..694a0010ad2753 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -17,6 +17,7 @@ import { createOrUpdatePreconfiguredOutputs, cleanPreconfiguredOutputs, getPreconfiguredOutputFromConfig, + hash, } from './outputs'; jest.mock('../agent_policy_update'); @@ -46,16 +47,18 @@ const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn( ); describe('output preconfiguration', () => { - beforeEach(() => { + beforeEach(async () => { mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.delete.mockReset(); mockedOutputService.getDefaultDataOutputId.mockReset(); mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']); + const keyHash = await hash('secretKey'); + const passwordHash = await hash('secretPassword'); mockedOutputService.bulkGet.mockImplementation(async (soClient, id): Promise => { return [ { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -75,17 +78,40 @@ describe('output preconfiguration', () => { is_preconfigured: true, }, { - id: 'existing-output-with-secrets-1', + id: 'existing-logstash-output-with-secrets-1', is_default: false, is_default_monitoring: false, - name: 'Secret Output 1', - // @ts-ignore - type: 'elasticsearch', - hosts: ['http://es.co:80'], + name: 'Logstash Output With Secrets 1', + type: 'logstash', + hosts: ['test:4343'], is_preconfigured: true, secrets: { ssl: { - key: 'secretKey', + key: { + id: '123', + hash: keyHash, + }, + }, + }, + }, + { + id: 'existing-kafka-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Kafka Output With Secrets 1', + type: 'kafka', + hosts: ['kafka.co:80'], + is_preconfigured: true, + secrets: { + password: { + id: '456', + hash: passwordHash, + }, + ssl: { + key: { + id: '789', + hash: keyHash, + }, }, }, }, @@ -206,17 +232,16 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should create a preconfigured output with secrets that does not exist', async () => { + it('should create a preconfigured logstash output with secrets that does not exist', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'non-existing-output-with-secrets-1', - name: 'Secret Output 2', - type: 'elasticsearch', + id: 'non-existing-logstash-output-with-secrets-1', + name: 'Logstash Output With Secrets 2', + type: 'logstash', is_default: false, is_default_monitoring: false, - hosts: ['http://test.fr'], secrets: { ssl: { key: 'secretKey', @@ -239,7 +264,31 @@ describe('output preconfiguration', () => { expect.anything() ); expect(mockedOutputService.update).not.toBeCalled(); - // expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + }); + + it('should create a preconfigured kafka output with secrets that does not exist', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'non-existing-kafka-output-with-secrets-1', + name: 'Kafka Output With Secrets 2', + type: 'kafka', + is_default: false, + is_default_monitoring: false, + secrets: { + password: 'secretPassword', + ssl: { + key: 'secretKey', + }, + }, + }, + ]); + + expect(mockedOutputService.create).toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); it('should set default hosts if hosts is not set output that does not exist', async () => { @@ -265,7 +314,7 @@ describe('output preconfiguration', () => { soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); mockedOutputService.bulkGet.mockResolvedValue([ { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -277,7 +326,7 @@ describe('output preconfiguration', () => { ]); await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -291,7 +340,7 @@ describe('output preconfiguration', () => { expect(mockedOutputService.update).toBeCalledWith( expect.anything(), expect.anything(), - 'existing-output-1', + 'existing-es-output-1', expect.objectContaining({ is_preconfigured: true, }), @@ -306,7 +355,7 @@ describe('output preconfiguration', () => { soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -320,17 +369,17 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); - it('should update ouput if secret hash has changed', async () => { + it('should update output if a preconfigured logstash ouput with secrets exists and has changed', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'existing-output-with-secrets-1', + id: 'existing-logstash-output-with-secrets-1', is_default: false, is_default_monitoring: false, - name: 'Secret Output 1', - type: 'elasticsearch', + name: 'Logstash Output With Secrets 1', + type: 'logstash', secrets: { ssl: { key: 'secretKey2', // field that changed @@ -355,13 +404,38 @@ describe('output preconfiguration', () => { is_default_monitoring: false, name: 'Kafka Output 1', type: 'kafka', - hosts: ['kafka.co:80'], + hosts: ['kafka.co:8080'], // field that changed }, ]); expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).not.toBeCalled(); - expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + }); + + it('should update ouput if a preconfigured kafka with secrets exists and has changed', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-kafka-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Kafka Output With Secrets 1', + type: 'kafka', + secrets: { + password: 'secretPassword2', // field that changed + ssl: { + key: 'secretKey2', + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); it('should not update output if preconfigured output exists and did not change', async () => { @@ -370,7 +444,7 @@ describe('output preconfiguration', () => { soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -395,29 +469,54 @@ describe('output preconfiguration', () => { is_default_monitoring: false, name: 'Kafka Output 1', type: 'kafka', - hosts: ['kafka.co:8080'], + hosts: ['kafka.co:80'], }, ]); expect(mockedOutputService.create).not.toBeCalled(); - expect(mockedOutputService.update).toBeCalled(); - expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); - it('should not update a preconfigured output with secrets if it exists and did not change', async () => { + it('should not update output if a preconfigured logstash output with secrets exists and did not change', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ { - id: 'existing-output-with-secrets-1', + id: 'existing-logstash-output-with-secrets-1', is_default: false, is_default_monitoring: false, - name: 'Secret Output 1', - type: 'elasticsearch', - hosts: ['http://es.co:80'], - is_preconfigured: true, + name: 'Logstash Output With Secrets 1', + type: 'logstash', + hosts: ['test:4343'], + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).not.toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); + }); + + it('should not update output if a preconfigured kafka output with secrets exists and did not change', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-kafka-output-with-secrets-1', + is_default: false, + is_default_monitoring: false, + name: 'Kafka Output With Secrets 1', + type: 'kafka', + hosts: ['kafka.co:80'], secrets: { + password: 'secretPassword', ssl: { key: 'secretKey', }, @@ -434,7 +533,7 @@ describe('output preconfiguration', () => { { name: 'no changes', data: { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', @@ -445,7 +544,7 @@ describe('output preconfiguration', () => { { name: 'hosts without port', data: { - id: 'existing-output-1', + id: 'existing-es-output-1', is_default: false, is_default_monitoring: false, name: 'Output 1', diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 3a9662354d14f9..76414e1700ba2d 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -137,7 +137,7 @@ export async function createOrUpdatePreconfiguredOutputs( ); } -async function hash(str: string) { +export async function hash(str: string) { return argon2.hash(str, { type: argon2.argon2id, memoryCost: 19456, From 99f10d9bbe3d3c52dc2aa58a705e6297ba3a2202 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Tue, 21 Nov 2023 11:19:56 +0000 Subject: [PATCH 13/18] Exclude hash from SO mappings --- .../kbn-check-mappings-update-cli/current_mappings.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 023a26e787bfa3..af8fae6214ff8d 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1829,10 +1829,6 @@ "properties": { "id": { "type": "keyword" - }, - "hash": { - "type": "keyword", - "index": false } } }, @@ -1844,10 +1840,6 @@ "properties": { "id": { "type": "keyword" - }, - "hash": { - "type": "keyword", - "index": false } } } From aa387ae75c2307b05c30bd83533a8e2edadeb268 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:45:00 +0000 Subject: [PATCH 14/18] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../kbn-check-mappings-update-cli/current_mappings.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 603f0efd54a879..91b110df76100f 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1829,6 +1829,10 @@ "properties": { "id": { "type": "keyword" + }, + "hash": { + "type": "keyword", + "index": false } } }, @@ -1840,6 +1844,10 @@ "properties": { "id": { "type": "keyword" + }, + "hash": { + "type": "keyword", + "index": false } } } From 8509230b64ee8e9229cef90da9b64ad573847c80 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Wed, 22 Nov 2023 08:50:10 +0000 Subject: [PATCH 15/18] Add extra unit tests --- .../services/preconfiguration/outputs.test.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index 694a0010ad2753..a0dd3fd5e7f5d0 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -94,6 +94,20 @@ describe('output preconfiguration', () => { }, }, }, + { + id: 'existing-logstash-output-with-secrets-2', + is_default: false, + is_default_monitoring: false, + name: 'Logstash Output With Secrets 2', + type: 'logstash', + hosts: ['test:4343'], + is_preconfigured: true, + secrets: { + ssl: { + key: 'secretKey', + }, + }, + }, { id: 'existing-kafka-output-with-secrets-1', is_default: false, @@ -115,6 +129,21 @@ describe('output preconfiguration', () => { }, }, }, + { + id: 'existing-kafka-output-with-secrets-2', + is_default: false, + is_default_monitoring: false, + name: 'Kafka Output With Secrets 2', + type: 'kafka', + hosts: ['kafka.co:80'], + is_preconfigured: true, + secrets: { + password: 'secretPassword', + ssl: { + key: 'secretKey', + }, + }, + }, ]; }); spyAgentPolicyServicBumpAllAgentPoliciesForOutput.mockClear(); @@ -529,6 +558,57 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled(); }); + it('should update output if a preconfigured logstash output with plain value secrets exists and did not change', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-logstash-output-with-secrets-2', + is_default: false, + is_default_monitoring: false, + name: 'Logstash Output With Secrets 2', + type: 'logstash', + hosts: ['test:4343'], + secrets: { + ssl: { + key: 'secretKey', // no change + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + }); + + it('should update output if a preconfigured kafka output with plain value secrets exists and did not change', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-kafka-output-with-secrets-2', + is_default: false, + is_default_monitoring: false, + name: 'Kafka Output With Secrets 2', + type: 'kafka', + hosts: ['kafka.co:80'], + secrets: { + password: 'secretPassword', // no change + ssl: { + key: 'secretKey', // no change + }, + }, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + }); + const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [ { name: 'no changes', From 38e12f06e65c2f193e5a1dda105701ba14f8cd55 Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Wed, 22 Nov 2023 08:50:26 +0000 Subject: [PATCH 16/18] Remove SO mappings --- .../kbn-check-mappings-update-cli/current_mappings.json | 8 -------- x-pack/plugins/fleet/server/saved_objects/index.ts | 2 -- 2 files changed, 10 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 91b110df76100f..603f0efd54a879 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1829,10 +1829,6 @@ "properties": { "id": { "type": "keyword" - }, - "hash": { - "type": "keyword", - "index": false } } }, @@ -1844,10 +1840,6 @@ "properties": { "id": { "type": "keyword" - }, - "hash": { - "type": "keyword", - "index": false } } } diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index dfaa5fbc5445a4..b87310e850c81f 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -257,7 +257,6 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ dynamic: false, properties: { id: { type: 'keyword' }, - hash: { type: 'keyword', index: false }, }, }, ssl: { @@ -267,7 +266,6 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ dynamic: false, properties: { id: { type: 'keyword' }, - hash: { type: 'keyword', index: false }, }, }, }, From b2921ce0bd32e07a99ddfa72768106f2165bf0bb Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 22 Nov 2023 09:25:22 +0000 Subject: [PATCH 17/18] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 0b19f0ff81e215..2c7d540132139d 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "9ddce96e9af2ea96a1e01483a13714654105c9a7", + "ingest-outputs": "8546f1123ec30dcbd6f238f72729c5f1656a4d9b", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", From 7707de5208d62ec32ad621466f3e087df97c78cd Mon Sep 17 00:00:00 2001 From: jillguyonnet Date: Wed, 22 Nov 2023 16:10:23 +0000 Subject: [PATCH 18/18] Improve log messages --- .../plugins/fleet/server/services/preconfiguration/outputs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 76414e1700ba2d..07636dd1266c0a 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -113,14 +113,14 @@ export async function createOrUpdatePreconfiguredOutputs( const secretHashes = await hashSecrets(output); if (isCreate) { - logger.debug(`Creating output ${output.id}`); + logger.debug(`Creating preconfigured output ${output.id}`); await outputService.create(soClient, esClient, data, { id, fromPreconfiguration: true, secretHashes, }); } else if (isUpdateWithNewData) { - logger.debug(`Updating output ${output.id}`); + logger.debug(`Updating preconfigured output ${output.id}`); await outputService.update(soClient, esClient, id, data, { fromPreconfiguration: true, secretHashes,