[Fleet] Output Secrets Backend (#169221)

## Summary Had to recreate this after pinging the whole of Kibana accidentally on the last one :D Part of #157458 Adds the ability to sepcify secrets in outputs. Currently the following secrets are supported: - kafka output SSL key - kafka output password - logstash output SSL key The behaviour is as follows: - on create, secrets are created and the plain string is replaced with a secret reference on the output saved object - on update, if a secret is updated, the old secret is deleted and a new one is created, the new secret ref is added to the output - on delete, all secrets are deleted - behaviour is behind a feature flag as flee tserver does not support these yet Secrets are only enabled if a fleet server of 8.10.0 or greater is connected. Integration tests added for all scenarios. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
elastic · Oct 24, 2023 · ce24d1a · ce24d1a
1 parent e027f7a
commit ce24d1a
Show file tree

Hide file tree

Showing 22 changed files with 852 additions and 25 deletions.
diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1816,6 +1816,32 @@
       },
       "channel_buffer_size": {
         "type": "integer"
+      },
+      "secrets": {
+        "dynamic": false,
+        "properties": {
+          "password": {
+            "dynamic": false,
+            "properties": {
+              "id": {
+                "type": "keyword"
+              }
+            }
+          },
+          "ssl": {
+            "dynamic": false,
+            "properties": {
+              "key": {
+                "dynamic": false,
+                "properties": {
+                  "id": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          }
+        }
       }
     }
   },

diff --git a/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts
@@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
-        "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0",
+        "ingest-outputs": "3982d6296373111467e839a0768d3e1c4d0ebc61",
         "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea",
         "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts
@@ -23,6 +23,7 @@ export const allowedExperimentalValues = Object.freeze<Record<string, boolean>>(
   agentTamperProtectionEnabled: true,
   secretsStorage: true,
   kafkaOutput: true,
+  outputSecretsStorage: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -8121,6 +8121,22 @@
           },
           "required_acks": {
             "type": "number"
+          },
+          "secrets": {
+            "type": "object",
+            "properties": {
+              "password": {
+                "type": "string"
+              },
+              "ssl": {
+                "type": "object",
+                "properties": {
+                  "key": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
           }
         },
         "required": [
@@ -8216,6 +8232,19 @@
                 "type": "boolean"
               }
             }
+          },
+          "secrets": {
+            "type": "object",
+            "properties": {
+              "ssl": {
+                "type": "object",
+                "properties": {
+                  "key": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
           }
         },
         "required": [

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -5240,6 +5240,16 @@ components:
           type: number
         required_acks:
           type: number
+        secrets:
+          type: object
+          properties:
+            password:
+              type: string
+            ssl:
+              type: object
+              properties:
+                key:
+                  type: string
       required:
         - name
         - type
@@ -5304,6 +5314,14 @@ components:
               type: number
             loadbalance:
               type: boolean
+        secrets:
+          type: object
+          properties:
+            ssl:
+              type: object
+              properties:
+                key:
+                  type: string
       required:
         - name
         - hosts

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml
@@ -122,6 +122,17 @@ properties:
     type: number
   required_acks:
     type: number
+  secrets:
+    type: object
+    properties:
+      password:
+        type: string
+      ssl:
+        type: object
+        properties:
+          key:
+            type: string
+
 required:
   - name
   - type

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml
@@ -54,6 +54,14 @@ properties:
         type: number
       loadbalance:
         type: boolean
+  secrets:
+      type: object
+      properties:
+        ssl:
+          type: object
+          properties:
+            key:
+              type: string
 required:
   - name
   - hosts

diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts
@@ -43,6 +43,15 @@ interface NewBaseOutput {
   proxy_id?: string | null;
   shipper?: ShipperOutput | null;
   allow_edit?: string[];
+  secrets?: {
+    ssl?: {
+      key?:
+        | string
+        | {
+            id: string;
+          };
+    };
+  };
 }
 
 export interface NewElasticsearchOutput extends NewBaseOutput {
@@ -112,4 +121,18 @@ export interface KafkaOutput extends NewBaseOutput {
   timeout?: number;
   broker_timeout?: number;
   required_acks?: ValueOf<KafkaAcknowledgeReliabilityLevel>;
+  secrets?: {
+    password?:
+      | string
+      | {
+          id: string;
+        };
+    ssl?: {
+      key?:
+        | string
+        | {
+            id: string;
+          };
+    };
+  };
 }
diff --git a/x-pack/plugins/fleet/common/types/models/secret.ts b/x-pack/plugins/fleet/common/types/models/secret.ts
@@ -21,6 +21,10 @@ export interface SecretPath {
   path: string;
   value: PackagePolicyConfigRecordEntry;
 }
+export interface OutputSecretPath {
+  path: string;
+  value: string | { id: string };
+}
 // this is used in the top level secret_refs array on package and agent policies
 export interface PolicySecretReference {
   id: string;

diff --git a/x-pack/plugins/fleet/server/routes/output/handler.ts b/x-pack/plugins/fleet/server/routes/output/handler.ts
@@ -8,6 +8,10 @@
 import type { RequestHandler } from '@kbn/core/server';
 import type { TypeOf } from '@kbn/config-schema';
 
+import Boom from '@hapi/boom';
+
+import { outputType } from '../../../common/constants';
+
 import type {
   DeleteOutputRequestSchema,
   GetOneOutputRequestSchema,
@@ -18,13 +22,23 @@ import type {
   DeleteOutputResponse,
   GetOneOutputResponse,
   GetOutputsResponse,
+  Output,
   PostLogstashApiKeyResponse,
 } from '../../../common/types';
 import { outputService } from '../../services/output';
 import { defaultFleetErrorHandler, FleetUnauthorizedError } from '../../errors';
 import { agentPolicyService } from '../../services';
 import { generateLogstashApiKey, canCreateLogstashApiKey } from '../../services/api_keys';
 
+function ensureNoDuplicateSecrets(output: Partial<Output>) {
+  if (output.type === outputType.Kafka && output?.password && output?.secrets?.password) {
+    throw Boom.badRequest('Cannot specify both password and secrets.password');
+  }
+  if (output.ssl?.key && output.secrets?.ssl?.key) {
+    throw Boom.badRequest('Cannot specify both ssl.key and secrets.ssl.key');
+  }
+}
+
 export const getOutputsHandler: RequestHandler = async (context, request, response) => {
   const soClient = (await context.core).savedObjects.client;
   try {
@@ -74,8 +88,10 @@ export const putOutputHandler: RequestHandler<
   const coreContext = await context.core;
   const soClient = coreContext.savedObjects.client;
   const esClient = coreContext.elasticsearch.client.asInternalUser;
+  const outputUpdate = request.body;
+  ensureNoDuplicateSecrets(outputUpdate);
   try {
-    await outputService.update(soClient, esClient, request.params.outputId, request.body);
+    await outputService.update(soClient, esClient, request.params.outputId, outputUpdate);
     const output = await outputService.get(soClient, request.params.outputId);
     if (output.is_default || output.is_default_monitoring) {
       await agentPolicyService.bumpAllAgentPolicies(soClient, esClient);
@@ -108,8 +124,9 @@ export const postOutputHandler: RequestHandler<
   const soClient = coreContext.savedObjects.client;
   const esClient = coreContext.elasticsearch.client.asInternalUser;
   try {
-    const { id, ...data } = request.body;
-    const output = await outputService.create(soClient, esClient, data, { id });
+    const { id, ...newOutput } = request.body;
+    ensureNoDuplicateSecrets(newOutput);
+    const output = await outputService.create(soClient, esClient, newOutput, { id });
     if (output.is_default || output.is_default_monitoring) {
       await agentPolicyService.bumpAllAgentPolicies(soClient, esClient);
     }

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -247,6 +247,28 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
         broker_buffer_size: { type: 'integer' },
         required_acks: { type: 'integer' },
         channel_buffer_size: { type: 'integer' },
+        secrets: {
+          dynamic: false,
+          properties: {
+            password: {
+              dynamic: false,
+              properties: {
+                id: { type: 'keyword' },
+              },
+            },
+            ssl: {
+              dynamic: false,
+              properties: {
+                key: {
+                  dynamic: false,
+                  properties: {
+                    id: { type: 'keyword' },
+                  },
+                },
+              },
+            },
+          },
+        },
       },
     },
     modelVersions: {

diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts
@@ -31,6 +31,8 @@ import { getPackageInfo } from '../epm/packages';
 import { pkgToPkgKey, splitPkgKey } from '../epm/registry';
 import { appContextService } from '../app_context';
 
+import { getOutputSecretReferences } from '../secrets';
+
 import { getMonitoringPermissions } from './monitoring_permissions';
 import { storedPackagePoliciesToAgentInputs } from '.';
 import {
@@ -110,6 +112,10 @@ export async function getFullAgentPolicy(
     return acc;
   }, {} as NonNullable<FullAgentPolicy['agent']>['features']);
 
+  const outputSecretReferences = outputs.flatMap((output) => getOutputSecretReferences(output));
+  const packagePolicySecretReferences = (agentPolicy?.package_policies || []).flatMap(
+    (policy) => policy.secret_references || []
+  );
   const defaultMonitoringConfig: FullAgentPolicyMonitoring = {
     enabled: false,
     logs: false,
@@ -152,9 +158,7 @@ export async function getFullAgentPolicy(
       }, {}),
     },
     inputs,
-    secret_references: (agentPolicy?.package_policies || []).flatMap(
-      (policy) => policy.secret_references || []
-    ),
+    secret_references: [...outputSecretReferences, ...packagePolicySecretReferences],
     revision: agentPolicy.revision,
     agent: {
       download: {
@@ -305,7 +309,8 @@ export function transformOutputToFullPolicyOutput(
   standalone = false
 ): FullAgentPolicyOutput {
   // eslint-disable-next-line @typescript-eslint/naming-convention
-  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper } = output;
+  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper, secrets } =
+    output;
 
   const configJs = config_yaml ? safeLoad(config_yaml) : {};
 
@@ -432,6 +437,7 @@ export function transformOutputToFullPolicyOutput(
     ...(!isShipperDisabled ? generalShipperData : {}),
     ...(ca_sha256 ? { ca_sha256 } : {}),
     ...(ssl ? { ssl } : {}),
+    ...(secrets ? { secrets } : {}),
     ...(ca_trusted_fingerprint ? { 'ssl.ca_trusted_fingerprint': ca_trusted_fingerprint } : {}),
   };
 

diff --git a/x-pack/plugins/fleet/server/services/agents/versions.ts b/x-pack/plugins/fleet/server/services/agents/versions.ts
@@ -75,9 +75,8 @@ export const getAvailableVersions = async ({
 
     return availableVersions;
   } catch (e) {
-    if (e.code === 'ENOENT' && !config?.internal?.onlyAllowAgentUpgradeToKnownVersions) {
-      // If the file does not exist, return the current version
-      return [kibanaVersion];
+    if (e.code === 'ENOENT') {
+      return config?.internal?.onlyAllowAgentUpgradeToKnownVersions ? [] : [kibanaVersion];
     }
     throw e;
   }

diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts
@@ -9,8 +9,9 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/serv
 
 import { securityMock } from '@kbn/security-plugin/server/mocks';
 
-import type { OutputSOAttributes } from '../types';
+import type { Logger } from '@kbn/logging';
 
+import type { OutputSOAttributes } from '../types';
 import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
 
 import { outputService, outputIdToUuid } from './output';
@@ -28,6 +29,17 @@ mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
   ...securityMock.createSetup(),
 }));
 
+mockedAppContextService.getLogger.mockImplementation(() => {
+  return {
+    debug: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+  } as unknown as Logger;
+});
+
+mockedAppContextService.getExperimentalFeatures.mockReturnValue({});
+
 const mockedAgentPolicyService = agentPolicyService as jest.Mocked<typeof agentPolicyService>;
 
 const CLOUD_ID =