using onboarding type to generate proper yaml configuration

elastic · Aug 2, 2023 · c63e8d3 · c63e8d3
1 parent 0a46b2b
commit c63e8d3
Show file tree

Hide file tree

Showing 17 changed files with 173 additions and 42 deletions.
diff --git a/...servability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx b/...servability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx
@@ -91,6 +91,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
+              type: 'logFiles',
               state: {
                 datasetName,
                 serviceName,

diff --git a/...gins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx b/...gins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx
@@ -40,7 +40,7 @@ export function InstallElasticAgent() {
   const [elasticAgentPlatform, setElasticAgentPlatform] =
     useState<ElasticAgentPlatform>('linux-tar');
 
-  const datasetName = 'elastic-agent';
+  const datasetName = 'system-logs';
   const namespace = 'default';
 
   function onBack() {
@@ -83,6 +83,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
+              type: 'systemLogs',
               state: {
                 datasetName,
                 namespace,

diff --git a/...t/__snapshots__/generate_yml.test.ts.snap → ...s__/generate_custom_logs_yml.test.ts.snap b/...t/__snapshots__/generate_yml.test.ts.snap → ...s__/generate_custom_logs_yml.test.ts.snap
diff --git a/...routes/elastic_agent/generate_yml.test.ts → ...tom_logs/generate_custom_logs_yml.test.ts b/...routes/elastic_agent/generate_yml.test.ts → ...tom_logs/generate_custom_logs_yml.test.ts
@@ -6,7 +6,7 @@
  */
 
 import { dump } from 'js-yaml';
-import { generateYml } from './generate_yml';
+import { generateCustomLogsYml } from './generate_custom_logs_yml';
 
 const baseMockConfig = {
   datasetName: 'my-dataset',
@@ -17,9 +17,9 @@ const baseMockConfig = {
   logfileId: 'my-logs-id',
 };
 
-describe('generateYml', () => {
+describe('generateCustomLogsYml', () => {
   it('should return a basic yml configuration', () => {
-    const result = generateYml(baseMockConfig);
+    const result = generateCustomLogsYml(baseMockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -29,7 +29,7 @@ describe('generateYml', () => {
       logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'],
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -39,7 +39,7 @@ describe('generateYml', () => {
       serviceName: 'my-service',
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -57,7 +57,7 @@ describe('generateYml', () => {
       }),
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 });
diff --git a/...rver/routes/elastic_agent/generate_yml.ts → ...t/custom_logs/generate_custom_logs_yml.ts b/...rver/routes/elastic_agent/generate_yml.ts → ...t/custom_logs/generate_custom_logs_yml.ts
@@ -7,7 +7,7 @@
 
 import { dump, load } from 'js-yaml';
 
-export const generateYml = ({
+export const generateCustomLogsYml = ({
   datasetName = '',
   serviceName,
   namespace = '',

diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/index.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/index.ts
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './custom_logs/generate_custom_logs_yml';
+export * from './system_logs/generate_system_logs_yaml';
diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts
@@ -6,11 +6,13 @@
  */
 
 import * as t from 'io-ts';
+import { v4 as uuidv4 } from 'uuid';
 import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
-import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
-import { generateYml } from './generate_yml';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { getObservabilityOnboardingFlow } from '../../lib/state';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml';
+import { generateSystemLogsYml } from './system_logs/generate_system_logs_yaml';
 
 const generateConfig = createObservabilityOnboardingServerRoute({
   endpoint: 'GET /internal/observability_onboarding/elastic_agent/config',
@@ -43,18 +45,24 @@ const generateConfig = createObservabilityOnboardingServerRoute({
       savedObjectId: onboardingId,
     });
 
-    const yaml = generateYml({
-      datasetName: savedState?.state?.datasetName,
-      customConfigurations: savedState?.state?.customConfigurations,
-      logFilePaths: savedState?.state?.logFilePaths,
-      namespace: savedState?.state?.namespace,
-      apiKey: authApiKey
-        ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
-        : '$API_KEY',
-      esHost: elasticsearchUrl,
-      logfileId: `custom-logs-${Date.now()}`,
-      serviceName: savedState?.state?.serviceName,
-    });
+    const yaml =
+      savedState?.type === 'logFiles'
+        ? generateCustomLogsYml({
+            ...savedState?.state,
+            apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
+            esHost: elasticsearchUrl,
+            logfileId: `custom-logs-${uuidv4()}`,
+          })
+        : generateSystemLogsYml({
+            ...savedState?.state,
+            apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
+            esHost: elasticsearchUrl,
+            uuid: uuidv4(),
+          });
 
     return yaml;
   },

diff --git a/...rvability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts b/...rvability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump } from 'js-yaml';
+
+export const generateSystemLogsYml = ({
+  namespace = 'default',
+  apiKey,
+  esHost,
+  uuid,
+}: {
+  namespace?: string;
+  apiKey: string;
+  esHost: string[];
+  uuid: string;
+}) => {
+  return dump({
+    outputs: {
+      default: {
+        type: 'elasticsearch',
+        hosts: esHost,
+        api_key: apiKey,
+      },
+    },
+    inputs: [
+      {
+        id: `system-logs-${uuid}`,
+        type: 'logfile',
+        data_stream: {
+          namespace,
+        },
+        streams: [
+          {
+            id: `logfile-system.auth-${uuid}`,
+            data_stream: {
+              dataset: 'system.auth',
+              type: 'logs',
+            },
+            paths: ['/var/log/auth.log*', '/var/log/secure*'],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            tags: ['system-auth'],
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+          {
+            id: `logfile-system.syslog-${uuid}`,
+            data_stream: {
+              dataset: 'system.syslog',
+              type: 'logs',
+            },
+            paths: [
+              '/var/log/messages*',
+              '/var/log/syslog*',
+              '/var/log/system*',
+            ],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            tags: ['system-auth'],
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+};
diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts
@@ -6,19 +6,34 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import {
+  LogFilesState,
+  ObservabilityOnboardingType,
+  SystemLogsState,
+} from '../../saved_objects/observability_onboarding_status';
 
 export async function getHasLogs({
-  dataset,
-  namespace,
+  type,
+  state,
   esClient,
 }: {
-  dataset: string;
-  namespace: string;
+  type: ObservabilityOnboardingType;
+  state?: LogFilesState | SystemLogsState;
   esClient: ElasticsearchClient;
 }) {
+  if (!state) {
+    return false;
+  }
+
   try {
+    const { namespace } = state;
+    const index =
+      type === 'logFiles'
+        ? `logs-${(state as LogFilesState).datasetName}-${namespace}`
+        : `logs-system.syslog-${namespace}`;
+
     const { hits } = await esClient.search({
-      index: `logs-${dataset}-${namespace}`,
+      index,
       terminate_after: 1,
     });
     const total = hits.total as { value: number };

diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts
@@ -148,16 +148,13 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({
     const esClient =
       coreStart.elasticsearch.client.asScoped(request).asCurrentUser;
 
-    const dataset = savedObservabilityOnboardingState.state
-      ?.datasetName as string;
-    const namespace = savedObservabilityOnboardingState.state
-      ?.namespace as string;
+    const type = savedObservabilityOnboardingState.type;
 
     if (progress['ea-status']?.status === 'complete') {
       try {
         const hasLogs = await getHasLogs({
-          dataset,
-          namespace,
+          type,
+          state: savedObservabilityOnboardingState.state,
           esClient,
         });
         if (hasLogs) {

diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts
@@ -15,7 +15,7 @@ export function createShipperApiKey(
   // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent
   return esClient.security.createApiKey({
     body: {
-      name: `standalone_agent_custom_logs_${name}`,
+      name: `standalone_agent_logs_onboarding_${name}`,
       metadata: { application: 'logs' },
       role_descriptors: {
         standalone_agent: {

diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts
@@ -66,6 +66,9 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
       t.type({
         name: t.string,
       }),
+      t.type({
+        type: t.union([t.literal('logFiles'), t.literal('systemLogs')]),
+      }),
       t.partial({
         state: t.record(t.string, t.unknown),
       }),
@@ -77,7 +80,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
     const {
       context,
       params: {
-        body: { name, state },
+        body: { name, type, state },
       },
       core,
       request,
@@ -96,7 +99,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
     const { id } = await saveObservabilityOnboardingFlow({
       savedObjectsClient,
       observabilityOnboardingState: {
-        type: 'logFiles',
+        type,
         state: state as ObservabilityOnboardingFlow['state'],
         progress: {},
       },

diff --git a/.../plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/.../plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts
@@ -17,10 +17,19 @@ export interface LogFilesState {
   namespace: string;
 }
 
-type ObservabilityOnboardingFlowState = LogFilesState | undefined;
+export interface SystemLogsState {
+  namespace: string;
+}
+
+export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs';
+
+type ObservabilityOnboardingFlowState =
+  | LogFilesState
+  | SystemLogsState
+  | undefined;
 
 export interface ObservabilityOnboardingFlow {
-  type: 'logFiles';
+  type: ObservabilityOnboardingType;
   state: ObservabilityOnboardingFlowState;
   progress: Record<
     string,

diff --git a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts
@@ -36,6 +36,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         endpoint: 'POST /internal/observability_onboarding/logs/flow',
         params: {
           body: {
+            type: 'logFiles',
             name: 'name',
             state: {
               datasetName,

diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts
@@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         endpoint: 'POST /internal/observability_onboarding/logs/flow',
         params: {
           body: {
+            type: 'logFiles',
             name: 'name',
             state: {
               datasetName,