[Logs onboarding] Generate elastic-agent.yml file for system logs (#1…

…62972)

Closes #154929.

This PR along with #162654,
#162706 and
#162600 completes the work
required for collect system logs.

### Changes
- `ObservabilityOnboardingType` now could be `logFiles | systemLogs`.
This help us to identify (without changing the script) whether we need
to retrieve the yaml configuration for customLogs or for systemLogs.
- Added `generateSystemLogsYml` which generates a specific configuration
for system logs.
- `get_has_logs.ts` was modified so we are querying the proper index
depending on the type of logs.

#### Demo


https://github.com/elastic/kibana/assets/1313018/47eca890-37b2-401e-9e41-67c978ab50ad

x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx

@@ -91,6 +91,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
+              type: 'logFiles',
               state: {
                 datasetName,
                 serviceName,

x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx

@@ -40,8 +40,7 @@ export function InstallElasticAgent() {
   const [elasticAgentPlatform, setElasticAgentPlatform] =
     useState<ElasticAgentPlatform>('linux-tar');
 
-  const datasetName = 'elastic-agent';
-  const namespace = 'default';
+  const datasetName = 'system-logs';
 
   function onBack() {
     navigateToKibanaUrl('/app/observabilityOnboarding');
@@ -83,10 +82,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
-              state: {
-                datasetName,
-                namespace,
-              },
+              type: 'systemLogs',
             },
           },
         });
@@ -95,26 +91,6 @@ export function InstallElasticAgent() {
     [monitoringRole?.hasPrivileges]
   );
 
-  const { status: saveOnboardingStateDataStatus } = useFetcher((callApi) => {
-    const { onboardingId } = getState();
-    if (onboardingId) {
-      return callApi(
-        'PUT /internal/observability_onboarding/flow/{onboardingId}',
-        {
-          params: {
-            path: { onboardingId },
-            body: {
-              state: {
-                datasetName,
-                namespace,
-              },
-            },
-          },
-        }
-      );
-    }
-  }, []);
-
   const { apiKeyEncoded, onboardingId } = installShipperSetup ?? getState();
 
   const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher(
@@ -132,7 +108,7 @@ export function InstallElasticAgent() {
     [
       apiKeyEncoded,
       onboardingId,
-      saveOnboardingStateDataStatus === FETCH_STATUS.SUCCESS,
+      installShipperSetupStatus === FETCH_STATUS.SUCCESS,
     ]
   );
 

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts → x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { dump } from 'js-yaml';
-import { generateYml } from './generate_yml';
+import { generateCustomLogsYml } from './generate_custom_logs_yml';
 
 const baseMockConfig = {
   datasetName: 'my-dataset',
@@ -17,9 +17,9 @@ const baseMockConfig = {
   logfileId: 'my-logs-id',
 };
 
-describe('generateYml', () => {
+describe('generateCustomLogsYml', () => {
   it('should return a basic yml configuration', () => {
-    const result = generateYml(baseMockConfig);
+    const result = generateCustomLogsYml(baseMockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -29,7 +29,7 @@ describe('generateYml', () => {
       logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'],
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -39,7 +39,7 @@ describe('generateYml', () => {
       serviceName: 'my-service',
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -57,7 +57,7 @@ describe('generateYml', () => {
       }),
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 });

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts → x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts

@@ -7,7 +7,7 @@
 
 import { dump, load } from 'js-yaml';
 
-export const generateYml = ({
+export const generateCustomLogsYml = ({
   datasetName = '',
   serviceName,
   namespace = '',

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts

@@ -6,11 +6,13 @@
  */
 
 import * as t from 'io-ts';
+import { v4 as uuidv4 } from 'uuid';
 import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
-import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
-import { generateYml } from './generate_yml';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { getObservabilityOnboardingFlow } from '../../lib/state';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml';
+import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml';
 
 const generateConfig = createObservabilityOnboardingServerRoute({
   endpoint: 'GET /internal/observability_onboarding/elastic_agent/config',
@@ -43,18 +45,24 @@ const generateConfig = createObservabilityOnboardingServerRoute({
       savedObjectId: onboardingId,
     });
 
-    const yaml = generateYml({
-      datasetName: savedState?.state?.datasetName,
-      customConfigurations: savedState?.state?.customConfigurations,
-      logFilePaths: savedState?.state?.logFilePaths,
-      namespace: savedState?.state?.namespace,
-      apiKey: authApiKey
-        ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
-        : '$API_KEY',
-      esHost: elasticsearchUrl,
-      logfileId: `custom-logs-${Date.now()}`,
-      serviceName: savedState?.state?.serviceName,
-    });
+    const yaml =
+      savedState?.type === 'systemLogs'
+        ? generateSystemLogsYml({
+            ...savedState?.state,
+            apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
+            esHost: elasticsearchUrl,
+            uuid: uuidv4(),
+          })
+        : generateCustomLogsYml({
+            ...savedState?.state,
+            apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
+            esHost: elasticsearchUrl,
+            logfileId: `custom-logs-${uuidv4()}`,
+          });
 
     return yaml;
   },

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { generateSystemLogsYml } from './generate_system_logs_yml';
+
+const baseMockConfig = {
+  namespace: 'default',
+  apiKey: 'elastic:changeme',
+  esHost: ['http://localhost:9200'],
+  uuid: '8df0ff52-6f3b-4b5a-a2da-f06c55d111d1',
+};
+
+describe('generateSystemLogsYml', () => {
+  it('should return system logs oriented yml configuration', () => {
+    const result = generateSystemLogsYml(baseMockConfig);
+    expect(result).toMatchSnapshot();
+  });
+});

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump } from 'js-yaml';
+
+export const generateSystemLogsYml = ({
+  namespace = 'default',
+  apiKey,
+  esHost,
+  uuid,
+}: {
+  namespace?: string;
+  apiKey: string;
+  esHost: string[];
+  uuid: string;
+}) => {
+  return dump({
+    outputs: {
+      default: {
+        type: 'elasticsearch',
+        hosts: esHost,
+        api_key: apiKey,
+      },
+    },
+    inputs: [
+      {
+        id: `system-logs-${uuid}`,
+        type: 'logfile',
+        data_stream: {
+          namespace,
+        },
+        streams: [
+          {
+            id: `logfile-system.auth-${uuid}`,
+            data_stream: {
+              dataset: 'system.auth',
+              type: 'logs',
+            },
+            paths: ['/var/log/auth.log*', '/var/log/secure*'],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            tags: ['system-auth'],
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+          {
+            id: `logfile-system.syslog-${uuid}`,
+            data_stream: {
+              dataset: 'system.syslog',
+              type: 'logs',
+            },
+            paths: [
+              '/var/log/messages*',
+              '/var/log/syslog*',
+              '/var/log/system*',
+            ],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+};