diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts index 5f2537756126a4..11741726e344c9 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -12,7 +12,7 @@ import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { getObservabilityOnboardingFlow } from '../../lib/state'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml'; -import { generateSystemLogsYml } from './system_logs/generate_system_logs_yaml'; +import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml'; const generateConfig = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/elastic_agent/config', diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap new file mode 100644 index 00000000000000..074876d9d9119d --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateSystemLogsYml should return system logs oriented yml configuration 1`] = ` +"outputs: + default: + type: elasticsearch + hosts: + - 'http://localhost:9200' + api_key: 'elastic:changeme' +inputs: + - id: system-logs-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + type: logfile + data_stream: + namespace: default + streams: + - id: logfile-system.auth-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + data_stream: + dataset: system.auth + type: logs + paths: + - /var/log/auth.log* + - /var/log/secure* + exclude_files: + - .gz$ + multiline: + pattern: ^s + match: after + tags: + - system-auth + processors: + - add_locale: null + - id: logfile-system.syslog-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + data_stream: + dataset: system.syslog + type: logs + paths: + - /var/log/messages* + - /var/log/syslog* + - /var/log/system* + exclude_files: + - .gz$ + multiline: + pattern: ^s + match: after + processors: + - add_locale: null +" +`; diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts new file mode 100644 index 00000000000000..51169311044c24 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { generateSystemLogsYml } from './generate_system_logs_yml'; + +const baseMockConfig = { + namespace: 'default', + apiKey: 'elastic:changeme', + esHost: ['http://localhost:9200'], + uuid: uuidv4(), +}; + +describe('generateSystemLogsYml', () => { + it('should return system logs oriented yml configuration', () => { + const result = generateSystemLogsYml(baseMockConfig); + expect(result).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts similarity index 100% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts diff --git a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts index 87695c1ddc7fd9..9a86035bb44093 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts @@ -31,26 +31,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { const logFilepath = '/my-logs.log'; const serviceName = 'my-service'; - before(async () => { - const req = await observabilityOnboardingApiClient.logMonitoringUser({ - endpoint: 'POST /internal/observability_onboarding/logs/flow', - params: { - body: { - type: 'logFiles', - name: 'name', - state: { - datasetName, - namespace, - logFilePaths: [logFilepath], - serviceName, - }, - }, - }, - }); - - onboardingId = req.body.onboardingId; - }); - describe("when onboardingId doesn't exists", () => { it('should return input properties empty', async () => { const req = await callApi({ @@ -67,17 +47,72 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when onboardingId exists', () => { - it('should return input properties configured', async () => { - const req = await callApi({ - onboardingId, + describe('and onboarding type is logFiles', () => { + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'logFiles', + name: 'name', + state: { + datasetName, + namespace, + logFilePaths: [logFilepath], + serviceName, + }, + }, + }, + }); + + onboardingId = req.body.onboardingId; }); - expect(req.status).to.be(200); + it('should return input properties configured', async () => { + const req = await callApi({ + onboardingId, + }); - const ymlConfig = load(req.text); - expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace); - expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName); - expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]); + expect(req.status).to.be(200); + + const ymlConfig = load(req.text); + expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace); + expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName); + expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]); + expect(ymlConfig.inputs[0].streams[0].processors[0].add_fields.fields.name).to.be.eql( + serviceName + ); + }); + }); + + describe('and onboarding type is systemLogs', () => { + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'systemLogs', + name: 'name', + }, + }, + }); + + onboardingId = req.body.onboardingId; + }); + + it('should return input properties configured', async () => { + const req = await callApi({ + onboardingId, + }); + + expect(req.status).to.be(200); + + const ymlConfig = load(req.text); + expect(ymlConfig.inputs[0].data_stream.namespace).to.be('default'); + expect(ymlConfig.inputs[0].streams.length).to.be(2); + expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be('system.auth'); + expect(ymlConfig.inputs[0].streams[1].data_stream.dataset).to.be('system.syslog'); + }); }); }); }); diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts index a82ccc230b1247..001415ca59abbc 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts @@ -15,7 +15,7 @@ export function createLogDoc({ }: { time: number; logFilepath: string; - serviceName: string; + serviceName?: string; namespace: string; datasetName: string; message: string; @@ -30,9 +30,13 @@ export function createLogDoc({ path: logFilepath, }, }, - service: { - name: serviceName, - }, + ...(serviceName + ? { + service: { + name: serviceName, + }, + } + : {}), data_stream: { namespace, type: 'logs', diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts index b5dccdb5d9256b..93fd4294a0778b 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts @@ -132,40 +132,107 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when logs have been ingested', () => { - before(async () => { - await es.indices.createDataStream({ - name: `logs-${datasetName}-${namespace}`, + describe('and onboarding type is logFiles', () => { + before(async () => { + await es.indices.createDataStream({ + name: `logs-${datasetName}-${namespace}`, + }); + + const doc = createLogDoc({ + time: new Date('06/28/2023').getTime(), + logFilepath: '/my-service.log', + serviceName: 'my-service', + namespace, + datasetName, + message: 'This is a log message', + }); + + await es.bulk({ + body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc], + refresh: 'wait_for', + }); }); - const doc = createLogDoc({ - time: new Date('06/28/2023').getTime(), - logFilepath: '/my-service.log', - serviceName: 'my-service', - namespace, - datasetName, - message: 'This is a log message', + it('should return log-ingest as complete', async () => { + const request = await callApi({ + onboardingId, + }); + + expect(request.status).to.be(200); + + const logsIngestProgress = request.body.progress['logs-ingest']; + expect(logsIngestProgress).to.have.property('status', 'complete'); }); - await es.bulk({ - body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc], - refresh: 'wait_for', + after(async () => { + await es.indices.deleteDataStream({ + name: `logs-${datasetName}-${namespace}`, + }); }); }); - it('should return log-ingest as complete', async () => { - const request = await callApi({ - onboardingId, + describe('and onboarding type is systemLogs', () => { + let systemLogsOnboardingId: string; + + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'systemLogs', + name: 'name', + }, + }, + }); + + systemLogsOnboardingId = req.body.onboardingId; + + await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/flow/{id}/step/{name}', + params: { + path: { + id: systemLogsOnboardingId, + name: 'ea-status', + }, + body: { + status: 'complete', + }, + }, + }); + + await es.indices.createDataStream({ + name: `logs-system.syslog-${namespace}`, + }); + + const doc = createLogDoc({ + time: new Date('06/28/2023').getTime(), + logFilepath: '/var/log/system.log', + namespace, + datasetName: 'system.syslog', + message: 'This is a system log message', + }); + + await es.bulk({ + body: [{ create: { _index: `logs-system.syslog-${namespace}` } }, doc], + refresh: 'wait_for', + }); }); - expect(request.status).to.be(200); + it('should return log-ingest as complete', async () => { + const request = await callApi({ + onboardingId: systemLogsOnboardingId, + }); - const logsIngestProgress = request.body.progress['logs-ingest']; - expect(logsIngestProgress).to.have.property('status', 'complete'); - }); + expect(request.status).to.be(200); + + const logsIngestProgress = request.body.progress['logs-ingest']; + expect(logsIngestProgress).to.have.property('status', 'complete'); + }); - after(async () => { - await es.indices.deleteDataStream({ - name: `logs-${datasetName}-${namespace}`, + after(async () => { + await es.indices.deleteDataStream({ + name: `logs-system.syslog-${namespace}`, + }); }); }); }); diff --git a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts index a6c96f550d9b91..e263fb24d75809 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts @@ -29,12 +29,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } - async function callApiWithPrivileges(state = {}) { + async function callApiWithPrivileges(type: 'logFiles' | 'systemLogs', state = {}) { return await observabilityOnboardingApiClient.logMonitoringUser({ endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { - type: 'logFiles', + type, name: 'name', state, }, @@ -56,14 +56,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when required privileges are set', () => { it('returns a flow id and apiKey encoded', async () => { - const request = await callApiWithPrivileges(); + const request = await callApiWithPrivileges('logFiles'); expect(request.status).to.be(200); expect(request.body.apiKeyEncoded).to.not.empty(); expect(request.body.onboardingId).to.not.empty(); }); - it('saves the expected state', async () => { + it('saves the expected state for logFiles', async () => { const state = { datasetName: 'my-dataset', serviceName: 'my-service', @@ -71,7 +71,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { logFilePaths: 'my-service-logs.log', }; - const request = await callApiWithPrivileges(state); + const request = await callApiWithPrivileges('logFiles', state); const savedState = await kibanaServer.savedObjects.get({ type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, @@ -80,6 +80,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(savedState.attributes).to.be.eql({ type: 'logFiles', state, progress: {} }); }); + + it('saves the expected state for systemLogs', async () => { + const state = { + namespace: 'default', + }; + + const request = await callApiWithPrivileges('systemLogs'); + + const savedState = await kibanaServer.savedObjects.get({ + type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + id: request.body.onboardingId, + }); + + expect(savedState.attributes).to.be.eql({ type: 'systemLogs', state, progress: {} }); + }); }); }); }