diff --git a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx index 7fc7c12fa47d43..dab59e7999068e 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx @@ -91,6 +91,7 @@ export function InstallElasticAgent() { params: { body: { name: datasetName, + type: 'logFiles', state: { datasetName, serviceName, diff --git a/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx index bcbc4e861248b2..5eb80c9e11525f 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx @@ -40,8 +40,7 @@ export function InstallElasticAgent() { const [elasticAgentPlatform, setElasticAgentPlatform] = useState('linux-tar'); - const datasetName = 'elastic-agent'; - const namespace = 'default'; + const datasetName = 'system-logs'; function onBack() { navigateToKibanaUrl('/app/observabilityOnboarding'); @@ -83,10 +82,7 @@ export function InstallElasticAgent() { params: { body: { name: datasetName, - state: { - datasetName, - namespace, - }, + type: 'systemLogs', }, }, }); @@ -95,26 +91,6 @@ export function InstallElasticAgent() { [monitoringRole?.hasPrivileges] ); - const { status: saveOnboardingStateDataStatus } = useFetcher((callApi) => { - const { onboardingId } = getState(); - if (onboardingId) { - return callApi( - 'PUT /internal/observability_onboarding/flow/{onboardingId}', - { - params: { - path: { onboardingId }, - body: { - state: { - datasetName, - namespace, - }, - }, - }, - } - ); - } - }, []); - const { apiKeyEncoded, onboardingId } = installShipperSetup ?? getState(); const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher( @@ -132,7 +108,7 @@ export function InstallElasticAgent() { [ apiKeyEncoded, onboardingId, - saveOnboardingStateDataStatus === FETCH_STATUS.SUCCESS, + installShipperSetupStatus === FETCH_STATUS.SUCCESS, ] ); diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap similarity index 82% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap index 1023c1dcd8b1c8..dc388bb5e260d9 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`generateYml should return a basic yml configuration 1`] = ` +exports[`generateCustomLogsYml should return a basic yml configuration 1`] = ` "outputs: default: type: elasticsearch @@ -21,7 +21,7 @@ inputs: " `; -exports[`generateYml should return a yml configuration with customConfigurations 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with customConfigurations 1`] = ` "outputs: default: type: elasticsearch @@ -47,7 +47,7 @@ agent.monitoring: " `; -exports[`generateYml should return a yml configuration with multiple logFilePaths 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with multiple logFilePaths 1`] = ` "outputs: default: type: elasticsearch @@ -69,7 +69,7 @@ inputs: " `; -exports[`generateYml should return a yml configuration with service name 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with service name 1`] = ` "outputs: default: type: elasticsearch diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts similarity index 81% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts index 7b50ba67f27961..6cacfab2b792e3 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts @@ -6,7 +6,7 @@ */ import { dump } from 'js-yaml'; -import { generateYml } from './generate_yml'; +import { generateCustomLogsYml } from './generate_custom_logs_yml'; const baseMockConfig = { datasetName: 'my-dataset', @@ -17,9 +17,9 @@ const baseMockConfig = { logfileId: 'my-logs-id', }; -describe('generateYml', () => { +describe('generateCustomLogsYml', () => { it('should return a basic yml configuration', () => { - const result = generateYml(baseMockConfig); + const result = generateCustomLogsYml(baseMockConfig); expect(result).toMatchSnapshot(); }); @@ -29,7 +29,7 @@ describe('generateYml', () => { logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'], }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); @@ -39,7 +39,7 @@ describe('generateYml', () => { serviceName: 'my-service', }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); @@ -57,7 +57,7 @@ describe('generateYml', () => { }), }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts similarity index 97% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts index 7a2c0030572871..b836012f55fda1 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts @@ -7,7 +7,7 @@ import { dump, load } from 'js-yaml'; -export const generateYml = ({ +export const generateCustomLogsYml = ({ datasetName = '', serviceName, namespace = '', diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts index fe5549b7b8f163..11741726e344c9 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -6,11 +6,13 @@ */ import * as t from 'io-ts'; +import { v4 as uuidv4 } from 'uuid'; import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key'; -import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; -import { generateYml } from './generate_yml'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { getObservabilityOnboardingFlow } from '../../lib/state'; +import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; +import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml'; +import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml'; const generateConfig = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/elastic_agent/config', @@ -43,18 +45,24 @@ const generateConfig = createObservabilityOnboardingServerRoute({ savedObjectId: onboardingId, }); - const yaml = generateYml({ - datasetName: savedState?.state?.datasetName, - customConfigurations: savedState?.state?.customConfigurations, - logFilePaths: savedState?.state?.logFilePaths, - namespace: savedState?.state?.namespace, - apiKey: authApiKey - ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` - : '$API_KEY', - esHost: elasticsearchUrl, - logfileId: `custom-logs-${Date.now()}`, - serviceName: savedState?.state?.serviceName, - }); + const yaml = + savedState?.type === 'systemLogs' + ? generateSystemLogsYml({ + ...savedState?.state, + apiKey: authApiKey + ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` + : '$API_KEY', + esHost: elasticsearchUrl, + uuid: uuidv4(), + }) + : generateCustomLogsYml({ + ...savedState?.state, + apiKey: authApiKey + ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` + : '$API_KEY', + esHost: elasticsearchUrl, + logfileId: `custom-logs-${uuidv4()}`, + }); return yaml; }, diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap new file mode 100644 index 00000000000000..074876d9d9119d --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateSystemLogsYml should return system logs oriented yml configuration 1`] = ` +"outputs: + default: + type: elasticsearch + hosts: + - 'http://localhost:9200' + api_key: 'elastic:changeme' +inputs: + - id: system-logs-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + type: logfile + data_stream: + namespace: default + streams: + - id: logfile-system.auth-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + data_stream: + dataset: system.auth + type: logs + paths: + - /var/log/auth.log* + - /var/log/secure* + exclude_files: + - .gz$ + multiline: + pattern: ^s + match: after + tags: + - system-auth + processors: + - add_locale: null + - id: logfile-system.syslog-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1 + data_stream: + dataset: system.syslog + type: logs + paths: + - /var/log/messages* + - /var/log/syslog* + - /var/log/system* + exclude_files: + - .gz$ + multiline: + pattern: ^s + match: after + processors: + - add_locale: null +" +`; diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts new file mode 100644 index 00000000000000..f26222f1ec0f2b --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { generateSystemLogsYml } from './generate_system_logs_yml'; + +const baseMockConfig = { + namespace: 'default', + apiKey: 'elastic:changeme', + esHost: ['http://localhost:9200'], + uuid: '8df0ff52-6f3b-4b5a-a2da-f06c55d111d1', +}; + +describe('generateSystemLogsYml', () => { + it('should return system logs oriented yml configuration', () => { + const result = generateSystemLogsYml(baseMockConfig); + expect(result).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts new file mode 100644 index 00000000000000..c9335cb97fa28d --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dump } from 'js-yaml'; + +export const generateSystemLogsYml = ({ + namespace = 'default', + apiKey, + esHost, + uuid, +}: { + namespace?: string; + apiKey: string; + esHost: string[]; + uuid: string; +}) => { + return dump({ + outputs: { + default: { + type: 'elasticsearch', + hosts: esHost, + api_key: apiKey, + }, + }, + inputs: [ + { + id: `system-logs-${uuid}`, + type: 'logfile', + data_stream: { + namespace, + }, + streams: [ + { + id: `logfile-system.auth-${uuid}`, + data_stream: { + dataset: 'system.auth', + type: 'logs', + }, + paths: ['/var/log/auth.log*', '/var/log/secure*'], + exclude_files: ['.gz$'], + multiline: { + pattern: '^s', + match: 'after', + }, + tags: ['system-auth'], + processors: [ + { + add_locale: null, + }, + ], + }, + { + id: `logfile-system.syslog-${uuid}`, + data_stream: { + dataset: 'system.syslog', + type: 'logs', + }, + paths: [ + '/var/log/messages*', + '/var/log/syslog*', + '/var/log/system*', + ], + exclude_files: ['.gz$'], + multiline: { + pattern: '^s', + match: 'after', + }, + processors: [ + { + add_locale: null, + }, + ], + }, + ], + }, + ], + }); +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts index 59e96b9797cc8a..1fb2baad1e4bb8 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts @@ -6,19 +6,34 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; +import { + LogFilesState, + ObservabilityOnboardingType, + SystemLogsState, +} from '../../saved_objects/observability_onboarding_status'; export async function getHasLogs({ - dataset, - namespace, + type, + state, esClient, }: { - dataset: string; - namespace: string; + type: ObservabilityOnboardingType; + state?: LogFilesState | SystemLogsState; esClient: ElasticsearchClient; }) { + if (!state) { + return false; + } + try { + const { namespace } = state; + const index = + type === 'logFiles' + ? `logs-${(state as LogFilesState).datasetName}-${namespace}` + : `logs-system.syslog-${namespace}`; + const { hits } = await esClient.search({ - index: `logs-${dataset}-${namespace}`, + index, terminate_after: 1, }); const total = hits.total as { value: number }; diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts index 2776af522388fb..7689167945966d 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts @@ -148,16 +148,13 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({ const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; - const dataset = savedObservabilityOnboardingState.state - ?.datasetName as string; - const namespace = savedObservabilityOnboardingState.state - ?.namespace as string; + const type = savedObservabilityOnboardingState.type; if (progress['ea-status']?.status === 'complete') { try { const hasLogs = await getHasLogs({ - dataset, - namespace, + type, + state: savedObservabilityOnboardingState.state, esClient, }); if (hasLogs) { diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts index 6ad4a60cccc62f..ffc3f87b0603f4 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts @@ -15,7 +15,7 @@ export function createShipperApiKey( // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent return esClient.security.createApiKey({ body: { - name: `standalone_agent_custom_logs_${name}`, + name: `standalone_agent_logs_onboarding_${name}`, metadata: { application: 'logs' }, role_descriptors: { standalone_agent: { diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts index 4afe189ed7279c..c28c393382930c 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts @@ -66,6 +66,9 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ t.type({ name: t.string, }), + t.type({ + type: t.union([t.literal('logFiles'), t.literal('systemLogs')]), + }), t.partial({ state: t.record(t.string, t.unknown), }), @@ -77,7 +80,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ const { context, params: { - body: { name, state }, + body: { name, type, state }, }, core, request, @@ -91,13 +94,15 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ name ); + const generatedState = + type === 'systemLogs' ? { namespace: 'default' } : state; const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); const { id } = await saveObservabilityOnboardingFlow({ savedObjectsClient, observabilityOnboardingState: { - type: 'logFiles', - state: state as ObservabilityOnboardingFlow['state'], + type, + state: generatedState as ObservabilityOnboardingFlow['state'], progress: {}, }, }); diff --git a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts index 4c151e29a1b2a1..b7426ac048cd86 100644 --- a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts +++ b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts @@ -17,10 +17,19 @@ export interface LogFilesState { namespace: string; } -type ObservabilityOnboardingFlowState = LogFilesState | undefined; +export interface SystemLogsState { + namespace: string; +} + +export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs'; + +type ObservabilityOnboardingFlowState = + | LogFilesState + | SystemLogsState + | undefined; export interface ObservabilityOnboardingFlow { - type: 'logFiles'; + type: ObservabilityOnboardingType; state: ObservabilityOnboardingFlowState; progress: Record< string, diff --git a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts index 5b4f5041d1d11e..9a86035bb44093 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts @@ -31,25 +31,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { const logFilepath = '/my-logs.log'; const serviceName = 'my-service'; - before(async () => { - const req = await observabilityOnboardingApiClient.logMonitoringUser({ - endpoint: 'POST /internal/observability_onboarding/logs/flow', - params: { - body: { - name: 'name', - state: { - datasetName, - namespace, - logFilePaths: [logFilepath], - serviceName, - }, - }, - }, - }); - - onboardingId = req.body.onboardingId; - }); - describe("when onboardingId doesn't exists", () => { it('should return input properties empty', async () => { const req = await callApi({ @@ -66,17 +47,72 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when onboardingId exists', () => { - it('should return input properties configured', async () => { - const req = await callApi({ - onboardingId, + describe('and onboarding type is logFiles', () => { + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'logFiles', + name: 'name', + state: { + datasetName, + namespace, + logFilePaths: [logFilepath], + serviceName, + }, + }, + }, + }); + + onboardingId = req.body.onboardingId; }); - expect(req.status).to.be(200); + it('should return input properties configured', async () => { + const req = await callApi({ + onboardingId, + }); - const ymlConfig = load(req.text); - expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace); - expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName); - expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]); + expect(req.status).to.be(200); + + const ymlConfig = load(req.text); + expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace); + expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName); + expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]); + expect(ymlConfig.inputs[0].streams[0].processors[0].add_fields.fields.name).to.be.eql( + serviceName + ); + }); + }); + + describe('and onboarding type is systemLogs', () => { + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'systemLogs', + name: 'name', + }, + }, + }); + + onboardingId = req.body.onboardingId; + }); + + it('should return input properties configured', async () => { + const req = await callApi({ + onboardingId, + }); + + expect(req.status).to.be(200); + + const ymlConfig = load(req.text); + expect(ymlConfig.inputs[0].data_stream.namespace).to.be('default'); + expect(ymlConfig.inputs[0].streams.length).to.be(2); + expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be('system.auth'); + expect(ymlConfig.inputs[0].streams[1].data_stream.dataset).to.be('system.syslog'); + }); }); }); }); diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts index a82ccc230b1247..001415ca59abbc 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts @@ -15,7 +15,7 @@ export function createLogDoc({ }: { time: number; logFilepath: string; - serviceName: string; + serviceName?: string; namespace: string; datasetName: string; message: string; @@ -30,9 +30,13 @@ export function createLogDoc({ path: logFilepath, }, }, - service: { - name: serviceName, - }, + ...(serviceName + ? { + service: { + name: serviceName, + }, + } + : {}), data_stream: { namespace, type: 'logs', diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts index a6076261d81c4b..93fd4294a0778b 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts @@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state: { datasetName, @@ -131,40 +132,107 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when logs have been ingested', () => { - before(async () => { - await es.indices.createDataStream({ - name: `logs-${datasetName}-${namespace}`, + describe('and onboarding type is logFiles', () => { + before(async () => { + await es.indices.createDataStream({ + name: `logs-${datasetName}-${namespace}`, + }); + + const doc = createLogDoc({ + time: new Date('06/28/2023').getTime(), + logFilepath: '/my-service.log', + serviceName: 'my-service', + namespace, + datasetName, + message: 'This is a log message', + }); + + await es.bulk({ + body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc], + refresh: 'wait_for', + }); }); - const doc = createLogDoc({ - time: new Date('06/28/2023').getTime(), - logFilepath: '/my-service.log', - serviceName: 'my-service', - namespace, - datasetName, - message: 'This is a log message', + it('should return log-ingest as complete', async () => { + const request = await callApi({ + onboardingId, + }); + + expect(request.status).to.be(200); + + const logsIngestProgress = request.body.progress['logs-ingest']; + expect(logsIngestProgress).to.have.property('status', 'complete'); }); - await es.bulk({ - body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc], - refresh: 'wait_for', + after(async () => { + await es.indices.deleteDataStream({ + name: `logs-${datasetName}-${namespace}`, + }); }); }); - it('should return log-ingest as complete', async () => { - const request = await callApi({ - onboardingId, + describe('and onboarding type is systemLogs', () => { + let systemLogsOnboardingId: string; + + before(async () => { + const req = await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/logs/flow', + params: { + body: { + type: 'systemLogs', + name: 'name', + }, + }, + }); + + systemLogsOnboardingId = req.body.onboardingId; + + await observabilityOnboardingApiClient.logMonitoringUser({ + endpoint: 'POST /internal/observability_onboarding/flow/{id}/step/{name}', + params: { + path: { + id: systemLogsOnboardingId, + name: 'ea-status', + }, + body: { + status: 'complete', + }, + }, + }); + + await es.indices.createDataStream({ + name: `logs-system.syslog-${namespace}`, + }); + + const doc = createLogDoc({ + time: new Date('06/28/2023').getTime(), + logFilepath: '/var/log/system.log', + namespace, + datasetName: 'system.syslog', + message: 'This is a system log message', + }); + + await es.bulk({ + body: [{ create: { _index: `logs-system.syslog-${namespace}` } }, doc], + refresh: 'wait_for', + }); }); - expect(request.status).to.be(200); + it('should return log-ingest as complete', async () => { + const request = await callApi({ + onboardingId: systemLogsOnboardingId, + }); - const logsIngestProgress = request.body.progress['logs-ingest']; - expect(logsIngestProgress).to.have.property('status', 'complete'); - }); + expect(request.status).to.be(200); + + const logsIngestProgress = request.body.progress['logs-ingest']; + expect(logsIngestProgress).to.have.property('status', 'complete'); + }); - after(async () => { - await es.indices.deleteDataStream({ - name: `logs-${datasetName}-${namespace}`, + after(async () => { + await es.indices.deleteDataStream({ + name: `logs-system.syslog-${namespace}`, + }); }); }); }); diff --git a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts index 1de222e2d2368d..e263fb24d75809 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts @@ -21,6 +21,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state, }, @@ -28,11 +29,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } - async function callApiWithPrivileges(state = {}) { + async function callApiWithPrivileges(type: 'logFiles' | 'systemLogs', state = {}) { return await observabilityOnboardingApiClient.logMonitoringUser({ endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type, name: 'name', state, }, @@ -54,14 +56,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when required privileges are set', () => { it('returns a flow id and apiKey encoded', async () => { - const request = await callApiWithPrivileges(); + const request = await callApiWithPrivileges('logFiles'); expect(request.status).to.be(200); expect(request.body.apiKeyEncoded).to.not.empty(); expect(request.body.onboardingId).to.not.empty(); }); - it('saves the expected state', async () => { + it('saves the expected state for logFiles', async () => { const state = { datasetName: 'my-dataset', serviceName: 'my-service', @@ -69,7 +71,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { logFilePaths: 'my-service-logs.log', }; - const request = await callApiWithPrivileges(state); + const request = await callApiWithPrivileges('logFiles', state); const savedState = await kibanaServer.savedObjects.get({ type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, @@ -78,6 +80,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(savedState.attributes).to.be.eql({ type: 'logFiles', state, progress: {} }); }); + + it('saves the expected state for systemLogs', async () => { + const state = { + namespace: 'default', + }; + + const request = await callApiWithPrivileges('systemLogs'); + + const savedState = await kibanaServer.savedObjects.get({ + type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + id: request.body.onboardingId, + }); + + expect(savedState.attributes).to.be.eql({ type: 'systemLogs', state, progress: {} }); + }); }); }); } diff --git a/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts index 0826b3719837b0..4554cbbfba733c 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts @@ -67,6 +67,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state: {}, },