diff --git a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx index 7fc7c12fa47d43..dab59e7999068e 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx @@ -91,6 +91,7 @@ export function InstallElasticAgent() { params: { body: { name: datasetName, + type: 'logFiles', state: { datasetName, serviceName, diff --git a/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx index bcbc4e861248b2..1f84bef939f552 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx @@ -40,7 +40,7 @@ export function InstallElasticAgent() { const [elasticAgentPlatform, setElasticAgentPlatform] = useState('linux-tar'); - const datasetName = 'elastic-agent'; + const datasetName = 'system-logs'; const namespace = 'default'; function onBack() { @@ -83,6 +83,7 @@ export function InstallElasticAgent() { params: { body: { name: datasetName, + type: 'systemLogs', state: { datasetName, namespace, diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap similarity index 82% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap index 1023c1dcd8b1c8..dc388bb5e260d9 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`generateYml should return a basic yml configuration 1`] = ` +exports[`generateCustomLogsYml should return a basic yml configuration 1`] = ` "outputs: default: type: elasticsearch @@ -21,7 +21,7 @@ inputs: " `; -exports[`generateYml should return a yml configuration with customConfigurations 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with customConfigurations 1`] = ` "outputs: default: type: elasticsearch @@ -47,7 +47,7 @@ agent.monitoring: " `; -exports[`generateYml should return a yml configuration with multiple logFilePaths 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with multiple logFilePaths 1`] = ` "outputs: default: type: elasticsearch @@ -69,7 +69,7 @@ inputs: " `; -exports[`generateYml should return a yml configuration with service name 1`] = ` +exports[`generateCustomLogsYml should return a yml configuration with service name 1`] = ` "outputs: default: type: elasticsearch diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts similarity index 81% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts index 7b50ba67f27961..6cacfab2b792e3 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts @@ -6,7 +6,7 @@ */ import { dump } from 'js-yaml'; -import { generateYml } from './generate_yml'; +import { generateCustomLogsYml } from './generate_custom_logs_yml'; const baseMockConfig = { datasetName: 'my-dataset', @@ -17,9 +17,9 @@ const baseMockConfig = { logfileId: 'my-logs-id', }; -describe('generateYml', () => { +describe('generateCustomLogsYml', () => { it('should return a basic yml configuration', () => { - const result = generateYml(baseMockConfig); + const result = generateCustomLogsYml(baseMockConfig); expect(result).toMatchSnapshot(); }); @@ -29,7 +29,7 @@ describe('generateYml', () => { logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'], }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); @@ -39,7 +39,7 @@ describe('generateYml', () => { serviceName: 'my-service', }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); @@ -57,7 +57,7 @@ describe('generateYml', () => { }), }; - const result = generateYml(mockConfig); + const result = generateCustomLogsYml(mockConfig); expect(result).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts similarity index 97% rename from x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts rename to x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts index 7a2c0030572871..b836012f55fda1 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts @@ -7,7 +7,7 @@ import { dump, load } from 'js-yaml'; -export const generateYml = ({ +export const generateCustomLogsYml = ({ datasetName = '', serviceName, namespace = '', diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/index.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/index.ts new file mode 100644 index 00000000000000..44b3935f8ccee0 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './custom_logs/generate_custom_logs_yml'; +export * from './system_logs/generate_system_logs_yaml'; diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts index fe5549b7b8f163..92f0b4c5c266d8 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -6,11 +6,13 @@ */ import * as t from 'io-ts'; +import { v4 as uuidv4 } from 'uuid'; import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key'; -import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; -import { generateYml } from './generate_yml'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { getObservabilityOnboardingFlow } from '../../lib/state'; +import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; +import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml'; +import { generateSystemLogsYml } from './system_logs/generate_system_logs_yaml'; const generateConfig = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/elastic_agent/config', @@ -43,18 +45,24 @@ const generateConfig = createObservabilityOnboardingServerRoute({ savedObjectId: onboardingId, }); - const yaml = generateYml({ - datasetName: savedState?.state?.datasetName, - customConfigurations: savedState?.state?.customConfigurations, - logFilePaths: savedState?.state?.logFilePaths, - namespace: savedState?.state?.namespace, - apiKey: authApiKey - ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` - : '$API_KEY', - esHost: elasticsearchUrl, - logfileId: `custom-logs-${Date.now()}`, - serviceName: savedState?.state?.serviceName, - }); + const yaml = + savedState?.type === 'logFiles' + ? generateCustomLogsYml({ + ...savedState?.state, + apiKey: authApiKey + ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` + : '$API_KEY', + esHost: elasticsearchUrl, + logfileId: `custom-logs-${uuidv4()}`, + }) + : generateSystemLogsYml({ + ...savedState?.state, + apiKey: authApiKey + ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}` + : '$API_KEY', + esHost: elasticsearchUrl, + uuid: uuidv4(), + }); return yaml; }, diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts new file mode 100644 index 00000000000000..25d3c1ad82a2d6 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yaml.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dump } from 'js-yaml'; + +export const generateSystemLogsYml = ({ + namespace = 'default', + apiKey, + esHost, + uuid, +}: { + namespace?: string; + apiKey: string; + esHost: string[]; + uuid: string; +}) => { + return dump({ + outputs: { + default: { + type: 'elasticsearch', + hosts: esHost, + api_key: apiKey, + }, + }, + inputs: [ + { + id: `system-logs-${uuid}`, + type: 'logfile', + data_stream: { + namespace, + }, + streams: [ + { + id: `logfile-system.auth-${uuid}`, + data_stream: { + dataset: 'system.auth', + type: 'logs', + }, + paths: ['/var/log/auth.log*', '/var/log/secure*'], + exclude_files: ['.gz$'], + multiline: { + pattern: '^s', + match: 'after', + }, + tags: ['system-auth'], + processors: [ + { + add_locale: null, + }, + ], + }, + { + id: `logfile-system.syslog-${uuid}`, + data_stream: { + dataset: 'system.syslog', + type: 'logs', + }, + paths: [ + '/var/log/messages*', + '/var/log/syslog*', + '/var/log/system*', + ], + exclude_files: ['.gz$'], + multiline: { + pattern: '^s', + match: 'after', + }, + tags: ['system-auth'], + processors: [ + { + add_locale: null, + }, + ], + }, + ], + }, + ], + }); +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts index 59e96b9797cc8a..1fb2baad1e4bb8 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts @@ -6,19 +6,34 @@ */ import { ElasticsearchClient } from '@kbn/core/server'; +import { + LogFilesState, + ObservabilityOnboardingType, + SystemLogsState, +} from '../../saved_objects/observability_onboarding_status'; export async function getHasLogs({ - dataset, - namespace, + type, + state, esClient, }: { - dataset: string; - namespace: string; + type: ObservabilityOnboardingType; + state?: LogFilesState | SystemLogsState; esClient: ElasticsearchClient; }) { + if (!state) { + return false; + } + try { + const { namespace } = state; + const index = + type === 'logFiles' + ? `logs-${(state as LogFilesState).datasetName}-${namespace}` + : `logs-system.syslog-${namespace}`; + const { hits } = await esClient.search({ - index: `logs-${dataset}-${namespace}`, + index, terminate_after: 1, }); const total = hits.total as { value: number }; diff --git a/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts index 2776af522388fb..7689167945966d 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/flow/route.ts @@ -148,16 +148,13 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({ const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; - const dataset = savedObservabilityOnboardingState.state - ?.datasetName as string; - const namespace = savedObservabilityOnboardingState.state - ?.namespace as string; + const type = savedObservabilityOnboardingState.type; if (progress['ea-status']?.status === 'complete') { try { const hasLogs = await getHasLogs({ - dataset, - namespace, + type, + state: savedObservabilityOnboardingState.state, esClient, }); if (hasLogs) { diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts index 6ad4a60cccc62f..ffc3f87b0603f4 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts @@ -15,7 +15,7 @@ export function createShipperApiKey( // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent return esClient.security.createApiKey({ body: { - name: `standalone_agent_custom_logs_${name}`, + name: `standalone_agent_logs_onboarding_${name}`, metadata: { application: 'logs' }, role_descriptors: { standalone_agent: { diff --git a/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts b/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts index 4afe189ed7279c..2647835032d3c9 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/logs/route.ts @@ -66,6 +66,9 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ t.type({ name: t.string, }), + t.type({ + type: t.union([t.literal('logFiles'), t.literal('systemLogs')]), + }), t.partial({ state: t.record(t.string, t.unknown), }), @@ -77,7 +80,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ const { context, params: { - body: { name, state }, + body: { name, type, state }, }, core, request, @@ -96,7 +99,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ const { id } = await saveObservabilityOnboardingFlow({ savedObjectsClient, observabilityOnboardingState: { - type: 'logFiles', + type, state: state as ObservabilityOnboardingFlow['state'], progress: {}, }, diff --git a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts index 4c151e29a1b2a1..b7426ac048cd86 100644 --- a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts +++ b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts @@ -17,10 +17,19 @@ export interface LogFilesState { namespace: string; } -type ObservabilityOnboardingFlowState = LogFilesState | undefined; +export interface SystemLogsState { + namespace: string; +} + +export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs'; + +type ObservabilityOnboardingFlowState = + | LogFilesState + | SystemLogsState + | undefined; export interface ObservabilityOnboardingFlow { - type: 'logFiles'; + type: ObservabilityOnboardingType; state: ObservabilityOnboardingFlowState; progress: Record< string, diff --git a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts index 5b4f5041d1d11e..87695c1ddc7fd9 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts @@ -36,6 +36,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state: { datasetName, diff --git a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts index a6076261d81c4b..b5dccdb5d9256b 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts @@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state: { datasetName, diff --git a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts index 1de222e2d2368d..a6c96f550d9b91 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts @@ -21,6 +21,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state, }, @@ -33,6 +34,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state, }, diff --git a/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts b/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts index 0826b3719837b0..4554cbbfba733c 100644 --- a/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts +++ b/x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts @@ -67,6 +67,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'POST /internal/observability_onboarding/logs/flow', params: { body: { + type: 'logFiles', name: 'name', state: {}, },