From 697bb093414a0f127fbade3d13872bffca29daa5 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 10 May 2024 16:13:35 -0600 Subject: [PATCH 01/16] [OAM][POC] The POC for creating entity centric indices using asset defintions --- package.json | 1 + tsconfig.base.json | 2 + x-pack/packages/kbn-oam-schema/README.md | 3 + x-pack/packages/kbn-oam-schema/index.ts | 10 ++ x-pack/packages/kbn-oam-schema/jest.config.js | 12 ++ x-pack/packages/kbn-oam-schema/kibana.jsonc | 5 + x-pack/packages/kbn-oam-schema/package.json | 6 + .../kbn-oam-schema/src/schema/asset.ts | 22 +++ .../kbn-oam-schema/src/schema/common.ts | 99 ++++++++++++++ .../src/schema/oam_definition.ts | 42 ++++++ x-pack/packages/kbn-oam-schema/tsconfig.json | 20 +++ .../asset_manager/common/constants_oam.ts | 10 ++ .../server/lib/manage_index_templates.ts | 33 +++-- .../oam/create_and_install_ingest_pipeline.ts | 39 ++++++ .../lib/oam/create_and_install_transform.ts | 30 +++++ .../server/lib/oam/delete_index.ts | 24 ++++ .../server/lib/oam/delete_ingest_pipeline.ts | 27 ++++ .../server/lib/oam/delete_oam_definition.ts | 31 +++++ .../lib/oam/errors/invalid_transform_error.ts | 13 ++ .../lib/oam/errors/oam_id_conflict_error.ts | 18 +++ .../server/lib/oam/errors/oam_not_found.ts | 13 ++ .../lib/oam/errors/oam_security_exception.ts | 18 +++ .../oam/helpers/fixtures/oam_definition.ts | 47 +++++++ .../lib/oam/helpers/generate_index_name.ts | 13 ++ .../get_elasticsearch_query_or_throw.ts | 17 +++ .../server/lib/oam/helpers/retry.ts | 53 ++++++++ .../generate_processors.test.ts.snap | 81 ++++++++++++ .../generate_ingest_pipeline_id.ts | 13 ++ .../generate_processors.test.ts | 16 +++ .../ingest_pipeline/generate_processors.ts | 109 +++++++++++++++ .../server/lib/oam/preview_transform.ts | 28 ++++ .../server/lib/oam/read_oam_definition.ts | 36 +++++ .../server/lib/oam/save_oam_definition.ts | 34 +++++ .../server/lib/oam/start_transform.ts | 28 ++++ .../lib/oam/stop_and_delete_transform.ts | 37 ++++++ .../generate_transform.test.ts.snap | 125 ++++++++++++++++++ .../generate_metadata_aggregations.ts | 26 ++++ .../transform/generate_metric_aggregations.ts | 118 +++++++++++++++++ .../oam/transform/generate_transform.test.ts | 16 +++ .../lib/oam/transform/generate_transform.ts | 78 +++++++++++ .../oam/transform/generate_transform_id.ts | 13 ++ .../asset_manager/server/plugin.ts | 33 ++++- .../asset_manager/server/routes/index.ts | 21 ++- .../asset_manager/server/routes/oam/create.ts | 59 +++++++++ .../asset_manager/server/routes/oam/delete.ts | 56 ++++++++ .../asset_manager/server/routes/oam/reset.ts | 64 +++++++++ .../asset_manager/server/routes/types.ts | 2 + .../server/saved_objects/index.ts | 8 ++ .../server/saved_objects/oam_definition.ts | 40 ++++++ .../server/templates/assets_template.ts | 2 + .../server/templates/components/asset.ts | 44 ++++++ .../server/templates/components/base.ts | 32 +++++ .../server/templates/components/event.ts | 29 ++++ .../server/templates/oam_template.ts | 57 ++++++++ .../asset_manager/tsconfig.json | 3 +- yarn.lock | 4 + 56 files changed, 1797 insertions(+), 23 deletions(-) create mode 100644 x-pack/packages/kbn-oam-schema/README.md create mode 100644 x-pack/packages/kbn-oam-schema/index.ts create mode 100644 x-pack/packages/kbn-oam-schema/jest.config.js create mode 100644 x-pack/packages/kbn-oam-schema/kibana.jsonc create mode 100644 x-pack/packages/kbn-oam-schema/package.json create mode 100644 x-pack/packages/kbn-oam-schema/src/schema/asset.ts create mode 100644 x-pack/packages/kbn-oam-schema/src/schema/common.ts create mode 100644 x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts create mode 100644 x-pack/packages/kbn-oam-schema/tsconfig.json create mode 100644 x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts diff --git a/package.json b/package.json index 4e2a2f6df494cd..bd7d80afa48168 100644 --- a/package.json +++ b/package.json @@ -621,6 +621,7 @@ "@kbn/newsfeed-test-plugin": "link:test/common/plugins/newsfeed", "@kbn/no-data-page-plugin": "link:src/plugins/no_data_page", "@kbn/notifications-plugin": "link:x-pack/plugins/notifications", + "@kbn/oam-schema": "link:x-pack/packages/kbn-oam-schema", "@kbn/object-versioning": "link:packages/kbn-object-versioning", "@kbn/observability-ai-assistant-app-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_app", "@kbn/observability-ai-assistant-management-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_management", diff --git a/tsconfig.base.json b/tsconfig.base.json index 814033d8be3949..7d79f30f998bd8 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1194,6 +1194,8 @@ "@kbn/no-data-page-plugin/*": ["src/plugins/no_data_page/*"], "@kbn/notifications-plugin": ["x-pack/plugins/notifications"], "@kbn/notifications-plugin/*": ["x-pack/plugins/notifications/*"], + "@kbn/oam-schema": ["x-pack/packages/kbn-oam-schema"], + "@kbn/oam-schema/*": ["x-pack/packages/kbn-oam-schema/*"], "@kbn/object-versioning": ["packages/kbn-object-versioning"], "@kbn/object-versioning/*": ["packages/kbn-object-versioning/*"], "@kbn/observability-ai-assistant-app-plugin": ["x-pack/plugins/observability_solution/observability_ai_assistant_app"], diff --git a/x-pack/packages/kbn-oam-schema/README.md b/x-pack/packages/kbn-oam-schema/README.md new file mode 100644 index 00000000000000..c0e1da11cf9a8d --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/README.md @@ -0,0 +1,3 @@ +# @kbn/oam-schema + +Empty package generated by @kbn/generate diff --git a/x-pack/packages/kbn-oam-schema/index.ts b/x-pack/packages/kbn-oam-schema/index.ts new file mode 100644 index 00000000000000..2d13c00271ed11 --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './src/schema/oam_definition'; +export * from './src/schema/asset'; +export * from './src/schema/common'; diff --git a/x-pack/packages/kbn-oam-schema/jest.config.js b/x-pack/packages/kbn-oam-schema/jest.config.js new file mode 100644 index 00000000000000..3c40ad7480e306 --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/packages/kbn-oam-schema'], +}; diff --git a/x-pack/packages/kbn-oam-schema/kibana.jsonc b/x-pack/packages/kbn-oam-schema/kibana.jsonc new file mode 100644 index 00000000000000..40b6be5c530198 --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/oam-schema", + "owner": "@elastic/obs-knowledge-team" +} diff --git a/x-pack/packages/kbn-oam-schema/package.json b/x-pack/packages/kbn-oam-schema/package.json new file mode 100644 index 00000000000000..7cf91a9d0fc6e0 --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/oam-schema", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts new file mode 100644 index 00000000000000..9238e8d8cd12dd --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { arrayOfStringsSchema } from './common'; + +export const assetSchema = z.intersection( + z.object({ + asset: z.object({ + id: z.string(), + indexPattern: arrayOfStringsSchema, + category: arrayOfStringsSchema, + identityField: arrayOfStringsSchema, + metric: z.record(z.string(), z.number()), + }), + }), + z.record(z.string(), z.string().or(z.number())) +); diff --git a/x-pack/packages/kbn-oam-schema/src/schema/common.ts b/x-pack/packages/kbn-oam-schema/src/schema/common.ts new file mode 100644 index 00000000000000..2bf40e385d6028 --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/src/schema/common.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import moment from 'moment'; + +export enum AssetType { + service = 'service', + host = 'host', + pod = 'pod', + node = 'node', +} + +export const arrayOfStringsSchema = z.array(z.string()); + +export const assetTypeSchema = z.nativeEnum(AssetType); + +export enum BasicAggregations { + avg = 'avg', + max = 'max', + min = 'min', + sum = 'sum', + cardinality = 'cardinality', + last_value = 'last_value', + std_deviation = 'std_deviation', +} + +export const basicAggregationsSchema = z.nativeEnum(BasicAggregations); + +const metricNameSchema = z + .string() + .length(1) + .regex(/[a-zA-Z]/) + .toUpperCase(); + +export const filterSchema = z.optional(z.string()); + +export const basicMetricWithFieldSchema = z.object({ + name: metricNameSchema, + aggregation: basicAggregationsSchema, + field: z.string(), + filter: filterSchema, +}); + +export const docCountMetricSchema = z.object({ + name: metricNameSchema, + aggregation: z.literal('doc_count'), + filter: filterSchema, +}); + +export const durationSchema = z + .string() + .regex(/\d+[m|d|s|h]/) + .transform((val: string) => { + const parts = val.match(/(\d+)([m|s|h|d])/); + if (parts === null) { + throw new Error('Unable to parse duration'); + } + const value = parseInt(parts[1], 10); + const unit = parts[2] as 'm' | 's' | 'h' | 'd'; + const duration = moment.duration(value, unit); + return { ...duration, toJSON: () => val }; + }); + +export const percentileMetricSchema = z.object({ + name: metricNameSchema, + aggregation: z.literal('percentile'), + field: z.string(), + percentile: z.number(), + filter: filterSchema, +}); + +export const metricSchema = z.discriminatedUnion('aggregation', [ + basicMetricWithFieldSchema, + docCountMetricSchema, + percentileMetricSchema, +]); + +export type Metric = z.infer; + +export const keyMetricSchema = z.object({ + name: z.string(), + metrics: z.array(metricSchema), + equation: z.string(), +}); + +export type KeyMetric = z.infer; + +export const metadataSchema = z + .object({ + source: z.string(), + destination: z.optional(z.string()), + limit: z.optional(z.number().default(1000)), + }) + .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); diff --git a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts new file mode 100644 index 00000000000000..b0aaa39344727c --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { + arrayOfStringsSchema, + assetTypeSchema, + keyMetricSchema, + metadataSchema, + filterSchema, + durationSchema, +} from './common'; + +export const oamDefinitionSchema = z.object({ + id: z.string().regex(/^[\w-]+$/), + name: z.string(), + description: z.optional(z.string()), + type: assetTypeSchema, + filter: filterSchema, + indexPatterns: arrayOfStringsSchema, + identityFields: arrayOfStringsSchema, + identityTemplate: z.string(), + categories: arrayOfStringsSchema, + metadata: z.optional(z.array(metadataSchema)), + metrics: z.optional(z.array(keyMetricSchema)), + staticFields: z.optional(z.record(z.string(), z.string())), + lookback: durationSchema, + timestampField: z.string(), + settings: z.optional( + z.object({ + syncField: z.optional(z.string()), + syncDelay: z.optional(z.string()), + frequency: z.optional(z.string()), + }) + ), +}); + +export type OAMDefinition = z.infer; diff --git a/x-pack/packages/kbn-oam-schema/tsconfig.json b/x-pack/packages/kbn-oam-schema/tsconfig.json new file mode 100644 index 00000000000000..e5e31852eb412b --- /dev/null +++ b/x-pack/packages/kbn-oam-schema/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/std", + "@kbn/io-ts-utils" + ] +} diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts new file mode 100644 index 00000000000000..e5a9f031763221 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const OAM_VERSION = 'v1'; +export const OAM_BASE_PREFIX = `.oam-observability.asset-${OAM_VERSION}`; +export const OAM_TRANSFORM_PREFIX = `oam-observability-asset-${OAM_VERSION}`; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts index b853d7a360e1d1..b364e63ff9a1f3 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts @@ -6,6 +6,7 @@ */ import { + ClusterPutComponentTemplateRequest, IndicesGetIndexTemplateResponse, IndicesPutIndexTemplateRequest, } from '@elastic/elasticsearch/lib/api/types'; @@ -47,21 +48,18 @@ function templateExists( }); } -// interface IndexPatternJson { -// index_patterns: string[]; -// name: string; -// template: { -// mappings: Record; -// settings: Record; -// }; -// } - interface TemplateManagementOptions { esClient: ElasticsearchClient; template: IndicesPutIndexTemplateRequest; logger: Logger; } +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + export async function maybeCreateTemplate({ esClient, template, @@ -93,9 +91,6 @@ export async function maybeCreateTemplate({ } export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { - const pattern = ASSETS_INDEX_PREFIX + '*'; - template.index_patterns = [pattern]; - try { await esClient.indices.putIndexTemplate(template); } catch (error: any) { @@ -108,3 +103,17 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan ); logger.debug(`Asset manager index template: ${JSON.stringify(template)}`); } + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await esClient.cluster.putComponentTemplate(component); + } catch (error: any) { + logger.error(`Error updating asset manager component template: ${error.message}`); + return; + } + + logger.info( + `Asset manager component template is up to date (use debug logging to see what was installed)` + ); + logger.debug(`Asset manager component template: ${JSON.stringify(component)}`); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts new file mode 100644 index 00000000000000..878c5475ee6da8 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateProcessors } from './ingest_pipeline/generate_processors'; +import { retryTransientEsErrors } from './helpers/retry'; +import { OAMSecurityException } from './errors/oam_security_exception'; +import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; + +export async function createAndInstallIngestPipeline( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const processors = generateProcessors(definition); + const id = generateIngestPipelineId(definition); + try { + await retryTransientEsErrors( + () => + esClient.ingest.putPipeline({ + id, + processors, + }), + { logger } + ); + } catch (e) { + logger.error(`Cannot create OAM tranform for [${definition.id}] asset defintion`); + if (e.meta?.body?.error?.type === 'security_exception') { + throw new OAMSecurityException(e.meta.body.error.reason, definition); + } + throw e; + } + return id; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts new file mode 100644 index 00000000000000..ff41aadab7188b --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateTransform } from './transform/generate_transform'; +import { retryTransientEsErrors } from './helpers/retry'; +import { OAMSecurityException } from './errors/oam_security_exception'; + +export async function createAndInstallTransform( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const transform = generateTransform(definition); + try { + await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger }); + } catch (e) { + logger.error(`Cannot create OAM transform for [${definition.id}] asset definition`); + if (e.meta?.body?.error?.type === 'security_exception') { + throw new OAMSecurityException(e.meta.body.error.reason, definition); + } + throw e; + } + return transform.transform_id; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts new file mode 100644 index 00000000000000..daa1452fd301fd --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateIndexName } from './helpers/generate_index_name'; + +export async function deleteIndex( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const indexName = generateIndexName(definition); + try { + esClient.indices.delete({ index: indexName, ignore_unavailable: true }); + } catch (e) { + logger.error(`Unable to remove OAM Defintion index [${definition.id}}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts new file mode 100644 index 00000000000000..5058c4ad25e8bf --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; +import { retryTransientEsErrors } from './helpers/retry'; + +export async function deleteIngestPipeline( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const pipelineId = generateIngestPipelineId(definition); + try { + await retryTransientEsErrors(() => + esClient.ingest.deletePipeline({ id: pipelineId }, { ignore: [404] }) + ); + } catch (e) { + logger.error(`Unable to delete ingest pipeline [${pipelineId}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts new file mode 100644 index 00000000000000..d255be5e9173b9 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; +import { OAMNotFound } from './errors/oam_not_found'; + +export async function deleteOAMDefinition( + soClient: SavedObjectsClientContract, + definition: OAMDefinition, + logger: Logger +) { + const response = await soClient.find({ + type: SO_OAM_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 0) { + logger.error(`Unable to delete OAM Definition [${definition.id}] because it doesn't exist.`); + throw new OAMNotFound(`OAM Defintion with [${definition.id}] not found.`); + } + + await soClient.delete(SO_OAM_DEFINITION_TYPE, response.saved_objects[0].id); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts new file mode 100644 index 00000000000000..5d1c98d5dc3ae7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class InvalidTransformError extends Error { + constructor(message: string) { + super(message); + this.name = 'InvalidTransformError'; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts new file mode 100644 index 00000000000000..af598d37d01f11 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; + +export class OAMIdConflict extends Error { + public defintion: OAMDefinition; + + constructor(message: string, def: OAMDefinition) { + super(message); + this.name = 'OAMIdConflict'; + this.defintion = def; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts new file mode 100644 index 00000000000000..fc00c97e62f4f3 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class OAMNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'OAMNotFound'; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts new file mode 100644 index 00000000000000..f2f5dfea2570e2 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; + +export class OAMSecurityException extends Error { + public defintion: OAMDefinition; + + constructor(message: string, def: OAMDefinition) { + super(message); + this.name = 'OAMSecurityException'; + this.defintion = def; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts new file mode 100644 index 00000000000000..023adc99e4acc9 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { oamDefinitionSchema } from '@kbn/oam-schema'; +export const oamDefinition = oamDefinitionSchema.parse({ + id: 'admin-console-logs-service', + name: 'Services for Admin Console', + type: 'service', + indexPatterns: ['kbn-data-forge-fake_stack.*'], + timestampField: '@timestamp', + categories: ['service', 'logs'], + identityFields: ['log.logger'], + identityTemplate: 'service:{{log.logger}}', + metadata: ['tags', 'host.name', 'kubernetes.pod.name'], + staticFields: { + projectId: '1234', + }, + lookback: '5m', + metrics: [ + { + name: 'logRate', + equation: 'A / 5', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: *', + }, + ], + }, + { + name: 'errorRate', + equation: 'A / 5', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: error', + }, + ], + }, + ], +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts new file mode 100644 index 00000000000000..8ca63de51db074 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; +import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; + +export function generateIndexName(definition: OAMDefinition) { + return `${OAM_BASE_PREFIX}.${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts new file mode 100644 index 00000000000000..3b24344efd83b9 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; +import { InvalidTransformError } from '../errors/invalid_transform_error'; + +export function getElasticsearchQueryOrThrow(kuery: string) { + try { + return toElasticsearchQuery(fromKueryExpression(kuery)); + } catch (err) { + throw new InvalidTransformError(`Invalid KQL: ${kuery}`); + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts new file mode 100644 index 00000000000000..421289d1c04791 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setTimeout } from 'timers/promises'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import type { Logger } from '@kbn/logging'; + +const MAX_ATTEMPTS = 5; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: any) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +/** + * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. + * Should only be used to wrap operations that are idempotent and can be safely executed more than once. + */ +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger?: Logger; attempt?: number } = {} +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... + + logger?.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + await setTimeout(retryDelaySec * 1000); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap new file mode 100644 index 00000000000000..777aa1e9987efb --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -0,0 +1,81 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateProcessors(definition) should genearte a valid pipeline 1`] = ` +Array [ + Object { + "set": Object { + "field": "event.ingested", + "value": "{{{_ingest.timestamp}}}", + }, + }, + Object { + "set": Object { + "field": "asset.definitionId", + "value": "admin-console-logs-service", + }, + }, + Object { + "set": Object { + "field": "asset.indexPattern", + "value": "[\\"kbn-data-forge-fake_stack.*\\"]", + }, + }, + Object { + "json": Object { + "field": "asset.indexPattern", + }, + }, + Object { + "set": Object { + "field": "asset.category", + "value": "[\\"service\\",\\"logs\\"]", + }, + }, + Object { + "json": Object { + "field": "asset.category", + }, + }, + Object { + "set": Object { + "field": "asset.id", + "value": "service:{{asset.identity.log.logger}}", + }, + }, + Object { + "set": Object { + "field": "projectId", + "value": "1234", + }, + }, + Object { + "script": Object { + "source": "if (ctx.asset?.metadata?.tags != null) { + ctx[\\"tags\\"] = ctx.asset.metadata.tags.keySet(); +} +if (ctx.asset?.metadata?.host?.name != null) { + ctx[\\"host\\"] = new HashMap(); + ctx[\\"host\\"][\\"name\\"] = ctx.asset.metadata.host.name.keySet(); +} +if (ctx.asset?.metadata?.kubernetes?.pod?.name != null) { + ctx[\\"kubernetes\\"] = new HashMap(); + ctx[\\"kubernetes\\"][\\"pod\\"] = new HashMap(); + ctx[\\"kubernetes\\"][\\"pod\\"][\\"name\\"] = ctx.asset.metadata.kubernetes.pod.name.keySet(); +} +", + }, + }, + Object { + "remove": Object { + "field": "asset.metadata", + "ignore_missing": true, + }, + }, + Object { + "set": Object { + "field": "_index", + "value": ".oam-observability.asset-v1.admin-console-logs-service", + }, + }, +] +`; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts new file mode 100644 index 00000000000000..b21a69954ce194 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; +import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; + +export function generateIngestPipelineId(definition: OAMDefinition) { + return `${OAM_BASE_PREFIX}.${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts new file mode 100644 index 00000000000000..b0e15f054ddb09 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { generateProcessors } from './generate_processors'; +import { oamDefinition } from '../helpers/fixtures/oam_definition'; + +describe('generateProcessors(definition)', () => { + it('should genearte a valid pipeline', () => { + const processors = generateProcessors(oamDefinition); + expect(processors).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts new file mode 100644 index 00000000000000..ba4e85875aecf8 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateIndexName } from '../helpers/generate_index_name'; + +function createIdTemplate(definition: OAMDefinition) { + return definition.identityFields.reduce((template, field) => { + return template.replaceAll(field, `asset.identity.${field}`); + }, definition.identityTemplate); +} + +function mapDesitnationToPainless(destination: string, source: string) { + const fieldParts = destination.split('.'); + return fieldParts.reduce((acc, _part, currentIndex, parts) => { + if (currentIndex + 1 === parts.length) { + return `${acc}\n ctx${parts + .map((s) => `["${s}"]`) + .join('')} = ctx.asset.metadata.${source}.keySet();`; + } + return `${acc}\n ctx${parts + .slice(0, currentIndex + 1) + .map((s) => `["${s}"]`) + .join('')} = new HashMap();`; + }, ''); +} + +function createMetadataPainlessScript(definition: OAMDefinition) { + if (!definition.metadata) { + return ''; + } + return definition.metadata.reduce((script, def) => { + const source = def.source; + const destination = def.destination || def.source; + return `${script}if (ctx.asset?.metadata?.${source.replaceAll( + '.', + '?.' + )} != null) {${mapDesitnationToPainless(destination, source)}\n}\n`; + }, ''); +} + +export function generateProcessors(definition: OAMDefinition) { + return [ + { + set: { + field: 'event.ingested', + value: '{{{_ingest.timestamp}}}', + }, + }, + { + set: { + field: 'asset.definitionId', + value: definition.id, + }, + }, + { + set: { + field: 'asset.indexPattern', + value: JSON.stringify(definition.indexPatterns), + }, + }, + { + json: { + field: 'asset.indexPattern', + }, + }, + { + set: { + field: 'asset.category', + value: JSON.stringify(definition.categories), + }, + }, + { + json: { + field: 'asset.category', + }, + }, + { + set: { + field: 'asset.id', + value: createIdTemplate(definition), + }, + }, + ...(definition.staticFields != null + ? Object.keys(definition.staticFields).map((field) => ({ + set: { field, value: definition.staticFields![field] }, + })) + : []), + ...(definition.metadata != null + ? [{ script: { source: createMetadataPainlessScript(definition) } }] + : []), + { + remove: { + field: 'asset.metadata', + ignore_missing: true, + }, + }, + { + set: { + field: '_index', + value: generateIndexName(definition), + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts new file mode 100644 index 00000000000000..3403cf41c8afa7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { retryTransientEsErrors } from './helpers/retry'; +import { generateTransformId } from './transform/generate_transform_id'; + +export async function previewTransform( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const transformId = generateTransformId(definition); + try { + await retryTransientEsErrors( + () => esClient.transform.previewTransform({ transform_id: transformId }), + { logger } + ); + } catch (err) { + logger.error(`Cannot preview SLO transform [${transformId}]`); + throw err; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts new file mode 100644 index 00000000000000..411f69b25d60a7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { OAMDefinition, oamDefinitionSchema } from '@kbn/oam-schema'; +import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; +import { OAMNotFound } from './errors/oam_not_found'; + +export async function readOAMDefinition( + soClient: SavedObjectsClientContract, + id: string, + logger: Logger +) { + const response = await soClient.find({ + type: SO_OAM_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${id})`, + }); + if (response.total === 0) { + const message = `Unable to find OAM Defintion with [${id}]`; + logger.error(message); + throw new OAMNotFound(message); + } + + try { + return oamDefinitionSchema.parse(response.saved_objects[0].attributes); + } catch (e) { + logger.error(`Unable to parse OAM Defintion with [${id}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts new file mode 100644 index 00000000000000..e2ac929bcb7011 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; +import { OAMIdConflict } from './errors/oam_id_conflict_error'; + +export async function saveOAMDefinition( + soClient: SavedObjectsClientContract, + definition: OAMDefinition +): Promise { + const response = await soClient.find({ + type: SO_OAM_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 1) { + throw new OAMIdConflict(`OAM Defintion with [${definition.id}] already exists.`, definition); + } + + await soClient.create(SO_OAM_DEFINITION_TYPE, definition, { + id: definition.id, + overwrite: true, + }); + + return definition; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts new file mode 100644 index 00000000000000..5a1960b0435eb0 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { retryTransientEsErrors } from './helpers/retry'; +import { generateTransformId } from './transform/generate_transform_id'; + +export async function startTransform( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const transformId = generateTransformId(definition); + try { + await retryTransientEsErrors( + () => esClient.transform.startTransform({ transform_id: transformId }, { ignore: [409] }), + { logger } + ); + } catch (err) { + logger.error(`Cannot start SLO transform [${transformId}]`); + throw err; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts new file mode 100644 index 00000000000000..ecf7ecc0b4eb79 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; +import { generateTransformId } from './transform/generate_transform_id'; +import { retryTransientEsErrors } from './helpers/retry'; + +export async function stopAndDeleteTransform( + esClient: ElasticsearchClient, + definition: OAMDefinition, + logger: Logger +) { + const transformId = generateTransformId(definition); + try { + await retryTransientEsErrors( + async () => { + await esClient.transform.stopTransform( + { transform_id: transformId, wait_for_completion: true, force: true }, + { ignore: [409] } + ); + await esClient.transform.deleteTransform( + { transform_id: transformId, force: true }, + { ignore: [404] } + ); + }, + { logger } + ); + } catch (e) { + logger.error(`Cannot stop or delete OAM transform [${transformId}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap new file mode 100644 index 00000000000000..cdb172a6afcd2f --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap @@ -0,0 +1,125 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateTransform(definition) should generate a valid summary transform 1`] = ` +Object { + "defer_validation": true, + "dest": Object { + "index": ".oam-observability.asset-v1.noop", + "pipeline": ".oam-observability.asset-v1.admin-console-logs-service", + }, + "frequency": "1m", + "pivot": Object { + "aggs": Object { + "_errorRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "log.level": "error", + }, + }, + ], + }, + }, + }, + "_logRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "log.level", + }, + }, + ], + }, + }, + }, + "asset.latestTimestamp": Object { + "max": Object { + "field": "@timestamp", + }, + }, + "asset.metadata.host.name": Object { + "terms": Object { + "field": "host.name", + "size": 1000, + }, + }, + "asset.metadata.kubernetes.pod.name": Object { + "terms": Object { + "field": "kubernetes.pod.name", + "size": 1000, + }, + }, + "asset.metadata.tags": Object { + "terms": Object { + "field": "tags", + "size": 1000, + }, + }, + "asset.metric.errorRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_errorRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A / 5", + }, + }, + }, + "asset.metric.logRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_logRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A / 5", + }, + }, + }, + }, + "group_by": Object { + "asset.identity.log.logger": Object { + "terms": Object { + "field": "log.logger", + }, + }, + }, + }, + "settings": Object { + "deduce_mappings": false, + "unattended": true, + }, + "source": Object { + "index": Array [ + "kbn-data-forge-fake_stack.*", + ], + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-5m", + }, + }, + }, + ], + }, + }, + }, + "sync": Object { + "time": Object { + "delay": "60s", + "field": "@timestamp", + }, + }, + "transform_id": "oam-observability-asset-v1-admin-console-logs-service", +} +`; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts new file mode 100644 index 00000000000000..ee483fa4dc198e --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; + +export function generateMetadataAggregations(definition: OAMDefinition) { + if (!definition.metadata) { + return {}; + } + return definition.metadata.reduce( + (aggs, metadata) => ({ + ...aggs, + [`asset.metadata.${metadata.destination ?? metadata.source}`]: { + terms: { + field: metadata.source, + size: metadata.limit ?? 1000, + }, + }, + }), + {} + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts new file mode 100644 index 00000000000000..4c3611e10a63e9 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KeyMetric, Metric, OAMDefinition } from '@kbn/oam-schema'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { InvalidTransformError } from '../errors/invalid_transform_error'; + +function buildAggregation(metric: Metric, timestampField: string) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return {}; + case 'std_deviation': + return { + extended_stats: { field: metric.field }, + }; + case 'percentile': + if (metric.percentile == null) { + throw new InvalidTransformError( + 'You must provide a percentile value for percentile aggregations.' + ); + } + return { + percentiles: { + field: metric.field, + percents: [metric.percentile], + keyed: true, + }, + }; + case 'last_value': + return { + top_metrics: { + metrics: { field: metric.field }, + sort: { [timestampField]: 'desc' }, + }, + }; + default: + if (metric.field == null) { + throw new InvalidTransformError('You must provide a field for basic metric aggregations.'); + } + return { + [aggregation]: { field: metric.field }, + }; + } +} + +function buildMetricAggregations(keyMetric: KeyMetric, timestampField: string) { + return keyMetric.metrics.reduce((acc, metric) => { + const filter = metric.filter ? getElasticsearchQueryOrThrow(metric.filter) : { match_all: {} }; + const aggs = { metric: buildAggregation(metric, timestampField) }; + return { + ...acc, + [`_${keyMetric.name}_${metric.name}`]: { + filter, + ...(metric.aggregation !== 'doc_count' ? { aggs } : {}), + }, + }; + }, {}); +} + +function buildBucketPath(prefix: string, metric: Metric) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return `${prefix}>_count`; + case 'std_deviation': + return `${prefix}>metric[std_deviation]`; + case 'percentile': + return `${prefix}>metric[${metric.percentile}]`; + case 'last_value': + return `${prefix}>metric[${metric.field}]`; + default: + return `${prefix}>metric`; + } +} + +function convertEquationToPainless(bucketsPath: Record, equation: string) { + const workingEquation = equation || Object.keys(bucketsPath).join(' + '); + return Object.keys(bucketsPath).reduce((acc, key) => { + return acc.replaceAll(key, `params.${key}`); + }, workingEquation); +} + +function buildMetricEquation(keyMetric: KeyMetric) { + const bucketsPath = keyMetric.metrics.reduce( + (acc, metric) => ({ + ...acc, + [metric.name]: buildBucketPath(`_${keyMetric.name}_${metric.name}`, metric), + }), + {} + ); + return { + bucket_script: { + buckets_path: bucketsPath, + script: { + source: convertEquationToPainless(bucketsPath, keyMetric.equation), + lang: 'painless', + }, + }, + }; +} + +export function generateMetricAggregations(definition: OAMDefinition) { + if (!definition.metrics) { + return {}; + } + return definition.metrics.reduce((aggs, keyMetric) => { + return { + ...aggs, + ...buildMetricAggregations(keyMetric, definition.timestampField), + [`asset.metric.${keyMetric.name}`]: buildMetricEquation(keyMetric), + }; + }, {}); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts new file mode 100644 index 00000000000000..78ce7d8a3c6195 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { oamDefinition } from '../helpers/fixtures/oam_definition'; +import { generateTransform } from './generate_transform'; + +describe('generateTransform(definition)', () => { + it('should generate a valid summary transform', () => { + const transform = generateTransform(oamDefinition); + expect(transform).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts new file mode 100644 index 00000000000000..81efdfae800b1d --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; +import { + QueryDslQueryContainer, + TransformPutTransformRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { generateMetricAggregations } from './generate_metric_aggregations'; +import { OAM_BASE_PREFIX, OAM_TRANSFORM_PREFIX } from '../../../../common/constants_oam'; +import { generateMetadataAggregations } from './generate_metadata_aggregations'; + +export function generateTransform(definition: OAMDefinition): TransformPutTransformRequest { + const filter: QueryDslQueryContainer[] = [ + { + range: { + [definition.timestampField]: { + gte: `now-${definition.lookback.toJSON()}`, + }, + }, + }, + ]; + + if (definition.filter) { + filter.push(getElasticsearchQueryOrThrow(definition.filter)); + } + + return { + transform_id: `${OAM_TRANSFORM_PREFIX}-${definition.id}`, + defer_validation: true, + source: { + index: definition.indexPatterns, + query: { + bool: { + filter, + }, + }, + }, + dest: { + index: `${OAM_BASE_PREFIX}.noop`, + pipeline: `${OAM_BASE_PREFIX}.${definition.id}`, + }, + frequency: definition.settings?.frequency || '1m', + sync: { + time: { + field: definition.settings?.syncField ?? definition.timestampField, + delay: definition.settings?.syncDelay ?? '60s', + }, + }, + settings: { + deduce_mappings: false, + unattended: true, + }, + pivot: { + group_by: definition.identityFields.reduce( + (acc, field) => ({ + ...acc, + [`asset.identity.${field}`]: { terms: { field } }, + }), + {} + ), + aggs: { + ...generateMetricAggregations(definition), + ...generateMetadataAggregations(definition), + 'asset.latestTimestamp': { + max: { + field: definition.timestampField, + }, + }, + }, + }, + }; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts new file mode 100644 index 00000000000000..370d896b5c83b0 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OAMDefinition } from '@kbn/oam-schema'; +import { OAM_TRANSFORM_PREFIX } from '../../../../common/constants_oam'; + +export function generateTransformId(definition: OAMDefinition) { + return `${OAM_TRANSFORM_PREFIX}-${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts index 73b2cbe87c1386..d11f59cb8ebd03 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts @@ -15,12 +15,17 @@ import { Logger, } from '@kbn/core/server'; -import { upsertTemplate } from './lib/manage_index_templates'; +import { upsertComponent, upsertTemplate } from './lib/manage_index_templates'; import { setupRoutes } from './routes'; import { assetsIndexTemplateConfig } from './templates/assets_template'; import { AssetClient } from './lib/asset_client'; import { AssetManagerPluginSetupDependencies, AssetManagerPluginStartDependencies } from './types'; import { AssetManagerConfig, configSchema, exposeToBrowserConfig } from '../common/config'; +import { oamBaseComponentTemplateConfig } from './templates/components/base'; +import { oamAssetComponentTemplateConfig } from './templates/components/asset'; +import { oamEventComponentTemplateConfig } from './templates/components/event'; +import { oamIndexTemplateConfig } from './templates/oam_template'; +import { oamDefinition } from './saved_objects'; export type AssetManagerServerPluginSetup = ReturnType; export type AssetManagerServerPluginStart = ReturnType; @@ -56,6 +61,8 @@ export class AssetManagerServerPlugin this.logger.info('Server is enabled'); + core.savedObjects.registerType(oamDefinition); + const assetClient = new AssetClient({ sourceIndices: this.config.sourceIndices, getApmIndices: plugins.apmDataAccess.getApmIndices, @@ -63,7 +70,7 @@ export class AssetManagerServerPlugin }); const router = core.http.createRouter(); - setupRoutes({ router, assetClient }); + setupRoutes({ router, assetClient, logger: this.logger }); return { assetClient, @@ -76,12 +83,32 @@ export class AssetManagerServerPlugin return; } + const esClient = core.elasticsearch.client.asInternalUser; upsertTemplate({ - esClient: core.elasticsearch.client.asInternalUser, + esClient, template: assetsIndexTemplateConfig, logger: this.logger, }).catch(() => {}); // it shouldn't reject, but just in case + // Install OAM compoent templates and index template + Promise.all([ + upsertComponent({ esClient, logger: this.logger, component: oamBaseComponentTemplateConfig }), + upsertComponent({ + esClient, + logger: this.logger, + component: oamEventComponentTemplateConfig, + }), + upsertComponent({ + esClient, + logger: this.logger, + component: oamAssetComponentTemplateConfig, + }), + ]) + .then(() => + upsertTemplate({ esClient, logger: this.logger, template: oamIndexTemplateConfig }) + ) + .catch(() => {}); + return {}; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts index 52d3198bb8a9f9..27a3bbc9d52edd 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts @@ -14,16 +14,23 @@ import { hostsRoutes } from './assets/hosts'; import { servicesRoutes } from './assets/services'; import { containersRoutes } from './assets/containers'; import { podsRoutes } from './assets/pods'; +import { createOAMDefinitionRoute } from './oam/create'; +import { deleteOAMDefinitionRoute } from './oam/delete'; +import { resetOAMDefinitionRoute } from './oam/reset'; export function setupRoutes({ router, assetClient, + logger, }: SetupRouteOptions) { - pingRoute({ router, assetClient }); - sampleAssetsRoutes({ router, assetClient }); - assetsRoutes({ router, assetClient }); - hostsRoutes({ router, assetClient }); - servicesRoutes({ router, assetClient }); - containersRoutes({ router, assetClient }); - podsRoutes({ router, assetClient }); + pingRoute({ router, assetClient, logger }); + sampleAssetsRoutes({ router, assetClient, logger }); + assetsRoutes({ router, assetClient, logger }); + hostsRoutes({ router, assetClient, logger }); + servicesRoutes({ router, assetClient, logger }); + containersRoutes({ router, assetClient, logger }); + podsRoutes({ router, assetClient, logger }); + createOAMDefinitionRoute({ router, assetClient, logger }); + deleteOAMDefinitionRoute({ router, assetClient, logger }); + resetOAMDefinitionRoute({ router, assetClient, logger }); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts new file mode 100644 index 00000000000000..69442b6b281402 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { OAMDefinition, oamDefinitionSchema } from '@kbn/oam-schema'; +import { stringifyZodError } from '@kbn/zod-helpers'; +import { SetupRouteOptions } from '../types'; +import { saveOAMDefinition } from '../../lib/oam/save_oam_definition'; +import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install_ingest_pipeline'; +import { OAMIdConflict } from '../../lib/oam/errors/oam_id_conflict_error'; +import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; +import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; +import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; +import { startTransform } from '../../lib/oam/start_transform'; + +export function createOAMDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.post( + { + path: '/api/oam', + validate: { + body: (body, res) => { + try { + return res.ok(oamDefinitionSchema.parse(body)); + } catch (e) { + return res.badRequest(stringifyZodError(e)); + } + }, + }, + }, + async (context, req, res) => { + try { + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const definition = await saveOAMDefinition(soClient, req.body); + await createAndInstallIngestPipeline(esClient, definition, logger); + await createAndInstallTransform(esClient, definition, logger); + await startTransform(esClient, definition, logger); + + return res.ok({ body: definition }); + } catch (e) { + if (e instanceof OAMIdConflict) { + return res.conflict({ body: e }); + } + if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts new file mode 100644 index 00000000000000..ea8797a3ef2c62 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { schema } from '@kbn/config-schema'; +import { SetupRouteOptions } from '../types'; +import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; +import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; +import { readOAMDefinition } from '../../lib/oam/read_oam_definition'; +import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; +import { deleteIndex } from '../../lib/oam/delete_index'; +import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; +import { OAMNotFound } from '../../lib/oam/errors/oam_not_found'; + +export function deleteOAMDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.delete<{ id: string }, unknown, unknown>( + { + path: '/api/oam/{id}', + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, req, res) => { + try { + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const definition = await readOAMDefinition(soClient, req.params.id, logger); + await stopAndDeleteTransform(esClient, definition, logger); + await deleteIngestPipeline(esClient, definition, logger); + await deleteIndex(esClient, definition, logger); + await deleteOAMDefinition(soClient, definition, logger); + + return res.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof OAMNotFound) { + return res.notFound({ body: e }); + } + if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts new file mode 100644 index 00000000000000..b6482e6360104a --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { schema } from '@kbn/config-schema'; +import { SetupRouteOptions } from '../types'; +import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; +import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; +import { readOAMDefinition } from '../../lib/oam/read_oam_definition'; +import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; +import { deleteIndex } from '../../lib/oam/delete_index'; +import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install_ingest_pipeline'; +import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; +import { startTransform } from '../../lib/oam/start_transform'; +import { OAMNotFound } from '../../lib/oam/errors/oam_not_found'; + +export function resetOAMDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.post<{ id: string }, unknown, unknown>( + { + path: '/api/oam/{id}/_reset', + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, req, res) => { + try { + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const definition = await readOAMDefinition(soClient, req.params.id, logger); + + // Delete the transform and ingest pipeline + await stopAndDeleteTransform(esClient, definition, logger); + await deleteIngestPipeline(esClient, definition, logger); + await deleteIndex(esClient, definition, logger); + + // Recreate everything + await createAndInstallIngestPipeline(esClient, definition, logger); + await createAndInstallTransform(esClient, definition, logger); + await startTransform(esClient, definition, logger); + + return res.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof OAMNotFound) { + return res.notFound({ body: e }); + } + if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts index ae1b967a5b5967..561819d18cdae2 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts @@ -6,9 +6,11 @@ */ import { IRouter, RequestHandlerContextBase } from '@kbn/core-http-server'; +import { Logger } from '@kbn/core/server'; import { AssetClient } from '../lib/asset_client'; export interface SetupRouteOptions { router: IRouter; assetClient: AssetClient; + logger: Logger; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts new file mode 100644 index 00000000000000..58e990bc2b72b2 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { oamDefinition, SO_OAM_DEFINITION_TYPE } from './oam_definition'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts new file mode 100644 index 00000000000000..375831368494a5 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject, SavedObjectsType } from '@kbn/core/server'; +import { OAMDefinition } from '@kbn/oam-schema'; + +export const SO_OAM_DEFINITION_TYPE = 'oam-definition'; + +export const oamDefinition: SavedObjectsType = { + name: SO_OAM_DEFINITION_TYPE, + hidden: false, + namespaceType: 'multiple-isolated', + mappings: { + dynamic: false, + properties: { + id: { type: 'keyword' }, + name: { type: 'text' }, + description: { type: 'text' }, + type: { type: 'keyword' }, + filter: { type: 'keyword' }, + indexPatterns: { type: 'keyword' }, + identityFields: { type: 'keyword' }, + categories: { type: 'keyword' }, + metadata: { type: 'object' }, + metrics: { type: 'object' }, + staticFields: { type: 'object' }, + }, + }, + management: { + displayName: 'OAM Definition', + importableAndExportable: false, + getTitle(sloSavedObject: SavedObject) { + return `OAM: [${sloSavedObject.attributes.name}]`; + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts index 71d4058eba4c0e..b99ecc4559187e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts @@ -6,11 +6,13 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSETS_INDEX_PREFIX } from '../constants'; export const assetsIndexTemplateConfig: IndicesPutIndexTemplateRequest = { name: 'assets', priority: 100, data_stream: {}, + index_patterns: [`${ASSETS_INDEX_PREFIX}*`], template: { settings: {}, mappings: { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts new file mode 100644 index 00000000000000..f9d51db79ed8b4 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const oamAssetComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'oam_v1_asset', + _meta: { + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + asset: { + properties: { + category: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + indexPattern: { + ignore_above: 1024, + type: 'keyword', + }, + defintionId: { + ignore_above: 1024, + type: 'keyword', + }, + latestTimestamp: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts new file mode 100644 index 00000000000000..eab6a09ae920a2 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const oamBaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'oam_v1_base', + _meta: { + documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-base.html', + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + labels: { + type: 'object', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts new file mode 100644 index 00000000000000..b516f5ba64ffb6 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const oamEventComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'oam_v1_event', + _meta: { + documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-event.html', + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts new file mode 100644 index 00000000000000..c06b3e1c367e41 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const oamIndexTemplateConfig: IndicesPutIndexTemplateRequest = { + name: 'oam_v1_index_template', + _meta: { + description: 'The OAM index template', + ecs_version: '8.0.0', + }, + composed_of: ['oam_v1_base', 'oam_v1_event', 'oam_v1_asset'], + index_patterns: ['.oam-observability.asset-v1.*'], + priority: 1, + template: { + mappings: { + _meta: { + version: '1.6.0', + }, + date_detection: false, + dynamic_templates: [ + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + { + asset_metrics: { + mapping: { + type: '{dynamic_type}', + }, + match_mapping_type: ['long', 'double'], + path_match: 'asset.metric.*', + }, + }, + ], + }, + settings: { + index: { + codec: 'best_compression', + mapping: { + total_fields: { + limit: 2000, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json index da1095d989afb0..e8b54118fd5985 100644 --- a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json +++ b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json @@ -26,6 +26,7 @@ "@kbn/metrics-data-access-plugin", "@kbn/core-elasticsearch-server", "@kbn/core-saved-objects-api-server", - "@kbn/core-saved-objects-api-server-mocks" + "@kbn/core-saved-objects-api-server-mocks", + "@kbn/oam-schema" ] } diff --git a/yarn.lock b/yarn.lock index b4529660306285..5d2c02d1f53c9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5438,6 +5438,10 @@ version "0.0.0" uid "" +"@kbn/oam-schema@link:x-pack/packages/kbn-oam-schema": + version "0.0.0" + uid "" + "@kbn/object-versioning@link:packages/kbn-object-versioning": version "0.0.0" uid "" From 575f13360aab1b3496c3310cd091f34ee6c81f78 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 10 May 2024 23:11:31 +0000 Subject: [PATCH 02/16] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/packages/kbn-oam-schema/tsconfig.json | 2 -- .../observability_solution/asset_manager/tsconfig.json | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/packages/kbn-oam-schema/tsconfig.json b/x-pack/packages/kbn-oam-schema/tsconfig.json index e5e31852eb412b..f722f3587e7a29 100644 --- a/x-pack/packages/kbn-oam-schema/tsconfig.json +++ b/x-pack/packages/kbn-oam-schema/tsconfig.json @@ -14,7 +14,5 @@ "target/**/*" ], "kbn_references": [ - "@kbn/std", - "@kbn/io-ts-utils" ] } diff --git a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json index e8b54118fd5985..3a20f5cb2a7f4a 100644 --- a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json +++ b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json @@ -27,6 +27,8 @@ "@kbn/core-elasticsearch-server", "@kbn/core-saved-objects-api-server", "@kbn/core-saved-objects-api-server-mocks", - "@kbn/oam-schema" + "@kbn/oam-schema", + "@kbn/es-query", + "@kbn/zod-helpers" ] } From 8b56d03250182e1fbe833c5e14a09cff735f658a Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 10 May 2024 17:15:23 -0600 Subject: [PATCH 03/16] Use id generators --- .../server/lib/oam/transform/generate_transform.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts index 81efdfae800b1d..33f71e9c42bf3d 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts @@ -12,8 +12,10 @@ import { } from '@elastic/elasticsearch/lib/api/types'; import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; import { generateMetricAggregations } from './generate_metric_aggregations'; -import { OAM_BASE_PREFIX, OAM_TRANSFORM_PREFIX } from '../../../../common/constants_oam'; +import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; import { generateMetadataAggregations } from './generate_metadata_aggregations'; +import { generateTransformId } from './generate_transform_id'; +import { generateIngestPipelineId } from '../ingest_pipeline/generate_ingest_pipeline_id'; export function generateTransform(definition: OAMDefinition): TransformPutTransformRequest { const filter: QueryDslQueryContainer[] = [ @@ -31,7 +33,7 @@ export function generateTransform(definition: OAMDefinition): TransformPutTransf } return { - transform_id: `${OAM_TRANSFORM_PREFIX}-${definition.id}`, + transform_id: generateTransformId(definition), defer_validation: true, source: { index: definition.indexPatterns, @@ -43,7 +45,7 @@ export function generateTransform(definition: OAMDefinition): TransformPutTransf }, dest: { index: `${OAM_BASE_PREFIX}.noop`, - pipeline: `${OAM_BASE_PREFIX}.${definition.id}`, + pipeline: generateIngestPipelineId(definition), }, frequency: definition.settings?.frequency || '1m', sync: { From 131bfcd70f74fbe8e9a1bb2a43d30e8351e660c0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 10 May 2024 23:24:14 +0000 Subject: [PATCH 04/16] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d8e8361061b78d..9764f7e8d3c21e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -600,6 +600,7 @@ src/plugins/newsfeed @elastic/kibana-core test/common/plugins/newsfeed @elastic/kibana-core src/plugins/no_data_page @elastic/appex-sharedux x-pack/plugins/notifications @elastic/appex-sharedux +x-pack/packages/kbn-oam-schema @elastic/obs-knowledge-team packages/kbn-object-versioning @elastic/appex-sharedux x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant From cd05235ccbc0811f792b5c0b81517f818b9f99c7 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 13 May 2024 10:16:52 -0600 Subject: [PATCH 05/16] Adding await --- .../asset_manager/server/lib/oam/delete_index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts index daa1452fd301fd..2aeded140fb1d7 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts @@ -16,7 +16,7 @@ export async function deleteIndex( ) { const indexName = generateIndexName(definition); try { - esClient.indices.delete({ index: indexName, ignore_unavailable: true }); + await esClient.indices.delete({ index: indexName, ignore_unavailable: true }); } catch (e) { logger.error(`Unable to remove OAM Defintion index [${definition.id}}]`); throw e; From 18a273a4bda23a9202be6ddfbe45529d4720dab1 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 13 May 2024 15:54:37 -0600 Subject: [PATCH 06/16] Applying feedback from Tom - making indexPattern plural - removing categories and assst.category - fixing typos - adding clean up when creation fails - changing path from `/api/oam` to `/api/oam/definition` - removing unused `preview_transform.ts` - updating fixtures and tests - changing OAMNotFound to OAMDefinitionNotFound --- .../kbn-oam-schema/src/schema/asset.ts | 5 ++-- .../src/schema/oam_definition.ts | 1 - .../asset_manager/common/constants_oam.ts | 2 ++ .../oam/create_and_install_ingest_pipeline.ts | 2 +- .../server/lib/oam/delete_oam_definition.ts | 4 +-- .../server/lib/oam/errors/oam_not_found.ts | 2 +- .../oam/helpers/fixtures/oam_definition.ts | 1 - .../generate_processors.test.ts.snap | 15 ++-------- .../ingest_pipeline/generate_processors.ts | 15 ++-------- .../server/lib/oam/preview_transform.ts | 28 ------------------- .../server/lib/oam/read_oam_definition.ts | 4 +-- .../server/lib/oam/start_transform.ts | 2 +- .../lib/oam/transform/generate_transform.ts | 10 +++++-- .../asset_manager/server/routes/oam/create.ts | 26 ++++++++++++++--- .../asset_manager/server/routes/oam/delete.ts | 8 ++---- .../asset_manager/server/routes/oam/reset.ts | 6 ++-- .../server/templates/components/asset.ts | 6 +--- 17 files changed, 51 insertions(+), 86 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts diff --git a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts index 9238e8d8cd12dd..4ffc328ff24768 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts @@ -12,9 +12,8 @@ export const assetSchema = z.intersection( z.object({ asset: z.object({ id: z.string(), - indexPattern: arrayOfStringsSchema, - category: arrayOfStringsSchema, - identityField: arrayOfStringsSchema, + indexPatterns: arrayOfStringsSchema, + identityFields: arrayOfStringsSchema, metric: z.record(z.string(), z.number()), }), }), diff --git a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts index b0aaa39344727c..6ebfdf1c8853de 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts @@ -24,7 +24,6 @@ export const oamDefinitionSchema = z.object({ indexPatterns: arrayOfStringsSchema, identityFields: arrayOfStringsSchema, identityTemplate: z.string(), - categories: arrayOfStringsSchema, metadata: z.optional(z.array(metadataSchema)), metrics: z.optional(z.array(keyMetricSchema)), staticFields: z.optional(z.record(z.string(), z.string())), diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts index e5a9f031763221..4f5762fc0a8633 100644 --- a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts @@ -8,3 +8,5 @@ export const OAM_VERSION = 'v1'; export const OAM_BASE_PREFIX = `.oam-observability.asset-${OAM_VERSION}`; export const OAM_TRANSFORM_PREFIX = `oam-observability-asset-${OAM_VERSION}`; +export const OAM_DEFAULT_FREQUENCY = '1m'; +export const OAM_DEFAULT_SYNC_DELAY = '60s'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts index 878c5475ee6da8..0a5a34a5a0aa34 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts @@ -29,7 +29,7 @@ export async function createAndInstallIngestPipeline( { logger } ); } catch (e) { - logger.error(`Cannot create OAM tranform for [${definition.id}] asset defintion`); + logger.error(`Cannot create OAM ingest pipeline for [${definition.id}] asset defintion`); if (e.meta?.body?.error?.type === 'security_exception') { throw new OAMSecurityException(e.meta.body.error.reason, definition); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts index d255be5e9173b9..d73d6429e136f8 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts @@ -8,7 +8,7 @@ import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { OAMDefinition } from '@kbn/oam-schema'; import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; -import { OAMNotFound } from './errors/oam_not_found'; +import { OAMDefinitionNotFound } from './errors/oam_not_found'; export async function deleteOAMDefinition( soClient: SavedObjectsClientContract, @@ -24,7 +24,7 @@ export async function deleteOAMDefinition( if (response.total === 0) { logger.error(`Unable to delete OAM Definition [${definition.id}] because it doesn't exist.`); - throw new OAMNotFound(`OAM Defintion with [${definition.id}] not found.`); + throw new OAMDefinitionNotFound(`OAM Defintion with [${definition.id}] not found.`); } await soClient.delete(SO_OAM_DEFINITION_TYPE, response.saved_objects[0].id); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts index fc00c97e62f4f3..08911c3ad0ab66 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts @@ -5,7 +5,7 @@ * 2.0. */ -export class OAMNotFound extends Error { +export class OAMDefinitionNotFound extends Error { constructor(message: string) { super(message); this.name = 'OAMNotFound'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts index 023adc99e4acc9..7ddd199c3cd748 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts @@ -12,7 +12,6 @@ export const oamDefinition = oamDefinitionSchema.parse({ type: 'service', indexPatterns: ['kbn-data-forge-fake_stack.*'], timestampField: '@timestamp', - categories: ['service', 'logs'], identityFields: ['log.logger'], identityTemplate: 'service:{{log.logger}}', metadata: ['tags', 'host.name', 'kubernetes.pod.name'], diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap index 777aa1e9987efb..9848267d70bc94 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -16,24 +16,13 @@ Array [ }, Object { "set": Object { - "field": "asset.indexPattern", + "field": "asset.indexPatterns", "value": "[\\"kbn-data-forge-fake_stack.*\\"]", }, }, Object { "json": Object { - "field": "asset.indexPattern", - }, - }, - Object { - "set": Object { - "field": "asset.category", - "value": "[\\"service\\",\\"logs\\"]", - }, - }, - Object { - "json": Object { - "field": "asset.category", + "field": "asset.indexPatterns", }, }, Object { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts index ba4e85875aecf8..cd613ea111d7b6 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts @@ -59,24 +59,13 @@ export function generateProcessors(definition: OAMDefinition) { }, { set: { - field: 'asset.indexPattern', + field: 'asset.indexPatterns', value: JSON.stringify(definition.indexPatterns), }, }, { json: { - field: 'asset.indexPattern', - }, - }, - { - set: { - field: 'asset.category', - value: JSON.stringify(definition.categories), - }, - }, - { - json: { - field: 'asset.category', + field: 'asset.indexPatterns', }, }, { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts deleted file mode 100644 index 3403cf41c8afa7..00000000000000 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/preview_transform.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; -import { retryTransientEsErrors } from './helpers/retry'; -import { generateTransformId } from './transform/generate_transform_id'; - -export async function previewTransform( - esClient: ElasticsearchClient, - definition: OAMDefinition, - logger: Logger -) { - const transformId = generateTransformId(definition); - try { - await retryTransientEsErrors( - () => esClient.transform.previewTransform({ transform_id: transformId }), - { logger } - ); - } catch (err) { - logger.error(`Cannot preview SLO transform [${transformId}]`); - throw err; - } -} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts index 411f69b25d60a7..ce23a59112c0cc 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts @@ -8,7 +8,7 @@ import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { OAMDefinition, oamDefinitionSchema } from '@kbn/oam-schema'; import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; -import { OAMNotFound } from './errors/oam_not_found'; +import { OAMDefinitionNotFound } from './errors/oam_not_found'; export async function readOAMDefinition( soClient: SavedObjectsClientContract, @@ -24,7 +24,7 @@ export async function readOAMDefinition( if (response.total === 0) { const message = `Unable to find OAM Defintion with [${id}]`; logger.error(message); - throw new OAMNotFound(message); + throw new OAMDefinitionNotFound(message); } try { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts index 5a1960b0435eb0..d2e3c9c69b8ed3 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts @@ -22,7 +22,7 @@ export async function startTransform( { logger } ); } catch (err) { - logger.error(`Cannot start SLO transform [${transformId}]`); + logger.error(`Cannot start OAM transform [${transformId}]`); throw err; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts index 33f71e9c42bf3d..24839bf5c0a425 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts @@ -12,7 +12,11 @@ import { } from '@elastic/elasticsearch/lib/api/types'; import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; import { generateMetricAggregations } from './generate_metric_aggregations'; -import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; +import { + OAM_BASE_PREFIX, + OAM_DEFAULT_FREQUENCY, + OAM_DEFAULT_SYNC_DELAY, +} from '../../../../common/constants_oam'; import { generateMetadataAggregations } from './generate_metadata_aggregations'; import { generateTransformId } from './generate_transform_id'; import { generateIngestPipelineId } from '../ingest_pipeline/generate_ingest_pipeline_id'; @@ -47,11 +51,11 @@ export function generateTransform(definition: OAMDefinition): TransformPutTransf index: `${OAM_BASE_PREFIX}.noop`, pipeline: generateIngestPipelineId(definition), }, - frequency: definition.settings?.frequency || '1m', + frequency: definition.settings?.frequency || OAM_DEFAULT_FREQUENCY, sync: { time: { field: definition.settings?.syncField ?? definition.timestampField, - delay: definition.settings?.syncDelay ?? '60s', + delay: definition.settings?.syncDelay ?? OAM_DEFAULT_SYNC_DELAY, }, }, settings: { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts index 69442b6b281402..3da9d0f6d9cfac 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts @@ -16,6 +16,9 @@ import { createAndInstallTransform } from '../../lib/oam/create_and_install_tran import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; import { startTransform } from '../../lib/oam/start_transform'; +import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; +import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; +import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; export function createOAMDefinitionRoute({ router, @@ -23,7 +26,7 @@ export function createOAMDefinitionRoute({ }: SetupRouteOptions) { router.post( { - path: '/api/oam', + path: '/api/oam/defintion', validate: { body: (body, res) => { try { @@ -35,17 +38,32 @@ export function createOAMDefinitionRoute({ }, }, async (context, req, res) => { + let definitionCreated = false; + let ingestPipelineCreated = false; + let transformCreated = false; + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; try { - const soClient = (await context.core).savedObjects.client; - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const definition = await saveOAMDefinition(soClient, req.body); + definitionCreated = true; await createAndInstallIngestPipeline(esClient, definition, logger); + ingestPipelineCreated = true; await createAndInstallTransform(esClient, definition, logger); + transformCreated = true; await startTransform(esClient, definition, logger); return res.ok({ body: definition }); } catch (e) { + // Clean up anything that was successful. + if (definitionCreated) { + deleteOAMDefinition(soClient, req.body, logger); + } + if (ingestPipelineCreated) { + deleteIngestPipeline(esClient, req.body, logger); + } + if (transformCreated) { + stopAndDeleteTransform(esClient, req.body, logger); + } if (e instanceof OAMIdConflict) { return res.conflict({ body: e }); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts index ea8797a3ef2c62..7955a70820e354 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts @@ -13,9 +13,8 @@ import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_er import { readOAMDefinition } from '../../lib/oam/read_oam_definition'; import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; -import { deleteIndex } from '../../lib/oam/delete_index'; import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; -import { OAMNotFound } from '../../lib/oam/errors/oam_not_found'; +import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; export function deleteOAMDefinitionRoute({ router, @@ -23,7 +22,7 @@ export function deleteOAMDefinitionRoute({ }: SetupRouteOptions) { router.delete<{ id: string }, unknown, unknown>( { - path: '/api/oam/{id}', + path: '/api/oam/definition/{id}', validate: { params: schema.object({ id: schema.string(), @@ -38,12 +37,11 @@ export function deleteOAMDefinitionRoute({ const definition = await readOAMDefinition(soClient, req.params.id, logger); await stopAndDeleteTransform(esClient, definition, logger); await deleteIngestPipeline(esClient, definition, logger); - await deleteIndex(esClient, definition, logger); await deleteOAMDefinition(soClient, definition, logger); return res.ok({ body: { acknowledged: true } }); } catch (e) { - if (e instanceof OAMNotFound) { + if (e instanceof OAMDefinitionNotFound) { return res.notFound({ body: e }); } if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts index b6482e6360104a..a346c3d0f58347 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts @@ -17,7 +17,7 @@ import { deleteIndex } from '../../lib/oam/delete_index'; import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install_ingest_pipeline'; import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; import { startTransform } from '../../lib/oam/start_transform'; -import { OAMNotFound } from '../../lib/oam/errors/oam_not_found'; +import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; export function resetOAMDefinitionRoute({ router, @@ -25,7 +25,7 @@ export function resetOAMDefinitionRoute({ }: SetupRouteOptions) { router.post<{ id: string }, unknown, unknown>( { - path: '/api/oam/{id}/_reset', + path: '/api/oam/definition/{id}/_reset', validate: { params: schema.object({ id: schema.string(), @@ -51,7 +51,7 @@ export function resetOAMDefinitionRoute({ return res.ok({ body: { acknowledged: true } }); } catch (e) { - if (e instanceof OAMNotFound) { + if (e instanceof OAMDefinitionNotFound) { return res.notFound({ body: e }); } if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts index f9d51db79ed8b4..9c32bd15ef56f8 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts @@ -17,15 +17,11 @@ export const oamAssetComponentTemplateConfig: ClusterPutComponentTemplateRequest properties: { asset: { properties: { - category: { - ignore_above: 1024, - type: 'keyword', - }, id: { ignore_above: 1024, type: 'keyword', }, - indexPattern: { + indexPatterns: { ignore_above: 1024, type: 'keyword', }, From 784cd84e0a592fb69f27dfe9bd3de728ca561013 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 13 May 2024 16:14:13 -0600 Subject: [PATCH 07/16] fixing typo --- .../asset_manager/server/routes/oam/create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts index 3da9d0f6d9cfac..c65b52e9f45a8a 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts @@ -26,7 +26,7 @@ export function createOAMDefinitionRoute({ }: SetupRouteOptions) { router.post( { - path: '/api/oam/defintion', + path: '/api/oam/definition', validate: { body: (body, res) => { try { From 972a1313c8d7985a8e8ff47698db3a72a5761aea Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 13 May 2024 16:37:21 -0600 Subject: [PATCH 08/16] Add awaits --- .../asset_manager/server/routes/oam/create.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts index c65b52e9f45a8a..7d56048eea92e1 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts @@ -56,13 +56,13 @@ export function createOAMDefinitionRoute({ } catch (e) { // Clean up anything that was successful. if (definitionCreated) { - deleteOAMDefinition(soClient, req.body, logger); + await deleteOAMDefinition(soClient, req.body, logger); } if (ingestPipelineCreated) { - deleteIngestPipeline(esClient, req.body, logger); + await deleteIngestPipeline(esClient, req.body, logger); } if (transformCreated) { - stopAndDeleteTransform(esClient, req.body, logger); + await stopAndDeleteTransform(esClient, req.body, logger); } if (e instanceof OAMIdConflict) { return res.conflict({ body: e }); From df69a9a5a57c348e71a221d8f3e808e809a2e46f Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 14 May 2024 08:34:23 -0600 Subject: [PATCH 09/16] changing from /api/oam to /api/entities --- .../asset_manager/common/constants_oam.ts | 1 + .../asset_manager/server/routes/oam/create.ts | 3 ++- .../asset_manager/server/routes/oam/delete.ts | 3 ++- .../asset_manager/server/routes/oam/reset.ts | 3 ++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts index 4f5762fc0a8633..e5fd36bd46f198 100644 --- a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts @@ -10,3 +10,4 @@ export const OAM_BASE_PREFIX = `.oam-observability.asset-${OAM_VERSION}`; export const OAM_TRANSFORM_PREFIX = `oam-observability-asset-${OAM_VERSION}`; export const OAM_DEFAULT_FREQUENCY = '1m'; export const OAM_DEFAULT_SYNC_DELAY = '60s'; +export const OAM_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts index 7d56048eea92e1..c984c585caa72e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts @@ -19,6 +19,7 @@ import { startTransform } from '../../lib/oam/start_transform'; import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; +import { OAM_API_PREFIX } from '../../../common/constants_oam'; export function createOAMDefinitionRoute({ router, @@ -26,7 +27,7 @@ export function createOAMDefinitionRoute({ }: SetupRouteOptions) { router.post( { - path: '/api/oam/definition', + path: `${OAM_API_PREFIX}/definition`, validate: { body: (body, res) => { try { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts index 7955a70820e354..5698937dc7a681 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts @@ -15,6 +15,7 @@ import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform' import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; +import { OAM_API_PREFIX } from '../../../common/constants_oam'; export function deleteOAMDefinitionRoute({ router, @@ -22,7 +23,7 @@ export function deleteOAMDefinitionRoute({ }: SetupRouteOptions) { router.delete<{ id: string }, unknown, unknown>( { - path: '/api/oam/definition/{id}', + path: `${OAM_API_PREFIX}/definition/{id}`, validate: { params: schema.object({ id: schema.string(), diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts index a346c3d0f58347..6193eb1bc46936 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts @@ -18,6 +18,7 @@ import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; import { startTransform } from '../../lib/oam/start_transform'; import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; +import { OAM_API_PREFIX } from '../../../common/constants_oam'; export function resetOAMDefinitionRoute({ router, @@ -25,7 +26,7 @@ export function resetOAMDefinitionRoute({ }: SetupRouteOptions) { router.post<{ id: string }, unknown, unknown>( { - path: '/api/oam/definition/{id}/_reset', + path: `${OAM_API_PREFIX}/definition/{id}/_reset`, validate: { params: schema.object({ id: schema.string(), From c5abf4637e6d3e801930b35039891f55a6427e27 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 14 May 2024 08:52:48 -0600 Subject: [PATCH 10/16] swapping out asset for entity --- .../kbn-oam-schema/src/schema/asset.ts | 4 +-- .../kbn-oam-schema/src/schema/common.ts | 4 +-- .../src/schema/oam_definition.ts | 4 +-- .../asset_manager/common/constants_oam.ts | 4 +-- .../oam/create_and_install_ingest_pipeline.ts | 2 +- .../lib/oam/create_and_install_transform.ts | 2 +- .../server/lib/oam/errors/oam_not_found.ts | 2 +- .../generate_processors.test.ts.snap | 26 +++++++++---------- .../ingest_pipeline/generate_processors.ts | 16 ++++++------ .../generate_transform.test.ts.snap | 20 +++++++------- .../generate_metadata_aggregations.ts | 2 +- .../transform/generate_metric_aggregations.ts | 2 +- .../lib/oam/transform/generate_transform.ts | 4 +-- .../asset_manager/server/plugin.ts | 4 +-- .../components/{asset.ts => entity.ts} | 6 ++--- .../server/templates/oam_template.ts | 8 +++--- 16 files changed, 55 insertions(+), 55 deletions(-) rename x-pack/plugins/observability_solution/asset_manager/server/templates/components/{asset.ts => entity.ts} (87%) diff --git a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts index 4ffc328ff24768..514ed014940362 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/asset.ts @@ -8,9 +8,9 @@ import { z } from 'zod'; import { arrayOfStringsSchema } from './common'; -export const assetSchema = z.intersection( +export const entitySchema = z.intersection( z.object({ - asset: z.object({ + entity: z.object({ id: z.string(), indexPatterns: arrayOfStringsSchema, identityFields: arrayOfStringsSchema, diff --git a/x-pack/packages/kbn-oam-schema/src/schema/common.ts b/x-pack/packages/kbn-oam-schema/src/schema/common.ts index 2bf40e385d6028..3597adcca56871 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/common.ts @@ -8,7 +8,7 @@ import { z } from 'zod'; import moment from 'moment'; -export enum AssetType { +export enum EntityType { service = 'service', host = 'host', pod = 'pod', @@ -17,7 +17,7 @@ export enum AssetType { export const arrayOfStringsSchema = z.array(z.string()); -export const assetTypeSchema = z.nativeEnum(AssetType); +export const entityTypeSchema = z.nativeEnum(EntityType); export enum BasicAggregations { avg = 'avg', diff --git a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts index 6ebfdf1c8853de..6226c5c22d866c 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts @@ -8,7 +8,7 @@ import { z } from 'zod'; import { arrayOfStringsSchema, - assetTypeSchema, + entityTypeSchema, keyMetricSchema, metadataSchema, filterSchema, @@ -19,7 +19,7 @@ export const oamDefinitionSchema = z.object({ id: z.string().regex(/^[\w-]+$/), name: z.string(), description: z.optional(z.string()), - type: assetTypeSchema, + type: entityTypeSchema, filter: filterSchema, indexPatterns: arrayOfStringsSchema, identityFields: arrayOfStringsSchema, diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts index e5fd36bd46f198..2e72483f78f906 100644 --- a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts @@ -6,8 +6,8 @@ */ export const OAM_VERSION = 'v1'; -export const OAM_BASE_PREFIX = `.oam-observability.asset-${OAM_VERSION}`; -export const OAM_TRANSFORM_PREFIX = `oam-observability-asset-${OAM_VERSION}`; +export const OAM_BASE_PREFIX = `.oam-observability.entities-${OAM_VERSION}`; +export const OAM_TRANSFORM_PREFIX = `oam-observability-entities-${OAM_VERSION}`; export const OAM_DEFAULT_FREQUENCY = '1m'; export const OAM_DEFAULT_SYNC_DELAY = '60s'; export const OAM_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts index 0a5a34a5a0aa34..bbd0fee4118b72 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts @@ -29,7 +29,7 @@ export async function createAndInstallIngestPipeline( { logger } ); } catch (e) { - logger.error(`Cannot create OAM ingest pipeline for [${definition.id}] asset defintion`); + logger.error(`Cannot create OAM ingest pipeline for [${definition.id}] entity defintion`); if (e.meta?.body?.error?.type === 'security_exception') { throw new OAMSecurityException(e.meta.body.error.reason, definition); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts index ff41aadab7188b..2669949706efe3 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts @@ -20,7 +20,7 @@ export async function createAndInstallTransform( try { await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger }); } catch (e) { - logger.error(`Cannot create OAM transform for [${definition.id}] asset definition`); + logger.error(`Cannot create OAM transform for [${definition.id}] entity definition`); if (e.meta?.body?.error?.type === 'security_exception') { throw new OAMSecurityException(e.meta.body.error.reason, definition); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts index 08911c3ad0ab66..31a232bf88485f 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts @@ -8,6 +8,6 @@ export class OAMDefinitionNotFound extends Error { constructor(message: string) { super(message); - this.name = 'OAMNotFound'; + this.name = 'OAMDefinitionNotFound'; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap index 9848267d70bc94..333a2ec8ccf464 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -10,25 +10,25 @@ Array [ }, Object { "set": Object { - "field": "asset.definitionId", + "field": "entity.definitionId", "value": "admin-console-logs-service", }, }, Object { "set": Object { - "field": "asset.indexPatterns", + "field": "entity.indexPatterns", "value": "[\\"kbn-data-forge-fake_stack.*\\"]", }, }, Object { "json": Object { - "field": "asset.indexPatterns", + "field": "entity.indexPatterns", }, }, Object { "set": Object { - "field": "asset.id", - "value": "service:{{asset.identity.log.logger}}", + "field": "entity.id", + "value": "service:{{entity.identity.log.logger}}", }, }, Object { @@ -39,31 +39,31 @@ Array [ }, Object { "script": Object { - "source": "if (ctx.asset?.metadata?.tags != null) { - ctx[\\"tags\\"] = ctx.asset.metadata.tags.keySet(); + "source": "if (ctx.entity?.metadata?.tags != null) { + ctx[\\"tags\\"] = ctx.entity.metadata.tags.keySet(); } -if (ctx.asset?.metadata?.host?.name != null) { +if (ctx.entity?.metadata?.host?.name != null) { ctx[\\"host\\"] = new HashMap(); - ctx[\\"host\\"][\\"name\\"] = ctx.asset.metadata.host.name.keySet(); + ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.keySet(); } -if (ctx.asset?.metadata?.kubernetes?.pod?.name != null) { +if (ctx.entity?.metadata?.kubernetes?.pod?.name != null) { ctx[\\"kubernetes\\"] = new HashMap(); ctx[\\"kubernetes\\"][\\"pod\\"] = new HashMap(); - ctx[\\"kubernetes\\"][\\"pod\\"][\\"name\\"] = ctx.asset.metadata.kubernetes.pod.name.keySet(); + ctx[\\"kubernetes\\"][\\"pod\\"][\\"name\\"] = ctx.entity.metadata.kubernetes.pod.name.keySet(); } ", }, }, Object { "remove": Object { - "field": "asset.metadata", + "field": "entity.metadata", "ignore_missing": true, }, }, Object { "set": Object { "field": "_index", - "value": ".oam-observability.asset-v1.admin-console-logs-service", + "value": ".oam-observability.entities-v1.admin-console-logs-service", }, }, ] diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts index cd613ea111d7b6..15f60fb4dbfdf7 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts @@ -10,7 +10,7 @@ import { generateIndexName } from '../helpers/generate_index_name'; function createIdTemplate(definition: OAMDefinition) { return definition.identityFields.reduce((template, field) => { - return template.replaceAll(field, `asset.identity.${field}`); + return template.replaceAll(field, `entity.identity.${field}`); }, definition.identityTemplate); } @@ -20,7 +20,7 @@ function mapDesitnationToPainless(destination: string, source: string) { if (currentIndex + 1 === parts.length) { return `${acc}\n ctx${parts .map((s) => `["${s}"]`) - .join('')} = ctx.asset.metadata.${source}.keySet();`; + .join('')} = ctx.entity.metadata.${source}.keySet();`; } return `${acc}\n ctx${parts .slice(0, currentIndex + 1) @@ -36,7 +36,7 @@ function createMetadataPainlessScript(definition: OAMDefinition) { return definition.metadata.reduce((script, def) => { const source = def.source; const destination = def.destination || def.source; - return `${script}if (ctx.asset?.metadata?.${source.replaceAll( + return `${script}if (ctx.entity?.metadata?.${source.replaceAll( '.', '?.' )} != null) {${mapDesitnationToPainless(destination, source)}\n}\n`; @@ -53,24 +53,24 @@ export function generateProcessors(definition: OAMDefinition) { }, { set: { - field: 'asset.definitionId', + field: 'entity.definitionId', value: definition.id, }, }, { set: { - field: 'asset.indexPatterns', + field: 'entity.indexPatterns', value: JSON.stringify(definition.indexPatterns), }, }, { json: { - field: 'asset.indexPatterns', + field: 'entity.indexPatterns', }, }, { set: { - field: 'asset.id', + field: 'entity.id', value: createIdTemplate(definition), }, }, @@ -84,7 +84,7 @@ export function generateProcessors(definition: OAMDefinition) { : []), { remove: { - field: 'asset.metadata', + field: 'entity.metadata', ignore_missing: true, }, }, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap index cdb172a6afcd2f..b706949bbc9f42 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap @@ -4,8 +4,8 @@ exports[`generateTransform(definition) should generate a valid summary transform Object { "defer_validation": true, "dest": Object { - "index": ".oam-observability.asset-v1.noop", - "pipeline": ".oam-observability.asset-v1.admin-console-logs-service", + "index": ".oam-observability.entities-v1.noop", + "pipeline": ".oam-observability.entities-v1.admin-console-logs-service", }, "frequency": "1m", "pivot": Object { @@ -38,30 +38,30 @@ Object { }, }, }, - "asset.latestTimestamp": Object { + "entity.latestTimestamp": Object { "max": Object { "field": "@timestamp", }, }, - "asset.metadata.host.name": Object { + "entity.metadata.host.name": Object { "terms": Object { "field": "host.name", "size": 1000, }, }, - "asset.metadata.kubernetes.pod.name": Object { + "entity.metadata.kubernetes.pod.name": Object { "terms": Object { "field": "kubernetes.pod.name", "size": 1000, }, }, - "asset.metadata.tags": Object { + "entity.metadata.tags": Object { "terms": Object { "field": "tags", "size": 1000, }, }, - "asset.metric.errorRate": Object { + "entity.metric.errorRate": Object { "bucket_script": Object { "buckets_path": Object { "A": "_errorRate_A>_count", @@ -72,7 +72,7 @@ Object { }, }, }, - "asset.metric.logRate": Object { + "entity.metric.logRate": Object { "bucket_script": Object { "buckets_path": Object { "A": "_logRate_A>_count", @@ -85,7 +85,7 @@ Object { }, }, "group_by": Object { - "asset.identity.log.logger": Object { + "entity.identity.log.logger": Object { "terms": Object { "field": "log.logger", }, @@ -120,6 +120,6 @@ Object { "field": "@timestamp", }, }, - "transform_id": "oam-observability-asset-v1-admin-console-logs-service", + "transform_id": "oam-observability-entities-v1-admin-console-logs-service", } `; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts index ee483fa4dc198e..8177b07e8f39dc 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts @@ -14,7 +14,7 @@ export function generateMetadataAggregations(definition: OAMDefinition) { return definition.metadata.reduce( (aggs, metadata) => ({ ...aggs, - [`asset.metadata.${metadata.destination ?? metadata.source}`]: { + [`entity.metadata.${metadata.destination ?? metadata.source}`]: { terms: { field: metadata.source, size: metadata.limit ?? 1000, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts index 4c3611e10a63e9..e2947c584f19e8 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts @@ -112,7 +112,7 @@ export function generateMetricAggregations(definition: OAMDefinition) { return { ...aggs, ...buildMetricAggregations(keyMetric, definition.timestampField), - [`asset.metric.${keyMetric.name}`]: buildMetricEquation(keyMetric), + [`entity.metric.${keyMetric.name}`]: buildMetricEquation(keyMetric), }; }, {}); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts index 24839bf5c0a425..713bc77903f7f0 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts @@ -66,14 +66,14 @@ export function generateTransform(definition: OAMDefinition): TransformPutTransf group_by: definition.identityFields.reduce( (acc, field) => ({ ...acc, - [`asset.identity.${field}`]: { terms: { field } }, + [`entity.identity.${field}`]: { terms: { field } }, }), {} ), aggs: { ...generateMetricAggregations(definition), ...generateMetadataAggregations(definition), - 'asset.latestTimestamp': { + 'entity.latestTimestamp': { max: { field: definition.timestampField, }, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts index d11f59cb8ebd03..0870ff1f44bf0b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts @@ -22,10 +22,10 @@ import { AssetClient } from './lib/asset_client'; import { AssetManagerPluginSetupDependencies, AssetManagerPluginStartDependencies } from './types'; import { AssetManagerConfig, configSchema, exposeToBrowserConfig } from '../common/config'; import { oamBaseComponentTemplateConfig } from './templates/components/base'; -import { oamAssetComponentTemplateConfig } from './templates/components/asset'; import { oamEventComponentTemplateConfig } from './templates/components/event'; import { oamIndexTemplateConfig } from './templates/oam_template'; import { oamDefinition } from './saved_objects'; +import { oamEntityComponentTemplateConfig } from './templates/components/entity'; export type AssetManagerServerPluginSetup = ReturnType; export type AssetManagerServerPluginStart = ReturnType; @@ -101,7 +101,7 @@ export class AssetManagerServerPlugin upsertComponent({ esClient, logger: this.logger, - component: oamAssetComponentTemplateConfig, + component: oamEntityComponentTemplateConfig, }), ]) .then(() => diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts similarity index 87% rename from x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts rename to x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts index 9c32bd15ef56f8..5d2a615c01fa43 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/asset.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts @@ -7,15 +7,15 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -export const oamAssetComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: 'oam_v1_asset', +export const oamEntityComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'oam_v1_entity', _meta: { ecs_version: '8.0.0', }, template: { mappings: { properties: { - asset: { + entity: { properties: { id: { ignore_above: 1024, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts index c06b3e1c367e41..9017f86fdde97a 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts @@ -13,8 +13,8 @@ export const oamIndexTemplateConfig: IndicesPutIndexTemplateRequest = { description: 'The OAM index template', ecs_version: '8.0.0', }, - composed_of: ['oam_v1_base', 'oam_v1_event', 'oam_v1_asset'], - index_patterns: ['.oam-observability.asset-v1.*'], + composed_of: ['oam_v1_base', 'oam_v1_event', 'oam_v1_entity'], + index_patterns: ['.oam-observability.entities-v1.*'], priority: 1, template: { mappings: { @@ -33,12 +33,12 @@ export const oamIndexTemplateConfig: IndicesPutIndexTemplateRequest = { }, }, { - asset_metrics: { + entity_metrics: { mapping: { type: '{dynamic_type}', }, match_mapping_type: ['long', 'double'], - path_match: 'asset.metric.*', + path_match: 'entity.metric.*', }, }, ], From 230d0c08aedc9d5d4e3ec5679bb7f997bcfdb77e Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 14 May 2024 10:42:29 -0600 Subject: [PATCH 11/16] Renaming OAM to entities; renaming asset to entity --- x-pack/packages/kbn-oam-schema/index.ts | 4 +- .../src/schema/{asset.ts => entity.ts} | 0 ...oam_definition.ts => entity_definition.ts} | 4 +- .../common/constants_entities.ts | 13 ++++++ .../asset_manager/common/constants_oam.ts | 13 ------ .../create_and_install_ingest_pipeline.ts | 10 ++--- .../create_and_install_transform.ts | 10 ++--- .../lib/entities/delete_entity_definition.ts | 31 ++++++++++++++ .../lib/{oam => entities}/delete_index.ts | 6 +-- .../delete_ingest_pipeline.ts | 4 +- .../errors/entity_id_conflict_error.ts} | 10 ++--- .../errors/entity_not_found.ts} | 4 +- .../errors/entity_security_exception.ts} | 10 ++--- .../errors/invalid_transform_error.ts | 0 .../helpers/fixtures/entity_definition.ts} | 4 +- .../helpers/generate_index_name.ts | 8 ++-- .../get_elasticsearch_query_or_throw.ts | 0 .../lib/{oam => entities}/helpers/retry.ts | 0 .../generate_processors.test.ts.snap | 2 +- .../generate_ingest_pipeline_id.ts | 8 ++-- .../generate_processors.test.ts | 4 +- .../ingest_pipeline/generate_processors.ts | 8 ++-- .../lib/entities/read_entity_definition.ts | 36 +++++++++++++++++ .../lib/entities/save_entity_definition.ts | 37 +++++++++++++++++ .../lib/{oam => entities}/start_transform.ts | 6 +-- .../stop_and_delete_transform.ts | 6 +-- .../generate_transform.test.ts.snap | 6 +-- .../generate_metadata_aggregations.ts | 4 +- .../transform/generate_metric_aggregations.ts | 4 +- .../transform/generate_transform.test.ts | 4 +- .../transform/generate_transform.ts | 18 ++++----- .../transform/generate_transform_id.ts | 8 ++-- .../server/lib/oam/delete_oam_definition.ts | 31 -------------- .../server/lib/oam/read_oam_definition.ts | 36 ----------------- .../server/lib/oam/save_oam_definition.ts | 34 ---------------- .../asset_manager/server/plugin.ts | 26 +++++++----- .../server/routes/{oam => entities}/create.ts | 40 +++++++++---------- .../server/routes/{oam => entities}/delete.ts | 28 ++++++------- .../server/routes/{oam => entities}/reset.ts | 32 +++++++-------- .../asset_manager/server/routes/index.ts | 12 +++--- ...oam_definition.ts => entity_definition.ts} | 14 +++---- .../server/saved_objects/index.ts | 2 +- .../server/templates/components/base.ts | 4 +- .../server/templates/components/entity.ts | 4 +- .../server/templates/components/event.ts | 4 +- .../{oam_template.ts => entities_template.ts} | 10 ++--- 46 files changed, 283 insertions(+), 276 deletions(-) rename x-pack/packages/kbn-oam-schema/src/schema/{asset.ts => entity.ts} (100%) rename x-pack/packages/kbn-oam-schema/src/schema/{oam_definition.ts => entity_definition.ts} (89%) create mode 100644 x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts delete mode 100644 x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/create_and_install_ingest_pipeline.ts (75%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/create_and_install_transform.ts (71%) create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/delete_index.ts (79%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/delete_ingest_pipeline.ts (91%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam/errors/oam_id_conflict_error.ts => entities/errors/entity_id_conflict_error.ts} (57%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam/errors/oam_not_found.ts => entities/errors/entity_not_found.ts} (76%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam/errors/oam_security_exception.ts => entities/errors/entity_security_exception.ts} (56%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/errors/invalid_transform_error.ts (100%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam/helpers/fixtures/oam_definition.ts => entities/helpers/fixtures/entity_definition.ts} (89%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/helpers/generate_index_name.ts (51%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/helpers/get_elasticsearch_query_or_throw.ts (100%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/helpers/retry.ts (100%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap (95%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/ingest_pipeline/generate_ingest_pipeline_id.ts (50%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/ingest_pipeline/generate_processors.test.ts (77%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/ingest_pipeline/generate_processors.ts (90%) create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/start_transform.ts (84%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/stop_and_delete_transform.ts (86%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/__snapshots__/generate_transform.test.ts.snap (92%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/generate_metadata_aggregations.ts (82%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/generate_metric_aggregations.ts (95%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/generate_transform.test.ts (78%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/generate_transform.ts (81%) rename x-pack/plugins/observability_solution/asset_manager/server/lib/{oam => entities}/transform/generate_transform_id.ts (50%) delete mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts delete mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts delete mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts rename x-pack/plugins/observability_solution/asset_manager/server/routes/{oam => entities}/create.ts (54%) rename x-pack/plugins/observability_solution/asset_manager/server/routes/{oam => entities}/delete.ts (52%) rename x-pack/plugins/observability_solution/asset_manager/server/routes/{oam => entities}/reset.ts (55%) rename x-pack/plugins/observability_solution/asset_manager/server/saved_objects/{oam_definition.ts => entity_definition.ts} (71%) rename x-pack/plugins/observability_solution/asset_manager/server/templates/{oam_template.ts => entities_template.ts} (79%) diff --git a/x-pack/packages/kbn-oam-schema/index.ts b/x-pack/packages/kbn-oam-schema/index.ts index 2d13c00271ed11..92b93b79381251 100644 --- a/x-pack/packages/kbn-oam-schema/index.ts +++ b/x-pack/packages/kbn-oam-schema/index.ts @@ -5,6 +5,6 @@ * 2.0. */ -export * from './src/schema/oam_definition'; -export * from './src/schema/asset'; +export * from './src/schema/entity_definition'; +export * from './src/schema/entity'; export * from './src/schema/common'; diff --git a/x-pack/packages/kbn-oam-schema/src/schema/asset.ts b/x-pack/packages/kbn-oam-schema/src/schema/entity.ts similarity index 100% rename from x-pack/packages/kbn-oam-schema/src/schema/asset.ts rename to x-pack/packages/kbn-oam-schema/src/schema/entity.ts diff --git a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts b/x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts similarity index 89% rename from x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts rename to x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts index 6226c5c22d866c..8f1daac6900042 100644 --- a/x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts +++ b/x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts @@ -15,7 +15,7 @@ import { durationSchema, } from './common'; -export const oamDefinitionSchema = z.object({ +export const entityDefinitionSchema = z.object({ id: z.string().regex(/^[\w-]+$/), name: z.string(), description: z.optional(z.string()), @@ -38,4 +38,4 @@ export const oamDefinitionSchema = z.object({ ), }); -export type OAMDefinition = z.infer; +export type EntityDefinition = z.infer; diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts new file mode 100644 index 00000000000000..8b1eceb47a6ec0 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ENTITY_VERSION = 'v1'; +export const ENTITY_BASE_PREFIX = `.entities-observability.entity-${ENTITY_VERSION}`; +export const ENTITY_TRANSFORM_PREFIX = `entities-observability-entity-${ENTITY_VERSION}`; +export const ENTITY_DEFAULT_FREQUENCY = '1m'; +export const ENTITY_DEFAULT_SYNC_DELAY = '60s'; +export const ENTITY_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts deleted file mode 100644 index 2e72483f78f906..00000000000000 --- a/x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const OAM_VERSION = 'v1'; -export const OAM_BASE_PREFIX = `.oam-observability.entities-${OAM_VERSION}`; -export const OAM_TRANSFORM_PREFIX = `oam-observability-entities-${OAM_VERSION}`; -export const OAM_DEFAULT_FREQUENCY = '1m'; -export const OAM_DEFAULT_SYNC_DELAY = '60s'; -export const OAM_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts similarity index 75% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index bbd0fee4118b72..73253775afaed4 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -6,15 +6,15 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateProcessors } from './ingest_pipeline/generate_processors'; import { retryTransientEsErrors } from './helpers/retry'; -import { OAMSecurityException } from './errors/oam_security_exception'; +import { EntitySecurityException } from './errors/entity_security_exception'; import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; export async function createAndInstallIngestPipeline( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const processors = generateProcessors(definition); @@ -29,9 +29,9 @@ export async function createAndInstallIngestPipeline( { logger } ); } catch (e) { - logger.error(`Cannot create OAM ingest pipeline for [${definition.id}] entity defintion`); + logger.error(`Cannot create entity ingest pipeline for [${definition.id}] entity defintion`); if (e.meta?.body?.error?.type === 'security_exception') { - throw new OAMSecurityException(e.meta.body.error.reason, definition); + throw new EntitySecurityException(e.meta.body.error.reason, definition); } throw e; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts similarity index 71% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts index 2669949706efe3..7943ae306333c6 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts @@ -6,23 +6,23 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateTransform } from './transform/generate_transform'; import { retryTransientEsErrors } from './helpers/retry'; -import { OAMSecurityException } from './errors/oam_security_exception'; +import { EntitySecurityException } from './errors/entity_security_exception'; export async function createAndInstallTransform( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const transform = generateTransform(definition); try { await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger }); } catch (e) { - logger.error(`Cannot create OAM transform for [${definition.id}] entity definition`); + logger.error(`Cannot create entity transform for [${definition.id}] entity definition`); if (e.meta?.body?.error?.type === 'security_exception') { - throw new OAMSecurityException(e.meta.body.error.reason, definition); + throw new EntitySecurityException(e.meta.body.error.reason, definition); } throw e; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts new file mode 100644 index 00000000000000..b2b28a8f31cbac --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/oam-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityDefinitionNotFound } from './errors/entity_not_found'; + +export async function deleteEntityDefinition( + soClient: SavedObjectsClientContract, + definition: EntityDefinition, + logger: Logger +) { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 0) { + logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`); + throw new EntityDefinitionNotFound(`Entity defintion with [${definition.id}] not found.`); + } + + await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts similarity index 79% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts index 2aeded140fb1d7..fb78281b7fed5e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts @@ -6,19 +6,19 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateIndexName } from './helpers/generate_index_name'; export async function deleteIndex( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const indexName = generateIndexName(definition); try { await esClient.indices.delete({ index: indexName, ignore_unavailable: true }); } catch (e) { - logger.error(`Unable to remove OAM Defintion index [${definition.id}}]`); + logger.error(`Unable to remove entity defintion index [${definition.id}}]`); throw e; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts similarity index 91% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts index 5058c4ad25e8bf..a56527dfc1bdc2 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; import { retryTransientEsErrors } from './helpers/retry'; export async function deleteIngestPipeline( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const pipelineId = generateIngestPipelineId(definition); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts similarity index 57% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts index af598d37d01f11..9a15431392bf42 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts @@ -5,14 +5,14 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; -export class OAMIdConflict extends Error { - public defintion: OAMDefinition; +export class EntityIdConflict extends Error { + public defintion: EntityDefinition; - constructor(message: string, def: OAMDefinition) { + constructor(message: string, def: EntityDefinition) { super(message); - this.name = 'OAMIdConflict'; + this.name = 'EntityIdConflict'; this.defintion = def; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts similarity index 76% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts index 31a232bf88485f..d81cd3322cae6e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts @@ -5,9 +5,9 @@ * 2.0. */ -export class OAMDefinitionNotFound extends Error { +export class EntityDefinitionNotFound extends Error { constructor(message: string) { super(message); - this.name = 'OAMDefinitionNotFound'; + this.name = 'EntityDefinitionNotFound'; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts similarity index 56% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts index f2f5dfea2570e2..bfc2fab7cb083b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts @@ -5,14 +5,14 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; -export class OAMSecurityException extends Error { - public defintion: OAMDefinition; +export class EntitySecurityException extends Error { + public defintion: EntityDefinition; - constructor(message: string, def: OAMDefinition) { + constructor(message: string, def: EntityDefinition) { super(message); - this.name = 'OAMSecurityException'; + this.name = 'EntitySecurityException'; this.defintion = def; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts similarity index 100% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/invalid_transform_error.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts similarity index 89% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index 7ddd199c3cd748..f1ca045de874f9 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { oamDefinitionSchema } from '@kbn/oam-schema'; -export const oamDefinition = oamDefinitionSchema.parse({ +import { entityDefinitionSchema } from '@kbn/oam-schema'; +export const entityDefinition = entityDefinitionSchema.parse({ id: 'admin-console-logs-service', name: 'Services for Admin Console', type: 'service', diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts similarity index 51% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts index 8ca63de51db074..677c320c7f4cc5 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; -import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; +import { EntityDefinition } from '@kbn/oam-schema'; +import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; -export function generateIndexName(definition: OAMDefinition) { - return `${OAM_BASE_PREFIX}.${definition.id}`; +export function generateIndexName(definition: EntityDefinition) { + return `${ENTITY_BASE_PREFIX}.${definition.id}`; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts similarity index 100% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/get_elasticsearch_query_or_throw.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts similarity index 100% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/retry.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap similarity index 95% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap index 333a2ec8ccf464..cf0b6e7b5f7fce 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -63,7 +63,7 @@ if (ctx.entity?.metadata?.kubernetes?.pod?.name != null) { Object { "set": Object { "field": "_index", - "value": ".oam-observability.entities-v1.admin-console-logs-service", + "value": ".entities-observability.entity-v1.admin-console-logs-service", }, }, ] diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts similarity index 50% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts index b21a69954ce194..c89404fb54972c 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_ingest_pipeline_id.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; -import { OAM_BASE_PREFIX } from '../../../../common/constants_oam'; +import { EntityDefinition } from '@kbn/oam-schema'; +import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; -export function generateIngestPipelineId(definition: OAMDefinition) { - return `${OAM_BASE_PREFIX}.${definition.id}`; +export function generateIngestPipelineId(definition: EntityDefinition) { + return `${ENTITY_BASE_PREFIX}.${definition.id}`; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts similarity index 77% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts index b0e15f054ddb09..33919ec678dcf1 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.test.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts @@ -6,11 +6,11 @@ */ import { generateProcessors } from './generate_processors'; -import { oamDefinition } from '../helpers/fixtures/oam_definition'; +import { entityDefinition } from '../helpers/fixtures/entity_definition'; describe('generateProcessors(definition)', () => { it('should genearte a valid pipeline', () => { - const processors = generateProcessors(oamDefinition); + const processors = generateProcessors(entityDefinition); expect(processors).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts similarity index 90% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts index 15f60fb4dbfdf7..810762716a8f0b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/ingest_pipeline/generate_processors.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateIndexName } from '../helpers/generate_index_name'; -function createIdTemplate(definition: OAMDefinition) { +function createIdTemplate(definition: EntityDefinition) { return definition.identityFields.reduce((template, field) => { return template.replaceAll(field, `entity.identity.${field}`); }, definition.identityTemplate); @@ -29,7 +29,7 @@ function mapDesitnationToPainless(destination: string, source: string) { }, ''); } -function createMetadataPainlessScript(definition: OAMDefinition) { +function createMetadataPainlessScript(definition: EntityDefinition) { if (!definition.metadata) { return ''; } @@ -43,7 +43,7 @@ function createMetadataPainlessScript(definition: OAMDefinition) { }, ''); } -export function generateProcessors(definition: OAMDefinition) { +export function generateProcessors(definition: EntityDefinition) { return [ { set: { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts new file mode 100644 index 00000000000000..5a4132e716eec3 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/oam-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityDefinitionNotFound } from './errors/entity_not_found'; + +export async function readEntityDefinition( + soClient: SavedObjectsClientContract, + id: string, + logger: Logger +) { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${id})`, + }); + if (response.total === 0) { + const message = `Unable to find entity defintion with [${id}]`; + logger.error(message); + throw new EntityDefinitionNotFound(message); + } + + try { + return entityDefinitionSchema.parse(response.saved_objects[0].attributes); + } catch (e) { + logger.error(`Unable to parse entity defintion with [${id}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts new file mode 100644 index 00000000000000..855cab620de2c0 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/oam-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityIdConflict } from './errors/entity_id_conflict_error'; + +export async function saveEntityDefinition( + soClient: SavedObjectsClientContract, + definition: EntityDefinition +): Promise { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 1) { + throw new EntityIdConflict( + `Entity defintion with [${definition.id}] already exists.`, + definition + ); + } + + await soClient.create(SO_ENTITY_DEFINITION_TYPE, definition, { + id: definition.id, + overwrite: true, + }); + + return definition; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts similarity index 84% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts index d2e3c9c69b8ed3..5f7866d679f93c 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/start_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { retryTransientEsErrors } from './helpers/retry'; import { generateTransformId } from './transform/generate_transform_id'; export async function startTransform( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const transformId = generateTransformId(definition); @@ -22,7 +22,7 @@ export async function startTransform( { logger } ); } catch (err) { - logger.error(`Cannot start OAM transform [${transformId}]`); + logger.error(`Cannot start entity transform [${transformId}]`); throw err; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts similarity index 86% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts index ecf7ecc0b4eb79..7c0b89dd6aaa25 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/stop_and_delete_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { generateTransformId } from './transform/generate_transform_id'; import { retryTransientEsErrors } from './helpers/retry'; export async function stopAndDeleteTransform( esClient: ElasticsearchClient, - definition: OAMDefinition, + definition: EntityDefinition, logger: Logger ) { const transformId = generateTransformId(definition); @@ -31,7 +31,7 @@ export async function stopAndDeleteTransform( { logger } ); } catch (e) { - logger.error(`Cannot stop or delete OAM transform [${transformId}]`); + logger.error(`Cannot stop or delete entity transform [${transformId}]`); throw e; } } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap similarity index 92% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap index b706949bbc9f42..2b5fec6f220cfc 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/__snapshots__/generate_transform.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap @@ -4,8 +4,8 @@ exports[`generateTransform(definition) should generate a valid summary transform Object { "defer_validation": true, "dest": Object { - "index": ".oam-observability.entities-v1.noop", - "pipeline": ".oam-observability.entities-v1.admin-console-logs-service", + "index": ".entities-observability.entity-v1.noop", + "pipeline": ".entities-observability.entity-v1.admin-console-logs-service", }, "frequency": "1m", "pivot": Object { @@ -120,6 +120,6 @@ Object { "field": "@timestamp", }, }, - "transform_id": "oam-observability-entities-v1-admin-console-logs-service", + "transform_id": "entities-observability-entities-v1-admin-console-logs-service", } `; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts similarity index 82% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 8177b07e8f39dc..cf3ff3aee07dd8 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; -export function generateMetadataAggregations(definition: OAMDefinition) { +export function generateMetadataAggregations(definition: EntityDefinition) { if (!definition.metadata) { return {}; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts similarity index 95% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts index e2947c584f19e8..8e7a5d456844a0 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_metric_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { KeyMetric, Metric, OAMDefinition } from '@kbn/oam-schema'; +import { KeyMetric, Metric, EntityDefinition } from '@kbn/oam-schema'; import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; import { InvalidTransformError } from '../errors/invalid_transform_error'; @@ -104,7 +104,7 @@ function buildMetricEquation(keyMetric: KeyMetric) { }; } -export function generateMetricAggregations(definition: OAMDefinition) { +export function generateMetricAggregations(definition: EntityDefinition) { if (!definition.metrics) { return {}; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts similarity index 78% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts index 78ce7d8a3c6195..e97293b77dd4fe 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.test.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { oamDefinition } from '../helpers/fixtures/oam_definition'; +import { entityDefinition } from '../helpers/fixtures/entity_definition'; import { generateTransform } from './generate_transform'; describe('generateTransform(definition)', () => { it('should generate a valid summary transform', () => { - const transform = generateTransform(oamDefinition); + const transform = generateTransform(entityDefinition); expect(transform).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts similarity index 81% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts index 713bc77903f7f0..bcc6d1ad5900b5 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; import { QueryDslQueryContainer, TransformPutTransformRequest, @@ -13,15 +13,15 @@ import { import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; import { generateMetricAggregations } from './generate_metric_aggregations'; import { - OAM_BASE_PREFIX, - OAM_DEFAULT_FREQUENCY, - OAM_DEFAULT_SYNC_DELAY, -} from '../../../../common/constants_oam'; + ENTITY_BASE_PREFIX, + ENTITY_DEFAULT_FREQUENCY, + ENTITY_DEFAULT_SYNC_DELAY, +} from '../../../../common/constants_entities'; import { generateMetadataAggregations } from './generate_metadata_aggregations'; import { generateTransformId } from './generate_transform_id'; import { generateIngestPipelineId } from '../ingest_pipeline/generate_ingest_pipeline_id'; -export function generateTransform(definition: OAMDefinition): TransformPutTransformRequest { +export function generateTransform(definition: EntityDefinition): TransformPutTransformRequest { const filter: QueryDslQueryContainer[] = [ { range: { @@ -48,14 +48,14 @@ export function generateTransform(definition: OAMDefinition): TransformPutTransf }, }, dest: { - index: `${OAM_BASE_PREFIX}.noop`, + index: `${ENTITY_BASE_PREFIX}.noop`, pipeline: generateIngestPipelineId(definition), }, - frequency: definition.settings?.frequency || OAM_DEFAULT_FREQUENCY, + frequency: definition.settings?.frequency || ENTITY_DEFAULT_FREQUENCY, sync: { time: { field: definition.settings?.syncField ?? definition.timestampField, - delay: definition.settings?.syncDelay ?? OAM_DEFAULT_SYNC_DELAY, + delay: definition.settings?.syncDelay ?? ENTITY_DEFAULT_SYNC_DELAY, }, }, settings: { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts similarity index 50% rename from x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts rename to x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts index 370d896b5c83b0..7caceb6a8b9cc0 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/transform/generate_transform_id.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { OAMDefinition } from '@kbn/oam-schema'; -import { OAM_TRANSFORM_PREFIX } from '../../../../common/constants_oam'; +import { EntityDefinition } from '@kbn/oam-schema'; +import { ENTITY_TRANSFORM_PREFIX } from '../../../../common/constants_entities'; -export function generateTransformId(definition: OAMDefinition) { - return `${OAM_TRANSFORM_PREFIX}-${definition.id}`; +export function generateTransformId(definition: EntityDefinition) { + return `${ENTITY_TRANSFORM_PREFIX}-${definition.id}`; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts deleted file mode 100644 index d73d6429e136f8..00000000000000 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_oam_definition.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; -import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; -import { OAMDefinitionNotFound } from './errors/oam_not_found'; - -export async function deleteOAMDefinition( - soClient: SavedObjectsClientContract, - definition: OAMDefinition, - logger: Logger -) { - const response = await soClient.find({ - type: SO_OAM_DEFINITION_TYPE, - page: 1, - perPage: 1, - filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${definition.id})`, - }); - - if (response.total === 0) { - logger.error(`Unable to delete OAM Definition [${definition.id}] because it doesn't exist.`); - throw new OAMDefinitionNotFound(`OAM Defintion with [${definition.id}] not found.`); - } - - await soClient.delete(SO_OAM_DEFINITION_TYPE, response.saved_objects[0].id); -} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts deleted file mode 100644 index ce23a59112c0cc..00000000000000 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/read_oam_definition.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { OAMDefinition, oamDefinitionSchema } from '@kbn/oam-schema'; -import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; -import { OAMDefinitionNotFound } from './errors/oam_not_found'; - -export async function readOAMDefinition( - soClient: SavedObjectsClientContract, - id: string, - logger: Logger -) { - const response = await soClient.find({ - type: SO_OAM_DEFINITION_TYPE, - page: 1, - perPage: 1, - filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${id})`, - }); - if (response.total === 0) { - const message = `Unable to find OAM Defintion with [${id}]`; - logger.error(message); - throw new OAMDefinitionNotFound(message); - } - - try { - return oamDefinitionSchema.parse(response.saved_objects[0].attributes); - } catch (e) { - logger.error(`Unable to parse OAM Defintion with [${id}]`); - throw e; - } -} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts deleted file mode 100644 index e2ac929bcb7011..00000000000000 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/oam/save_oam_definition.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SavedObjectsClientContract } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; -import { SO_OAM_DEFINITION_TYPE } from '../../saved_objects'; -import { OAMIdConflict } from './errors/oam_id_conflict_error'; - -export async function saveOAMDefinition( - soClient: SavedObjectsClientContract, - definition: OAMDefinition -): Promise { - const response = await soClient.find({ - type: SO_OAM_DEFINITION_TYPE, - page: 1, - perPage: 1, - filter: `${SO_OAM_DEFINITION_TYPE}.attributes.id:(${definition.id})`, - }); - - if (response.total === 1) { - throw new OAMIdConflict(`OAM Defintion with [${definition.id}] already exists.`, definition); - } - - await soClient.create(SO_OAM_DEFINITION_TYPE, definition, { - id: definition.id, - overwrite: true, - }); - - return definition; -} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts index 0870ff1f44bf0b..4c947926cf571b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts @@ -21,11 +21,11 @@ import { assetsIndexTemplateConfig } from './templates/assets_template'; import { AssetClient } from './lib/asset_client'; import { AssetManagerPluginSetupDependencies, AssetManagerPluginStartDependencies } from './types'; import { AssetManagerConfig, configSchema, exposeToBrowserConfig } from '../common/config'; -import { oamBaseComponentTemplateConfig } from './templates/components/base'; -import { oamEventComponentTemplateConfig } from './templates/components/event'; -import { oamIndexTemplateConfig } from './templates/oam_template'; -import { oamDefinition } from './saved_objects'; -import { oamEntityComponentTemplateConfig } from './templates/components/entity'; +import { entitiesBaseComponentTemplateConfig } from './templates/components/base'; +import { entitiesEventComponentTemplateConfig } from './templates/components/event'; +import { entitiesIndexTemplateConfig } from './templates/entities_template'; +import { entityDefinition } from './saved_objects'; +import { entitiesEntityComponentTemplateConfig } from './templates/components/entity'; export type AssetManagerServerPluginSetup = ReturnType; export type AssetManagerServerPluginStart = ReturnType; @@ -61,7 +61,7 @@ export class AssetManagerServerPlugin this.logger.info('Server is enabled'); - core.savedObjects.registerType(oamDefinition); + core.savedObjects.registerType(entityDefinition); const assetClient = new AssetClient({ sourceIndices: this.config.sourceIndices, @@ -90,22 +90,26 @@ export class AssetManagerServerPlugin logger: this.logger, }).catch(() => {}); // it shouldn't reject, but just in case - // Install OAM compoent templates and index template + // Install entities compoent templates and index template Promise.all([ - upsertComponent({ esClient, logger: this.logger, component: oamBaseComponentTemplateConfig }), upsertComponent({ esClient, logger: this.logger, - component: oamEventComponentTemplateConfig, + component: entitiesBaseComponentTemplateConfig, }), upsertComponent({ esClient, logger: this.logger, - component: oamEntityComponentTemplateConfig, + component: entitiesEventComponentTemplateConfig, + }), + upsertComponent({ + esClient, + logger: this.logger, + component: entitiesEntityComponentTemplateConfig, }), ]) .then(() => - upsertTemplate({ esClient, logger: this.logger, template: oamIndexTemplateConfig }) + upsertTemplate({ esClient, logger: this.logger, template: entitiesIndexTemplateConfig }) ) .catch(() => {}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts similarity index 54% rename from x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts rename to x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts index c984c585caa72e..8dabdb806b6223 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts @@ -6,32 +6,32 @@ */ import { RequestHandlerContext } from '@kbn/core/server'; -import { OAMDefinition, oamDefinitionSchema } from '@kbn/oam-schema'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/oam-schema'; import { stringifyZodError } from '@kbn/zod-helpers'; import { SetupRouteOptions } from '../types'; -import { saveOAMDefinition } from '../../lib/oam/save_oam_definition'; -import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install_ingest_pipeline'; -import { OAMIdConflict } from '../../lib/oam/errors/oam_id_conflict_error'; -import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; -import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; -import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; -import { startTransform } from '../../lib/oam/start_transform'; -import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; -import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; -import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; -import { OAM_API_PREFIX } from '../../../common/constants_oam'; +import { saveEntityDefinition } from '../../lib/entities/save_entity_definition'; +import { createAndInstallIngestPipeline } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { EntityIdConflict } from '../../lib/entities/errors/entity_id_conflict_error'; +import { createAndInstallTransform } from '../../lib/entities/create_and_install_transform'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { startTransform } from '../../lib/entities/start_transform'; +import { deleteEntityDefinition } from '../../lib/entities/delete_entity_definition'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; -export function createOAMDefinitionRoute({ +export function createEntityDefinitionRoute({ router, logger, }: SetupRouteOptions) { - router.post( + router.post( { - path: `${OAM_API_PREFIX}/definition`, + path: `${ENTITY_API_PREFIX}/definition`, validate: { body: (body, res) => { try { - return res.ok(oamDefinitionSchema.parse(body)); + return res.ok(entityDefinitionSchema.parse(body)); } catch (e) { return res.badRequest(stringifyZodError(e)); } @@ -45,7 +45,7 @@ export function createOAMDefinitionRoute({ const soClient = (await context.core).savedObjects.client; const esClient = (await context.core).elasticsearch.client.asCurrentUser; try { - const definition = await saveOAMDefinition(soClient, req.body); + const definition = await saveEntityDefinition(soClient, req.body); definitionCreated = true; await createAndInstallIngestPipeline(esClient, definition, logger); ingestPipelineCreated = true; @@ -57,7 +57,7 @@ export function createOAMDefinitionRoute({ } catch (e) { // Clean up anything that was successful. if (definitionCreated) { - await deleteOAMDefinition(soClient, req.body, logger); + await deleteEntityDefinition(soClient, req.body, logger); } if (ingestPipelineCreated) { await deleteIngestPipeline(esClient, req.body, logger); @@ -65,10 +65,10 @@ export function createOAMDefinitionRoute({ if (transformCreated) { await stopAndDeleteTransform(esClient, req.body, logger); } - if (e instanceof OAMIdConflict) { + if (e instanceof EntityIdConflict) { return res.conflict({ body: e }); } - if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { return res.customError({ body: e, statusCode: 400 }); } return res.customError({ body: e, statusCode: 500 }); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts similarity index 52% rename from x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts rename to x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts index 5698937dc7a681..e1b273780a64fb 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/delete.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts @@ -8,22 +8,22 @@ import { RequestHandlerContext } from '@kbn/core/server'; import { schema } from '@kbn/config-schema'; import { SetupRouteOptions } from '../types'; -import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; -import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; -import { readOAMDefinition } from '../../lib/oam/read_oam_definition'; -import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; -import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; -import { deleteOAMDefinition } from '../../lib/oam/delete_oam_definition'; -import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; -import { OAM_API_PREFIX } from '../../../common/constants_oam'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { deleteEntityDefinition } from '../../lib/entities/delete_entity_definition'; +import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; -export function deleteOAMDefinitionRoute({ +export function deleteEntityDefinitionRoute({ router, logger, }: SetupRouteOptions) { router.delete<{ id: string }, unknown, unknown>( { - path: `${OAM_API_PREFIX}/definition/{id}`, + path: `${ENTITY_API_PREFIX}/definition/{id}`, validate: { params: schema.object({ id: schema.string(), @@ -35,17 +35,17 @@ export function deleteOAMDefinitionRoute({ const soClient = (await context.core).savedObjects.client; const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const definition = await readOAMDefinition(soClient, req.params.id, logger); + const definition = await readEntityDefinition(soClient, req.params.id, logger); await stopAndDeleteTransform(esClient, definition, logger); await deleteIngestPipeline(esClient, definition, logger); - await deleteOAMDefinition(soClient, definition, logger); + await deleteEntityDefinition(soClient, definition, logger); return res.ok({ body: { acknowledged: true } }); } catch (e) { - if (e instanceof OAMDefinitionNotFound) { + if (e instanceof EntityDefinitionNotFound) { return res.notFound({ body: e }); } - if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { return res.customError({ body: e, statusCode: 400 }); } return res.customError({ body: e, statusCode: 500 }); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts similarity index 55% rename from x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts rename to x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts index 6193eb1bc46936..3109ffc44520f5 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/oam/reset.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts @@ -8,25 +8,25 @@ import { RequestHandlerContext } from '@kbn/core/server'; import { schema } from '@kbn/config-schema'; import { SetupRouteOptions } from '../types'; -import { OAMSecurityException } from '../../lib/oam/errors/oam_security_exception'; -import { InvalidTransformError } from '../../lib/oam/errors/invalid_transform_error'; -import { readOAMDefinition } from '../../lib/oam/read_oam_definition'; -import { stopAndDeleteTransform } from '../../lib/oam/stop_and_delete_transform'; -import { deleteIngestPipeline } from '../../lib/oam/delete_ingest_pipeline'; -import { deleteIndex } from '../../lib/oam/delete_index'; -import { createAndInstallIngestPipeline } from '../../lib/oam/create_and_install_ingest_pipeline'; -import { createAndInstallTransform } from '../../lib/oam/create_and_install_transform'; -import { startTransform } from '../../lib/oam/start_transform'; -import { OAMDefinitionNotFound } from '../../lib/oam/errors/oam_not_found'; -import { OAM_API_PREFIX } from '../../../common/constants_oam'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { deleteIndex } from '../../lib/entities/delete_index'; +import { createAndInstallIngestPipeline } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { createAndInstallTransform } from '../../lib/entities/create_and_install_transform'; +import { startTransform } from '../../lib/entities/start_transform'; +import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; -export function resetOAMDefinitionRoute({ +export function resetEntityDefinitionRoute({ router, logger, }: SetupRouteOptions) { router.post<{ id: string }, unknown, unknown>( { - path: `${OAM_API_PREFIX}/definition/{id}/_reset`, + path: `${ENTITY_API_PREFIX}/definition/{id}/_reset`, validate: { params: schema.object({ id: schema.string(), @@ -38,7 +38,7 @@ export function resetOAMDefinitionRoute({ const soClient = (await context.core).savedObjects.client; const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const definition = await readOAMDefinition(soClient, req.params.id, logger); + const definition = await readEntityDefinition(soClient, req.params.id, logger); // Delete the transform and ingest pipeline await stopAndDeleteTransform(esClient, definition, logger); @@ -52,10 +52,10 @@ export function resetOAMDefinitionRoute({ return res.ok({ body: { acknowledged: true } }); } catch (e) { - if (e instanceof OAMDefinitionNotFound) { + if (e instanceof EntityDefinitionNotFound) { return res.notFound({ body: e }); } - if (e instanceof OAMSecurityException || e instanceof InvalidTransformError) { + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { return res.customError({ body: e, statusCode: 400 }); } return res.customError({ body: e, statusCode: 500 }); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts index 27a3bbc9d52edd..d0b6c9f7ff0f11 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts @@ -14,9 +14,9 @@ import { hostsRoutes } from './assets/hosts'; import { servicesRoutes } from './assets/services'; import { containersRoutes } from './assets/containers'; import { podsRoutes } from './assets/pods'; -import { createOAMDefinitionRoute } from './oam/create'; -import { deleteOAMDefinitionRoute } from './oam/delete'; -import { resetOAMDefinitionRoute } from './oam/reset'; +import { createEntityDefinitionRoute } from './entities/create'; +import { deleteEntityDefinitionRoute } from './entities/delete'; +import { resetEntityDefinitionRoute } from './entities/reset'; export function setupRoutes({ router, @@ -30,7 +30,7 @@ export function setupRoutes({ servicesRoutes({ router, assetClient, logger }); containersRoutes({ router, assetClient, logger }); podsRoutes({ router, assetClient, logger }); - createOAMDefinitionRoute({ router, assetClient, logger }); - deleteOAMDefinitionRoute({ router, assetClient, logger }); - resetOAMDefinitionRoute({ router, assetClient, logger }); + createEntityDefinitionRoute({ router, assetClient, logger }); + deleteEntityDefinitionRoute({ router, assetClient, logger }); + resetEntityDefinitionRoute({ router, assetClient, logger }); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts similarity index 71% rename from x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts rename to x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts index 375831368494a5..10889d70cdccfa 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/oam_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts @@ -6,12 +6,12 @@ */ import { SavedObject, SavedObjectsType } from '@kbn/core/server'; -import { OAMDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/oam-schema'; -export const SO_OAM_DEFINITION_TYPE = 'oam-definition'; +export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition'; -export const oamDefinition: SavedObjectsType = { - name: SO_OAM_DEFINITION_TYPE, +export const entityDefinition: SavedObjectsType = { + name: SO_ENTITY_DEFINITION_TYPE, hidden: false, namespaceType: 'multiple-isolated', mappings: { @@ -31,10 +31,10 @@ export const oamDefinition: SavedObjectsType = { }, }, management: { - displayName: 'OAM Definition', + displayName: 'Entity Definition', importableAndExportable: false, - getTitle(sloSavedObject: SavedObject) { - return `OAM: [${sloSavedObject.attributes.name}]`; + getTitle(sloSavedObject: SavedObject) { + return `EntityDefinition: [${sloSavedObject.attributes.name}]`; }, }, }; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts index 58e990bc2b72b2..6145b05438bb24 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { oamDefinition, SO_OAM_DEFINITION_TYPE } from './oam_definition'; +export { entityDefinition, SO_ENTITY_DEFINITION_TYPE } from './entity_definition'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts index eab6a09ae920a2..adf527d653d9c7 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts @@ -7,8 +7,8 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -export const oamBaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: 'oam_v1_base', +export const entitiesBaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_base', _meta: { documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-base.html', ecs_version: '8.0.0', diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts index 5d2a615c01fa43..e696d32e0dfb21 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts @@ -7,8 +7,8 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -export const oamEntityComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: 'oam_v1_entity', +export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_entity', _meta: { ecs_version: '8.0.0', }, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts index b516f5ba64ffb6..6ad4b628fdf36a 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts @@ -7,8 +7,8 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -export const oamEventComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: 'oam_v1_event', +export const entitiesEventComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_event', _meta: { documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-event.html', ecs_version: '8.0.0', diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts similarity index 79% rename from x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts rename to x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts index 9017f86fdde97a..05c50849f989e1 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/oam_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts @@ -7,14 +7,14 @@ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -export const oamIndexTemplateConfig: IndicesPutIndexTemplateRequest = { - name: 'oam_v1_index_template', +export const entitiesIndexTemplateConfig: IndicesPutIndexTemplateRequest = { + name: 'entities_v1_index_template', _meta: { - description: 'The OAM index template', + description: 'The entities index template', ecs_version: '8.0.0', }, - composed_of: ['oam_v1_base', 'oam_v1_event', 'oam_v1_entity'], - index_patterns: ['.oam-observability.entities-v1.*'], + composed_of: ['entities_v1_base', 'entities_v1_event', 'entities_v1_entity'], + index_patterns: ['.entities-observability.entity-v1.*'], priority: 1, template: { mappings: { From c2953ea98e61e321d79d7e5eca5098d4091202fd Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 14 May 2024 11:02:59 -0600 Subject: [PATCH 12/16] changing index name to summary from entity --- .../asset_manager/common/constants_entities.ts | 4 ++-- .../__snapshots__/generate_processors.test.ts.snap | 2 +- .../transform/__snapshots__/generate_transform.test.ts.snap | 6 +++--- .../asset_manager/server/templates/entities_template.ts | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts index 8b1eceb47a6ec0..aeeea398220a08 100644 --- a/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts @@ -6,8 +6,8 @@ */ export const ENTITY_VERSION = 'v1'; -export const ENTITY_BASE_PREFIX = `.entities-observability.entity-${ENTITY_VERSION}`; -export const ENTITY_TRANSFORM_PREFIX = `entities-observability-entity-${ENTITY_VERSION}`; +export const ENTITY_BASE_PREFIX = `.entities-observability.summary-${ENTITY_VERSION}`; +export const ENTITY_TRANSFORM_PREFIX = `entities-observability-summary-${ENTITY_VERSION}`; export const ENTITY_DEFAULT_FREQUENCY = '1m'; export const ENTITY_DEFAULT_SYNC_DELAY = '60s'; export const ENTITY_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap index cf0b6e7b5f7fce..443063d70db60d 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -63,7 +63,7 @@ if (ctx.entity?.metadata?.kubernetes?.pod?.name != null) { Object { "set": Object { "field": "_index", - "value": ".entities-observability.entity-v1.admin-console-logs-service", + "value": ".entities-observability.summary-v1.admin-console-logs-service", }, }, ] diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap index 2b5fec6f220cfc..f4707ca71dbdba 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap @@ -4,8 +4,8 @@ exports[`generateTransform(definition) should generate a valid summary transform Object { "defer_validation": true, "dest": Object { - "index": ".entities-observability.entity-v1.noop", - "pipeline": ".entities-observability.entity-v1.admin-console-logs-service", + "index": ".entities-observability.summary-v1.noop", + "pipeline": ".entities-observability.summary-v1.admin-console-logs-service", }, "frequency": "1m", "pivot": Object { @@ -120,6 +120,6 @@ Object { "field": "@timestamp", }, }, - "transform_id": "entities-observability-entities-v1-admin-console-logs-service", + "transform_id": "entities-observability-summary-v1-admin-console-logs-service", } `; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts index 05c50849f989e1..26ec5cc0661f81 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts @@ -6,6 +6,7 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ENTITY_BASE_PREFIX } from '../../common/constants_entities'; export const entitiesIndexTemplateConfig: IndicesPutIndexTemplateRequest = { name: 'entities_v1_index_template', @@ -14,7 +15,7 @@ export const entitiesIndexTemplateConfig: IndicesPutIndexTemplateRequest = { ecs_version: '8.0.0', }, composed_of: ['entities_v1_base', 'entities_v1_event', 'entities_v1_entity'], - index_patterns: ['.entities-observability.entity-v1.*'], + index_patterns: [`${ENTITY_BASE_PREFIX}.*`], priority: 1, template: { mappings: { From 1998069b9101fab4aba433c2caf53cc1b4adb6a3 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 16 May 2024 14:25:31 -0600 Subject: [PATCH 13/16] Moving kbn-oam-schema to kbn-entities-schema --- .github/CODEOWNERS | 2 +- package.json | 2 +- tsconfig.base.json | 4 ++-- x-pack/packages/kbn-entities-schema/README.md | 3 +++ .../{kbn-oam-schema => kbn-entities-schema}/index.ts | 0 .../jest.config.js | 2 +- .../{kbn-oam-schema => kbn-entities-schema}/kibana.jsonc | 2 +- .../{kbn-oam-schema => kbn-entities-schema}/package.json | 2 +- .../src/schema/common.ts | 0 .../src/schema/entity.ts | 0 .../src/schema/entity_definition.ts | 0 .../{kbn-oam-schema => kbn-entities-schema}/tsconfig.json | 0 x-pack/packages/kbn-oam-schema/README.md | 3 --- .../lib/entities/create_and_install_ingest_pipeline.ts | 2 +- .../server/lib/entities/create_and_install_transform.ts | 2 +- .../server/lib/entities/delete_entity_definition.ts | 2 +- .../asset_manager/server/lib/entities/delete_index.ts | 2 +- .../server/lib/entities/delete_ingest_pipeline.ts | 2 +- .../lib/entities/errors/entity_id_conflict_error.ts | 2 +- .../lib/entities/errors/entity_security_exception.ts | 2 +- .../lib/entities/helpers/fixtures/entity_definition.ts | 2 +- .../server/lib/entities/helpers/generate_index_name.ts | 2 +- .../ingest_pipeline/generate_ingest_pipeline_id.ts | 2 +- .../lib/entities/ingest_pipeline/generate_processors.ts | 2 +- .../server/lib/entities/read_entity_definition.ts | 2 +- .../server/lib/entities/save_entity_definition.ts | 2 +- .../asset_manager/server/lib/entities/start_transform.ts | 2 +- .../server/lib/entities/stop_and_delete_transform.ts | 2 +- .../entities/transform/generate_metadata_aggregations.ts | 2 +- .../entities/transform/generate_metric_aggregations.ts | 2 +- .../server/lib/entities/transform/generate_transform.ts | 2 +- .../lib/entities/transform/generate_transform_id.ts | 2 +- .../asset_manager/server/routes/entities/create.ts | 2 +- .../server/saved_objects/entity_definition.ts | 2 +- .../observability_solution/asset_manager/tsconfig.json | 2 +- yarn.lock | 8 ++++---- 36 files changed, 36 insertions(+), 36 deletions(-) create mode 100644 x-pack/packages/kbn-entities-schema/README.md rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/index.ts (100%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/jest.config.js (84%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/kibana.jsonc (69%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/package.json (69%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/src/schema/common.ts (100%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/src/schema/entity.ts (100%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/src/schema/entity_definition.ts (100%) rename x-pack/packages/{kbn-oam-schema => kbn-entities-schema}/tsconfig.json (100%) delete mode 100644 x-pack/packages/kbn-oam-schema/README.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9764f7e8d3c21e..15524fc5e8a959 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -600,7 +600,7 @@ src/plugins/newsfeed @elastic/kibana-core test/common/plugins/newsfeed @elastic/kibana-core src/plugins/no_data_page @elastic/appex-sharedux x-pack/plugins/notifications @elastic/appex-sharedux -x-pack/packages/kbn-oam-schema @elastic/obs-knowledge-team +x-pack/packages/kbn-entities-schema @elastic/obs-knowledge-team packages/kbn-object-versioning @elastic/appex-sharedux x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant diff --git a/package.json b/package.json index bd7d80afa48168..c29aaac5261e29 100644 --- a/package.json +++ b/package.json @@ -440,6 +440,7 @@ "@kbn/embedded-lens-example-plugin": "link:x-pack/examples/embedded_lens_example", "@kbn/encrypted-saved-objects-plugin": "link:x-pack/plugins/encrypted_saved_objects", "@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search", + "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema", "@kbn/error-boundary-example-plugin": "link:examples/error_boundary", "@kbn/es-errors": "link:packages/kbn-es-errors", "@kbn/es-query": "link:packages/kbn-es-query", @@ -621,7 +622,6 @@ "@kbn/newsfeed-test-plugin": "link:test/common/plugins/newsfeed", "@kbn/no-data-page-plugin": "link:src/plugins/no_data_page", "@kbn/notifications-plugin": "link:x-pack/plugins/notifications", - "@kbn/oam-schema": "link:x-pack/packages/kbn-oam-schema", "@kbn/object-versioning": "link:packages/kbn-object-versioning", "@kbn/observability-ai-assistant-app-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_app", "@kbn/observability-ai-assistant-management-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_management", diff --git a/tsconfig.base.json b/tsconfig.base.json index 7d79f30f998bd8..3559549c40f618 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -774,6 +774,8 @@ "@kbn/encrypted-saved-objects-plugin/*": ["x-pack/plugins/encrypted_saved_objects/*"], "@kbn/enterprise-search-plugin": ["x-pack/plugins/enterprise_search"], "@kbn/enterprise-search-plugin/*": ["x-pack/plugins/enterprise_search/*"], + "@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"], + "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"], "@kbn/error-boundary-example-plugin": ["examples/error_boundary"], "@kbn/error-boundary-example-plugin/*": ["examples/error_boundary/*"], "@kbn/es": ["packages/kbn-es"], @@ -1194,8 +1196,6 @@ "@kbn/no-data-page-plugin/*": ["src/plugins/no_data_page/*"], "@kbn/notifications-plugin": ["x-pack/plugins/notifications"], "@kbn/notifications-plugin/*": ["x-pack/plugins/notifications/*"], - "@kbn/oam-schema": ["x-pack/packages/kbn-oam-schema"], - "@kbn/oam-schema/*": ["x-pack/packages/kbn-oam-schema/*"], "@kbn/object-versioning": ["packages/kbn-object-versioning"], "@kbn/object-versioning/*": ["packages/kbn-object-versioning/*"], "@kbn/observability-ai-assistant-app-plugin": ["x-pack/plugins/observability_solution/observability_ai_assistant_app"], diff --git a/x-pack/packages/kbn-entities-schema/README.md b/x-pack/packages/kbn-entities-schema/README.md new file mode 100644 index 00000000000000..2601be6543c584 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/README.md @@ -0,0 +1,3 @@ +# @kbn/entities-schema + +The entities schema for the asset model for Observability \ No newline at end of file diff --git a/x-pack/packages/kbn-oam-schema/index.ts b/x-pack/packages/kbn-entities-schema/index.ts similarity index 100% rename from x-pack/packages/kbn-oam-schema/index.ts rename to x-pack/packages/kbn-entities-schema/index.ts diff --git a/x-pack/packages/kbn-oam-schema/jest.config.js b/x-pack/packages/kbn-entities-schema/jest.config.js similarity index 84% rename from x-pack/packages/kbn-oam-schema/jest.config.js rename to x-pack/packages/kbn-entities-schema/jest.config.js index 3c40ad7480e306..1d10119431ec35 100644 --- a/x-pack/packages/kbn-oam-schema/jest.config.js +++ b/x-pack/packages/kbn-entities-schema/jest.config.js @@ -8,5 +8,5 @@ module.exports = { preset: '@kbn/test', rootDir: '../../..', - roots: ['/x-pack/packages/kbn-oam-schema'], + roots: ['/x-pack/packages/kbn-entities-schema'], }; diff --git a/x-pack/packages/kbn-oam-schema/kibana.jsonc b/x-pack/packages/kbn-entities-schema/kibana.jsonc similarity index 69% rename from x-pack/packages/kbn-oam-schema/kibana.jsonc rename to x-pack/packages/kbn-entities-schema/kibana.jsonc index 40b6be5c530198..9895c2074a5847 100644 --- a/x-pack/packages/kbn-oam-schema/kibana.jsonc +++ b/x-pack/packages/kbn-entities-schema/kibana.jsonc @@ -1,5 +1,5 @@ { "type": "shared-common", - "id": "@kbn/oam-schema", + "id": "@kbn/entities-schema", "owner": "@elastic/obs-knowledge-team" } diff --git a/x-pack/packages/kbn-oam-schema/package.json b/x-pack/packages/kbn-entities-schema/package.json similarity index 69% rename from x-pack/packages/kbn-oam-schema/package.json rename to x-pack/packages/kbn-entities-schema/package.json index 7cf91a9d0fc6e0..0be44d9d750556 100644 --- a/x-pack/packages/kbn-oam-schema/package.json +++ b/x-pack/packages/kbn-entities-schema/package.json @@ -1,5 +1,5 @@ { - "name": "@kbn/oam-schema", + "name": "@kbn/entities-schema", "private": true, "version": "1.0.0", "license": "Elastic License 2.0" diff --git a/x-pack/packages/kbn-oam-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts similarity index 100% rename from x-pack/packages/kbn-oam-schema/src/schema/common.ts rename to x-pack/packages/kbn-entities-schema/src/schema/common.ts diff --git a/x-pack/packages/kbn-oam-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts similarity index 100% rename from x-pack/packages/kbn-oam-schema/src/schema/entity.ts rename to x-pack/packages/kbn-entities-schema/src/schema/entity.ts diff --git a/x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts similarity index 100% rename from x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts rename to x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts diff --git a/x-pack/packages/kbn-oam-schema/tsconfig.json b/x-pack/packages/kbn-entities-schema/tsconfig.json similarity index 100% rename from x-pack/packages/kbn-oam-schema/tsconfig.json rename to x-pack/packages/kbn-entities-schema/tsconfig.json diff --git a/x-pack/packages/kbn-oam-schema/README.md b/x-pack/packages/kbn-oam-schema/README.md deleted file mode 100644 index c0e1da11cf9a8d..00000000000000 --- a/x-pack/packages/kbn-oam-schema/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @kbn/oam-schema - -Empty package generated by @kbn/generate diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts index 73253775afaed4..e4d7116b95562b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateProcessors } from './ingest_pipeline/generate_processors'; import { retryTransientEsErrors } from './helpers/retry'; import { EntitySecurityException } from './errors/entity_security_exception'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts index 7943ae306333c6..f8cd02250d8983 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateTransform } from './transform/generate_transform'; import { retryTransientEsErrors } from './helpers/retry'; import { EntitySecurityException } from './errors/entity_security_exception'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts index b2b28a8f31cbac..1067f33abaca33 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts @@ -6,7 +6,7 @@ */ import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts index fb78281b7fed5e..7df6867115bbe8 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateIndexName } from './helpers/generate_index_name'; export async function deleteIndex( diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts index a56527dfc1bdc2..1e42282369ef32 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; import { retryTransientEsErrors } from './helpers/retry'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts index 9a15431392bf42..5108ca31ed5487 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; export class EntityIdConflict extends Error { public defintion: EntityDefinition; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts index bfc2fab7cb083b..6e2f721d4de141 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; export class EntitySecurityException extends Error { public defintion: EntityDefinition; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts index f1ca045de874f9..fdb808466ba834 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { entityDefinitionSchema } from '@kbn/oam-schema'; +import { entityDefinitionSchema } from '@kbn/entities-schema'; export const entityDefinition = entityDefinitionSchema.parse({ id: 'admin-console-logs-service', name: 'Services for Admin Console', diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts index 677c320c7f4cc5..365104f3571ebf 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; export function generateIndexName(definition: EntityDefinition) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts index c89404fb54972c..c772e198e64fd7 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; export function generateIngestPipelineId(definition: EntityDefinition) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts index 810762716a8f0b..29478d09fc669b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateIndexName } from '../helpers/generate_index_name'; function createIdTemplate(definition: EntityDefinition) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts index 5a4132e716eec3..e6817ab63f2afb 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts @@ -6,7 +6,7 @@ */ import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import { EntityDefinition, entityDefinitionSchema } from '@kbn/oam-schema'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts index 855cab620de2c0..f79f88c50ac584 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts @@ -6,7 +6,7 @@ */ import { SavedObjectsClientContract } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts index 5f7866d679f93c..766bbb10b1d675 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { retryTransientEsErrors } from './helpers/retry'; import { generateTransformId } from './transform/generate_transform_id'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts index 7c0b89dd6aaa25..60a250a33f0d9e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { generateTransformId } from './transform/generate_transform_id'; import { retryTransientEsErrors } from './helpers/retry'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index cf3ff3aee07dd8..8c11aea1385198 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; export function generateMetadataAggregations(definition: EntityDefinition) { if (!definition.metadata) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts index 8e7a5d456844a0..9527671768e357 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { KeyMetric, Metric, EntityDefinition } from '@kbn/oam-schema'; +import { KeyMetric, Metric, EntityDefinition } from '@kbn/entities-schema'; import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; import { InvalidTransformError } from '../errors/invalid_transform_error'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts index bcc6d1ad5900b5..e4956de2ae937c 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { QueryDslQueryContainer, TransformPutTransformRequest, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts index 7caceb6a8b9cc0..06faedb916774d 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; import { ENTITY_TRANSFORM_PREFIX } from '../../../../common/constants_entities'; export function generateTransformId(definition: EntityDefinition) { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts index 8dabdb806b6223..d403c39ae0ed12 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts @@ -6,7 +6,7 @@ */ import { RequestHandlerContext } from '@kbn/core/server'; -import { EntityDefinition, entityDefinitionSchema } from '@kbn/oam-schema'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; import { stringifyZodError } from '@kbn/zod-helpers'; import { SetupRouteOptions } from '../types'; import { saveEntityDefinition } from '../../lib/entities/save_entity_definition'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts index 10889d70cdccfa..509caa947a445f 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts @@ -6,7 +6,7 @@ */ import { SavedObject, SavedObjectsType } from '@kbn/core/server'; -import { EntityDefinition } from '@kbn/oam-schema'; +import { EntityDefinition } from '@kbn/entities-schema'; export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition'; diff --git a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json index 3a20f5cb2a7f4a..dbbc36252bfb19 100644 --- a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json +++ b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json @@ -27,7 +27,7 @@ "@kbn/core-elasticsearch-server", "@kbn/core-saved-objects-api-server", "@kbn/core-saved-objects-api-server-mocks", - "@kbn/oam-schema", + "@kbn/entities-schema", "@kbn/es-query", "@kbn/zod-helpers" ] diff --git a/yarn.lock b/yarn.lock index 5d2c02d1f53c9e..72f7d23ac822fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4598,6 +4598,10 @@ version "0.0.0" uid "" +"@kbn/entities-schema@link:x-pack/packages/kbn-entities-schema": + version "0.0.0" + uid "" + "@kbn/error-boundary-example-plugin@link:examples/error_boundary": version "0.0.0" uid "" @@ -5438,10 +5442,6 @@ version "0.0.0" uid "" -"@kbn/oam-schema@link:x-pack/packages/kbn-oam-schema": - version "0.0.0" - uid "" - "@kbn/object-versioning@link:packages/kbn-object-versioning": version "0.0.0" uid "" From 8fb56518cbf9e4741010aa10cba341b8e0982962 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 16 May 2024 15:08:06 -0600 Subject: [PATCH 14/16] Implimenting optional identity fields --- x-pack/packages/kbn-entities-schema/src/schema/common.ts | 7 +++++++ .../kbn-entities-schema/src/schema/entity_definition.ts | 3 ++- .../lib/entities/ingest_pipeline/generate_processors.ts | 4 ++-- .../__snapshots__/generate_transform.test.ts.snap | 1 + .../server/lib/entities/transform/generate_transform.ts | 6 ++++-- .../server/saved_objects/entity_definition.ts | 2 +- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index 3597adcca56871..6b4f0cc794c2be 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -97,3 +97,10 @@ export const metadataSchema = z limit: z.optional(z.number().default(1000)), }) .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); + +export const identityFieldsSchema = z + .object({ + field: z.string(), + optional: z.boolean(), + }) + .or(z.string().transform((value) => ({ field: value, optional: false }))); diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index 8f1daac6900042..48b8c8060efbc7 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -13,6 +13,7 @@ import { metadataSchema, filterSchema, durationSchema, + identityFieldsSchema, } from './common'; export const entityDefinitionSchema = z.object({ @@ -22,7 +23,7 @@ export const entityDefinitionSchema = z.object({ type: entityTypeSchema, filter: filterSchema, indexPatterns: arrayOfStringsSchema, - identityFields: arrayOfStringsSchema, + identityFields: z.array(identityFieldsSchema), identityTemplate: z.string(), metadata: z.optional(z.array(metadataSchema)), metrics: z.optional(z.array(keyMetricSchema)), diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts index 29478d09fc669b..33f27cc5daf711 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts @@ -9,8 +9,8 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { generateIndexName } from '../helpers/generate_index_name'; function createIdTemplate(definition: EntityDefinition) { - return definition.identityFields.reduce((template, field) => { - return template.replaceAll(field, `entity.identity.${field}`); + return definition.identityFields.reduce((template, id) => { + return template.replaceAll(id.field, `entity.identity.${id.field}`); }, definition.identityTemplate); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap index f4707ca71dbdba..e692f2068eafd6 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap @@ -88,6 +88,7 @@ Object { "entity.identity.log.logger": Object { "terms": Object { "field": "log.logger", + "missing_bucket": false, }, }, }, diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts index e4956de2ae937c..6a8c0bd6377150 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts @@ -64,9 +64,11 @@ export function generateTransform(definition: EntityDefinition): TransformPutTra }, pivot: { group_by: definition.identityFields.reduce( - (acc, field) => ({ + (acc, id) => ({ ...acc, - [`entity.identity.${field}`]: { terms: { field } }, + [`entity.identity.${id.field}`]: { + terms: { field: id.field, missing_bucket: id.optional }, + }, }), {} ), diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts index 509caa947a445f..5a63444974ad79 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts @@ -23,7 +23,7 @@ export const entityDefinition: SavedObjectsType = { type: { type: 'keyword' }, filter: { type: 'keyword' }, indexPatterns: { type: 'keyword' }, - identityFields: { type: 'keyword' }, + identityFields: { type: 'object' }, categories: { type: 'keyword' }, metadata: { type: 'object' }, metrics: { type: 'object' }, From 421d5caef650a17c5976f9beb61573e6e905042f Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 16 May 2024 16:09:01 -0600 Subject: [PATCH 15/16] Fixing type check for index templates --- .../asset_manager/server/templates/entities_template.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts index 26ec5cc0661f81..d3934b24ea82cc 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts @@ -36,8 +36,10 @@ export const entitiesIndexTemplateConfig: IndicesPutIndexTemplateRequest = { { entity_metrics: { mapping: { + // @ts-expect-error this should work per: https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html#match-mapping-type type: '{dynamic_type}', }, + // @ts-expect-error this should work per: https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html#match-mapping-type match_mapping_type: ['long', 'double'], path_match: 'entity.metric.*', }, From 4a53db85b8b97fcaeb025b42178fc372018fc17f Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 16 May 2024 22:18:02 +0000 Subject: [PATCH 16/16] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4b7d2f4fd14ebd..2769e9531de00e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -390,6 +390,7 @@ src/plugins/embeddable @elastic/kibana-presentation x-pack/examples/embedded_lens_example @elastic/kibana-visualizations x-pack/plugins/encrypted_saved_objects @elastic/kibana-security x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend +x-pack/packages/kbn-entities-schema @elastic/obs-knowledge-team examples/error_boundary @elastic/appex-sharedux packages/kbn-es @elastic/kibana-operations packages/kbn-es-archiver @elastic/kibana-operations @elastic/appex-qa @@ -600,7 +601,6 @@ src/plugins/newsfeed @elastic/kibana-core test/common/plugins/newsfeed @elastic/kibana-core src/plugins/no_data_page @elastic/appex-sharedux x-pack/plugins/notifications @elastic/appex-sharedux -x-pack/packages/kbn-entities-schema @elastic/obs-knowledge-team packages/kbn-object-versioning @elastic/appex-sharedux x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant