From 7ae07f89137cf839869a2233de5f9f26900f9e99 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 17 May 2024 12:03:42 -0600 Subject: [PATCH] [EEM][POC] The POC for creating entity-centric indices using entity definitions (#183205) ## Summary This is a "proof of concept" for generating entity-centric indices for the OAM. This exposes an API (`/api/entities`) for creating "asset definitions" (`EntityDefinition`) that manages a transform and ingest pipeline to produce documents into an index which could be used to create a search experience or lookups for different services. ### Features - Data schema agnostic, works with known schemas OR custom logs - Supports defining multiple `identityFields` along with an `identityTemplate` for formatting the `asset.id` - Supports optional `identityFields` using `{ "field": "path-to-field", "optional": true }` definition instead of a `string`. - Supports defining key `metrics` with equations which are compatible with the SLO product - Supports adding `metadata` fields which will include multiple values. - Supports `metadata` fields can be re-mapped to a new destination path using `{ "source": "path-to-source-field", "limit": 1000, "destination": "path-to-destination-in-output" }` definition instead of a `string` - Supports adding `staticFields` which can also use template variables - Support fine grain control over the frequency and sync settings for the underlying transform - Installs the index template components and index template settings for the destination index - Allow the user to configure the index patterns and timestamp field along with the lookback - The documents for each definition will be stored in their own index (`.entities-observability.summary-v1.{defintion.id}`) ### Notes - We are currently considering adding a historical index which will track changes to the assets over time. If we choose to do this, the summary index would remain the same but we'd add a second transform with a group_by on the `definition.timestampField` and break the indices into monthly indexes (configurable in the settings). - We are looking into ways to add `firstSeenTimestamp`, this is a difficult due to scaling issue. Essentially, we would need to find the `minimum` timestamp for each entity which could be extremely costly on a large datasets. - There is nothing stopping you from creating an asset definition that uses the `.entities-observability.summary-v1.*` index pattern to create summaries of summaries... it can be very "meta". ### API - `POST /api/entities/definition` - Creates a new asset definition and starts the indexing. See examples below. - `DELETE /api/entities/definition/{id}` - Deletes the asset definition along with cleaning up the transform, ingest pipeline, and deletes the destination index. - `POST /api/entities/definition/{id}/_reset` - Resets the transform, ingest pipeline, and destination index. This is useful for upgrading asset definitions to new features. ## Example Definitions and Output Here is a definition for creating services for each of the custom log sources in the `fake_stack` dataset from `x-pack/packages/data-forge`. ```JSON POST kbn:/api/entities/definition { "id": "admin-console-logs-service", "name": "Services for Admin Console", "type": "service", "indexPatterns": ["kbn-data-forge-fake_stack.*"], "timestampField": "@timestamp", "lookback": "5m", "identityFields": ["log.logger"], "identityTemplate": "{{log.logger}}", "metadata": [ "tags", "host.name" ], "metrics": [ { "name": "logRate", "equation": "A / 5", "metrics": [ { "name": "A", "aggregation": "doc_count", "filter": "log.level: *" } ] }, { "name": "errorRate", "equation": "A / 5", "metrics": [ { "name": "A", "aggregation": "doc_count", "filter": "log.level: \"ERROR\"" } ] } ] } ``` Which produces: ```JSON { "host": { "name": [ "admin-console.prod.020", "admin-console.prod.010", "admin-console.prod.011", "admin-console.prod.001", "admin-console.prod.012", "admin-console.prod.002", "admin-console.prod.013", "admin-console.prod.003", "admin-console.prod.014", "admin-console.prod.004", "admin-console.prod.015", "admin-console.prod.016", "admin-console.prod.005", "admin-console.prod.017", "admin-console.prod.006", "admin-console.prod.018", "admin-console.prod.007", "admin-console.prod.019", "admin-console.prod.008", "admin-console.prod.009" ] }, "entity": { "latestTimestamp": "2024-05-10T22:04:51.481Z", "metric": { "logRate": 37.4, "errorRate": 1 }, "identity": { "log": { "logger": "admin-console" } }, "id": "admin-console", "indexPatterns": [ "kbn-data-forge-fake_stack.*" ], "definitionId": "admin-console-logs-service" }, "event": { "ingested": "2024-05-10T22:05:51.955691Z" }, "tags": [ "infra:admin-console" ] } ``` Here is an example of a definition for APM Services: ```JSON POST kbn:/api/entities/definition { "id": "apm-services", "name": "Services for APM", "type": "service", "indexPatterns": ["logs-*", "metrics-*"], "timestampField": "@timestamp", "lookback": "5m", "identityFields": ["service.name", "service.environment"], "identityTemplate": "{{service.name}}:{{service.environment}}", "metadata": [ "tags", "host.name" ], "metrics": [ { "name": "latency", "equation": "A", "metrics": [ { "name": "A", "aggregation": "avg", "field": "transaction.duration.histogram" } ] }, { "name": "throughput", "equation": "A / 5", "metrics": [ { "name": "A", "aggregation": "doc_count" } ] }, { "name": "failedTransRate", "equation": "A / B", "metrics": [ { "name": "A", "aggregation": "doc_count", "filter": "event.outcome: \"failure\"" }, { "name": "B", "aggregation": "doc_count", "filter": "event.outcome: *" } ] } ] } ``` Which produces: ```JSON { "host": { "name": [ "simianhacker's-macbook-pro" ] }, "entity": { "latestTimestamp": "2024-05-10T21:38:22.513Z", "metric": { "latency": 615276.8812785388, "throughput": 50.6, "failedTransRate": 0.0091324200913242 }, "identity": { "service": { "environment": "development", "name": "admin-console" } }, "id": "admin-console:development", "indexPatterns": [ "logs-*", "metrics-*" ], "definitionId": "apm-services" }, "event": { "ingested": "2024-05-10T21:39:33.636225Z" }, "tags": [ "_geoip_database_unavailable_GeoLite2-City.mmdb" ] } ``` ### Getting Started The easiest way to get started is to use the`kbn-data-forge` config below. Save this YAML to `~/Desktop/fake_stack.yaml` then run `node x-pack/scripts/data_forge.js --config ~/Desktop/fake_stack.yaml`. Then create a definition using the first example above. ```YAML --- elasticsearch: installKibanaUser: false kibana: installAssets: true host: "http://localhost:5601/kibana" indexing: dataset: "fake_stack" eventsPerCycle: 50 reduceWeekendTrafficBy: 0.5 schedule: # Start with good events - template: "good" start: "now-1d" end: "now-20m" eventsPerCycle: 50 randomness: 0.8 - template: "bad" start: "now-20m" end: "now-10m" eventsPerCycle: 50 randomness: 0.8 - template: "good" start: "now-10m" end: false eventsPerCycle: 50 randomness: 0.8 ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + package.json | 1 + tsconfig.base.json | 2 + x-pack/packages/kbn-entities-schema/README.md | 3 + x-pack/packages/kbn-entities-schema/index.ts | 10 ++ .../kbn-entities-schema/jest.config.js | 12 ++ .../packages/kbn-entities-schema/kibana.jsonc | 5 + .../packages/kbn-entities-schema/package.json | 6 + .../kbn-entities-schema/src/schema/common.ts | 106 +++++++++++++++ .../kbn-entities-schema/src/schema/entity.ts | 21 +++ .../src/schema/entity_definition.ts | 42 ++++++ .../kbn-entities-schema/tsconfig.json | 18 +++ .../common/constants_entities.ts | 13 ++ .../create_and_install_ingest_pipeline.ts | 39 ++++++ .../entities/create_and_install_transform.ts | 30 +++++ .../lib/entities/delete_entity_definition.ts | 31 +++++ .../server/lib/entities/delete_index.ts | 24 ++++ .../lib/entities/delete_ingest_pipeline.ts | 27 ++++ .../errors/entity_id_conflict_error.ts | 18 +++ .../lib/entities/errors/entity_not_found.ts | 13 ++ .../errors/entity_security_exception.ts | 18 +++ .../errors/invalid_transform_error.ts | 13 ++ .../helpers/fixtures/entity_definition.ts | 46 +++++++ .../entities/helpers/generate_index_name.ts | 13 ++ .../get_elasticsearch_query_or_throw.ts | 17 +++ .../server/lib/entities/helpers/retry.ts | 53 ++++++++ .../generate_processors.test.ts.snap | 70 ++++++++++ .../generate_ingest_pipeline_id.ts | 13 ++ .../generate_processors.test.ts | 16 +++ .../ingest_pipeline/generate_processors.ts | 98 ++++++++++++++ .../lib/entities/read_entity_definition.ts | 36 +++++ .../lib/entities/save_entity_definition.ts | 37 +++++ .../server/lib/entities/start_transform.ts | 28 ++++ .../lib/entities/stop_and_delete_transform.ts | 37 +++++ .../generate_transform.test.ts.snap | 126 ++++++++++++++++++ .../generate_metadata_aggregations.ts | 26 ++++ .../transform/generate_metric_aggregations.ts | 118 ++++++++++++++++ .../transform/generate_transform.test.ts | 16 +++ .../entities/transform/generate_transform.ts | 86 ++++++++++++ .../transform/generate_transform_id.ts | 13 ++ .../server/lib/manage_index_templates.ts | 33 +++-- .../asset_manager/server/plugin.ts | 37 ++++- .../server/routes/entities/create.ts | 78 +++++++++++ .../server/routes/entities/delete.ts | 55 ++++++++ .../server/routes/entities/reset.ts | 65 +++++++++ .../asset_manager/server/routes/index.ts | 21 ++- .../asset_manager/server/routes/types.ts | 2 + .../server/saved_objects/entity_definition.ts | 40 ++++++ .../server/saved_objects/index.ts | 8 ++ .../server/templates/assets_template.ts | 2 + .../server/templates/components/base.ts | 32 +++++ .../server/templates/components/entity.ts | 40 ++++++ .../server/templates/components/event.ts | 29 ++++ .../server/templates/entities_template.ts | 60 +++++++++ .../asset_manager/tsconfig.json | 5 +- yarn.lock | 4 + 56 files changed, 1790 insertions(+), 23 deletions(-) create mode 100644 x-pack/packages/kbn-entities-schema/README.md create mode 100644 x-pack/packages/kbn-entities-schema/index.ts create mode 100644 x-pack/packages/kbn-entities-schema/jest.config.js create mode 100644 x-pack/packages/kbn-entities-schema/kibana.jsonc create mode 100644 x-pack/packages/kbn-entities-schema/package.json create mode 100644 x-pack/packages/kbn-entities-schema/src/schema/common.ts create mode 100644 x-pack/packages/kbn-entities-schema/src/schema/entity.ts create mode 100644 x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts create mode 100644 x-pack/packages/kbn-entities-schema/tsconfig.json create mode 100644 x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts create mode 100644 x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cddb6388566ee7..d160dda238f8d6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -390,6 +390,7 @@ src/plugins/embeddable @elastic/kibana-presentation x-pack/examples/embedded_lens_example @elastic/kibana-visualizations x-pack/plugins/encrypted_saved_objects @elastic/kibana-security x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend +x-pack/packages/kbn-entities-schema @elastic/obs-knowledge-team examples/error_boundary @elastic/appex-sharedux packages/kbn-es @elastic/kibana-operations packages/kbn-es-archiver @elastic/kibana-operations @elastic/appex-qa diff --git a/package.json b/package.json index 78dba79f5e6ba6..04316640a2952b 100644 --- a/package.json +++ b/package.json @@ -439,6 +439,7 @@ "@kbn/embedded-lens-example-plugin": "link:x-pack/examples/embedded_lens_example", "@kbn/encrypted-saved-objects-plugin": "link:x-pack/plugins/encrypted_saved_objects", "@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search", + "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema", "@kbn/error-boundary-example-plugin": "link:examples/error_boundary", "@kbn/es-errors": "link:packages/kbn-es-errors", "@kbn/es-query": "link:packages/kbn-es-query", diff --git a/tsconfig.base.json b/tsconfig.base.json index f87d111ac27aff..dd82d55b1b0ad6 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -774,6 +774,8 @@ "@kbn/encrypted-saved-objects-plugin/*": ["x-pack/plugins/encrypted_saved_objects/*"], "@kbn/enterprise-search-plugin": ["x-pack/plugins/enterprise_search"], "@kbn/enterprise-search-plugin/*": ["x-pack/plugins/enterprise_search/*"], + "@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"], + "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"], "@kbn/error-boundary-example-plugin": ["examples/error_boundary"], "@kbn/error-boundary-example-plugin/*": ["examples/error_boundary/*"], "@kbn/es": ["packages/kbn-es"], diff --git a/x-pack/packages/kbn-entities-schema/README.md b/x-pack/packages/kbn-entities-schema/README.md new file mode 100644 index 00000000000000..2601be6543c584 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/README.md @@ -0,0 +1,3 @@ +# @kbn/entities-schema + +The entities schema for the asset model for Observability \ No newline at end of file diff --git a/x-pack/packages/kbn-entities-schema/index.ts b/x-pack/packages/kbn-entities-schema/index.ts new file mode 100644 index 00000000000000..92b93b79381251 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './src/schema/entity_definition'; +export * from './src/schema/entity'; +export * from './src/schema/common'; diff --git a/x-pack/packages/kbn-entities-schema/jest.config.js b/x-pack/packages/kbn-entities-schema/jest.config.js new file mode 100644 index 00000000000000..1d10119431ec35 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/packages/kbn-entities-schema'], +}; diff --git a/x-pack/packages/kbn-entities-schema/kibana.jsonc b/x-pack/packages/kbn-entities-schema/kibana.jsonc new file mode 100644 index 00000000000000..9895c2074a5847 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/entities-schema", + "owner": "@elastic/obs-knowledge-team" +} diff --git a/x-pack/packages/kbn-entities-schema/package.json b/x-pack/packages/kbn-entities-schema/package.json new file mode 100644 index 00000000000000..0be44d9d750556 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/entities-schema", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts new file mode 100644 index 00000000000000..6b4f0cc794c2be --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import moment from 'moment'; + +export enum EntityType { + service = 'service', + host = 'host', + pod = 'pod', + node = 'node', +} + +export const arrayOfStringsSchema = z.array(z.string()); + +export const entityTypeSchema = z.nativeEnum(EntityType); + +export enum BasicAggregations { + avg = 'avg', + max = 'max', + min = 'min', + sum = 'sum', + cardinality = 'cardinality', + last_value = 'last_value', + std_deviation = 'std_deviation', +} + +export const basicAggregationsSchema = z.nativeEnum(BasicAggregations); + +const metricNameSchema = z + .string() + .length(1) + .regex(/[a-zA-Z]/) + .toUpperCase(); + +export const filterSchema = z.optional(z.string()); + +export const basicMetricWithFieldSchema = z.object({ + name: metricNameSchema, + aggregation: basicAggregationsSchema, + field: z.string(), + filter: filterSchema, +}); + +export const docCountMetricSchema = z.object({ + name: metricNameSchema, + aggregation: z.literal('doc_count'), + filter: filterSchema, +}); + +export const durationSchema = z + .string() + .regex(/\d+[m|d|s|h]/) + .transform((val: string) => { + const parts = val.match(/(\d+)([m|s|h|d])/); + if (parts === null) { + throw new Error('Unable to parse duration'); + } + const value = parseInt(parts[1], 10); + const unit = parts[2] as 'm' | 's' | 'h' | 'd'; + const duration = moment.duration(value, unit); + return { ...duration, toJSON: () => val }; + }); + +export const percentileMetricSchema = z.object({ + name: metricNameSchema, + aggregation: z.literal('percentile'), + field: z.string(), + percentile: z.number(), + filter: filterSchema, +}); + +export const metricSchema = z.discriminatedUnion('aggregation', [ + basicMetricWithFieldSchema, + docCountMetricSchema, + percentileMetricSchema, +]); + +export type Metric = z.infer; + +export const keyMetricSchema = z.object({ + name: z.string(), + metrics: z.array(metricSchema), + equation: z.string(), +}); + +export type KeyMetric = z.infer; + +export const metadataSchema = z + .object({ + source: z.string(), + destination: z.optional(z.string()), + limit: z.optional(z.number().default(1000)), + }) + .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); + +export const identityFieldsSchema = z + .object({ + field: z.string(), + optional: z.boolean(), + }) + .or(z.string().transform((value) => ({ field: value, optional: false }))); diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts new file mode 100644 index 00000000000000..514ed014940362 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { arrayOfStringsSchema } from './common'; + +export const entitySchema = z.intersection( + z.object({ + entity: z.object({ + id: z.string(), + indexPatterns: arrayOfStringsSchema, + identityFields: arrayOfStringsSchema, + metric: z.record(z.string(), z.number()), + }), + }), + z.record(z.string(), z.string().or(z.number())) +); diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts new file mode 100644 index 00000000000000..48b8c8060efbc7 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from 'zod'; +import { + arrayOfStringsSchema, + entityTypeSchema, + keyMetricSchema, + metadataSchema, + filterSchema, + durationSchema, + identityFieldsSchema, +} from './common'; + +export const entityDefinitionSchema = z.object({ + id: z.string().regex(/^[\w-]+$/), + name: z.string(), + description: z.optional(z.string()), + type: entityTypeSchema, + filter: filterSchema, + indexPatterns: arrayOfStringsSchema, + identityFields: z.array(identityFieldsSchema), + identityTemplate: z.string(), + metadata: z.optional(z.array(metadataSchema)), + metrics: z.optional(z.array(keyMetricSchema)), + staticFields: z.optional(z.record(z.string(), z.string())), + lookback: durationSchema, + timestampField: z.string(), + settings: z.optional( + z.object({ + syncField: z.optional(z.string()), + syncDelay: z.optional(z.string()), + frequency: z.optional(z.string()), + }) + ), +}); + +export type EntityDefinition = z.infer; diff --git a/x-pack/packages/kbn-entities-schema/tsconfig.json b/x-pack/packages/kbn-entities-schema/tsconfig.json new file mode 100644 index 00000000000000..f722f3587e7a29 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + ] +} diff --git a/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts new file mode 100644 index 00000000000000..aeeea398220a08 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ENTITY_VERSION = 'v1'; +export const ENTITY_BASE_PREFIX = `.entities-observability.summary-${ENTITY_VERSION}`; +export const ENTITY_TRANSFORM_PREFIX = `entities-observability-summary-${ENTITY_VERSION}`; +export const ENTITY_DEFAULT_FREQUENCY = '1m'; +export const ENTITY_DEFAULT_SYNC_DELAY = '60s'; +export const ENTITY_API_PREFIX = '/api/entities'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts new file mode 100644 index 00000000000000..e4d7116b95562b --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateProcessors } from './ingest_pipeline/generate_processors'; +import { retryTransientEsErrors } from './helpers/retry'; +import { EntitySecurityException } from './errors/entity_security_exception'; +import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; + +export async function createAndInstallIngestPipeline( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const processors = generateProcessors(definition); + const id = generateIngestPipelineId(definition); + try { + await retryTransientEsErrors( + () => + esClient.ingest.putPipeline({ + id, + processors, + }), + { logger } + ); + } catch (e) { + logger.error(`Cannot create entity ingest pipeline for [${definition.id}] entity defintion`); + if (e.meta?.body?.error?.type === 'security_exception') { + throw new EntitySecurityException(e.meta.body.error.reason, definition); + } + throw e; + } + return id; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts new file mode 100644 index 00000000000000..f8cd02250d8983 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateTransform } from './transform/generate_transform'; +import { retryTransientEsErrors } from './helpers/retry'; +import { EntitySecurityException } from './errors/entity_security_exception'; + +export async function createAndInstallTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const transform = generateTransform(definition); + try { + await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger }); + } catch (e) { + logger.error(`Cannot create entity transform for [${definition.id}] entity definition`); + if (e.meta?.body?.error?.type === 'security_exception') { + throw new EntitySecurityException(e.meta.body.error.reason, definition); + } + throw e; + } + return transform.transform_id; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts new file mode 100644 index 00000000000000..1067f33abaca33 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityDefinitionNotFound } from './errors/entity_not_found'; + +export async function deleteEntityDefinition( + soClient: SavedObjectsClientContract, + definition: EntityDefinition, + logger: Logger +) { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 0) { + logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`); + throw new EntityDefinitionNotFound(`Entity defintion with [${definition.id}] not found.`); + } + + await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts new file mode 100644 index 00000000000000..7df6867115bbe8 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateIndexName } from './helpers/generate_index_name'; + +export async function deleteIndex( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const indexName = generateIndexName(definition); + try { + await esClient.indices.delete({ index: indexName, ignore_unavailable: true }); + } catch (e) { + logger.error(`Unable to remove entity defintion index [${definition.id}}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts new file mode 100644 index 00000000000000..1e42282369ef32 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id'; +import { retryTransientEsErrors } from './helpers/retry'; + +export async function deleteIngestPipeline( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const pipelineId = generateIngestPipelineId(definition); + try { + await retryTransientEsErrors(() => + esClient.ingest.deletePipeline({ id: pipelineId }, { ignore: [404] }) + ); + } catch (e) { + logger.error(`Unable to delete ingest pipeline [${pipelineId}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts new file mode 100644 index 00000000000000..5108ca31ed5487 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; + +export class EntityIdConflict extends Error { + public defintion: EntityDefinition; + + constructor(message: string, def: EntityDefinition) { + super(message); + this.name = 'EntityIdConflict'; + this.defintion = def; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts new file mode 100644 index 00000000000000..d81cd3322cae6e --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class EntityDefinitionNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'EntityDefinitionNotFound'; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts new file mode 100644 index 00000000000000..6e2f721d4de141 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; + +export class EntitySecurityException extends Error { + public defintion: EntityDefinition; + + constructor(message: string, def: EntityDefinition) { + super(message); + this.name = 'EntitySecurityException'; + this.defintion = def; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts new file mode 100644 index 00000000000000..5d1c98d5dc3ae7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/invalid_transform_error.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class InvalidTransformError extends Error { + constructor(message: string) { + super(message); + this.name = 'InvalidTransformError'; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts new file mode 100644 index 00000000000000..fdb808466ba834 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { entityDefinitionSchema } from '@kbn/entities-schema'; +export const entityDefinition = entityDefinitionSchema.parse({ + id: 'admin-console-logs-service', + name: 'Services for Admin Console', + type: 'service', + indexPatterns: ['kbn-data-forge-fake_stack.*'], + timestampField: '@timestamp', + identityFields: ['log.logger'], + identityTemplate: 'service:{{log.logger}}', + metadata: ['tags', 'host.name', 'kubernetes.pod.name'], + staticFields: { + projectId: '1234', + }, + lookback: '5m', + metrics: [ + { + name: 'logRate', + equation: 'A / 5', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: *', + }, + ], + }, + { + name: 'errorRate', + equation: 'A / 5', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: error', + }, + ], + }, + ], +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts new file mode 100644 index 00000000000000..365104f3571ebf --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; + +export function generateIndexName(definition: EntityDefinition) { + return `${ENTITY_BASE_PREFIX}.${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts new file mode 100644 index 00000000000000..3b24344efd83b9 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/get_elasticsearch_query_or_throw.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; +import { InvalidTransformError } from '../errors/invalid_transform_error'; + +export function getElasticsearchQueryOrThrow(kuery: string) { + try { + return toElasticsearchQuery(fromKueryExpression(kuery)); + } catch (err) { + throw new InvalidTransformError(`Invalid KQL: ${kuery}`); + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts new file mode 100644 index 00000000000000..421289d1c04791 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/retry.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setTimeout } from 'timers/promises'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import type { Logger } from '@kbn/logging'; + +const MAX_ATTEMPTS = 5; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: any) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +/** + * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. + * Should only be used to wrap operations that are idempotent and can be safely executed more than once. + */ +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger?: Logger; attempt?: number } = {} +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... + + logger?.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + await setTimeout(retryDelaySec * 1000); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap new file mode 100644 index 00000000000000..443063d70db60d --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_processors.test.ts.snap @@ -0,0 +1,70 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateProcessors(definition) should genearte a valid pipeline 1`] = ` +Array [ + Object { + "set": Object { + "field": "event.ingested", + "value": "{{{_ingest.timestamp}}}", + }, + }, + Object { + "set": Object { + "field": "entity.definitionId", + "value": "admin-console-logs-service", + }, + }, + Object { + "set": Object { + "field": "entity.indexPatterns", + "value": "[\\"kbn-data-forge-fake_stack.*\\"]", + }, + }, + Object { + "json": Object { + "field": "entity.indexPatterns", + }, + }, + Object { + "set": Object { + "field": "entity.id", + "value": "service:{{entity.identity.log.logger}}", + }, + }, + Object { + "set": Object { + "field": "projectId", + "value": "1234", + }, + }, + Object { + "script": Object { + "source": "if (ctx.entity?.metadata?.tags != null) { + ctx[\\"tags\\"] = ctx.entity.metadata.tags.keySet(); +} +if (ctx.entity?.metadata?.host?.name != null) { + ctx[\\"host\\"] = new HashMap(); + ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.keySet(); +} +if (ctx.entity?.metadata?.kubernetes?.pod?.name != null) { + ctx[\\"kubernetes\\"] = new HashMap(); + ctx[\\"kubernetes\\"][\\"pod\\"] = new HashMap(); + ctx[\\"kubernetes\\"][\\"pod\\"][\\"name\\"] = ctx.entity.metadata.kubernetes.pod.name.keySet(); +} +", + }, + }, + Object { + "remove": Object { + "field": "entity.metadata", + "ignore_missing": true, + }, + }, + Object { + "set": Object { + "field": "_index", + "value": ".entities-observability.summary-v1.admin-console-logs-service", + }, + }, +] +`; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts new file mode 100644 index 00000000000000..c772e198e64fd7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_ingest_pipeline_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities'; + +export function generateIngestPipelineId(definition: EntityDefinition) { + return `${ENTITY_BASE_PREFIX}.${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts new file mode 100644 index 00000000000000..33919ec678dcf1 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { generateProcessors } from './generate_processors'; +import { entityDefinition } from '../helpers/fixtures/entity_definition'; + +describe('generateProcessors(definition)', () => { + it('should genearte a valid pipeline', () => { + const processors = generateProcessors(entityDefinition); + expect(processors).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts new file mode 100644 index 00000000000000..33f27cc5daf711 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/ingest_pipeline/generate_processors.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateIndexName } from '../helpers/generate_index_name'; + +function createIdTemplate(definition: EntityDefinition) { + return definition.identityFields.reduce((template, id) => { + return template.replaceAll(id.field, `entity.identity.${id.field}`); + }, definition.identityTemplate); +} + +function mapDesitnationToPainless(destination: string, source: string) { + const fieldParts = destination.split('.'); + return fieldParts.reduce((acc, _part, currentIndex, parts) => { + if (currentIndex + 1 === parts.length) { + return `${acc}\n ctx${parts + .map((s) => `["${s}"]`) + .join('')} = ctx.entity.metadata.${source}.keySet();`; + } + return `${acc}\n ctx${parts + .slice(0, currentIndex + 1) + .map((s) => `["${s}"]`) + .join('')} = new HashMap();`; + }, ''); +} + +function createMetadataPainlessScript(definition: EntityDefinition) { + if (!definition.metadata) { + return ''; + } + return definition.metadata.reduce((script, def) => { + const source = def.source; + const destination = def.destination || def.source; + return `${script}if (ctx.entity?.metadata?.${source.replaceAll( + '.', + '?.' + )} != null) {${mapDesitnationToPainless(destination, source)}\n}\n`; + }, ''); +} + +export function generateProcessors(definition: EntityDefinition) { + return [ + { + set: { + field: 'event.ingested', + value: '{{{_ingest.timestamp}}}', + }, + }, + { + set: { + field: 'entity.definitionId', + value: definition.id, + }, + }, + { + set: { + field: 'entity.indexPatterns', + value: JSON.stringify(definition.indexPatterns), + }, + }, + { + json: { + field: 'entity.indexPatterns', + }, + }, + { + set: { + field: 'entity.id', + value: createIdTemplate(definition), + }, + }, + ...(definition.staticFields != null + ? Object.keys(definition.staticFields).map((field) => ({ + set: { field, value: definition.staticFields![field] }, + })) + : []), + ...(definition.metadata != null + ? [{ script: { source: createMetadataPainlessScript(definition) } }] + : []), + { + remove: { + field: 'entity.metadata', + ignore_missing: true, + }, + }, + { + set: { + field: '_index', + value: generateIndexName(definition), + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts new file mode 100644 index 00000000000000..e6817ab63f2afb --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/read_entity_definition.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityDefinitionNotFound } from './errors/entity_not_found'; + +export async function readEntityDefinition( + soClient: SavedObjectsClientContract, + id: string, + logger: Logger +) { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${id})`, + }); + if (response.total === 0) { + const message = `Unable to find entity defintion with [${id}]`; + logger.error(message); + throw new EntityDefinitionNotFound(message); + } + + try { + return entityDefinitionSchema.parse(response.saved_objects[0].attributes); + } catch (e) { + logger.error(`Unable to parse entity defintion with [${id}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts new file mode 100644 index 00000000000000..f79f88c50ac584 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/save_entity_definition.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; +import { EntityIdConflict } from './errors/entity_id_conflict_error'; + +export async function saveEntityDefinition( + soClient: SavedObjectsClientContract, + definition: EntityDefinition +): Promise { + const response = await soClient.find({ + type: SO_ENTITY_DEFINITION_TYPE, + page: 1, + perPage: 1, + filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`, + }); + + if (response.total === 1) { + throw new EntityIdConflict( + `Entity defintion with [${definition.id}] already exists.`, + definition + ); + } + + await soClient.create(SO_ENTITY_DEFINITION_TYPE, definition, { + id: definition.id, + overwrite: true, + }); + + return definition; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts new file mode 100644 index 00000000000000..766bbb10b1d675 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/start_transform.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { retryTransientEsErrors } from './helpers/retry'; +import { generateTransformId } from './transform/generate_transform_id'; + +export async function startTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const transformId = generateTransformId(definition); + try { + await retryTransientEsErrors( + () => esClient.transform.startTransform({ transform_id: transformId }, { ignore: [409] }), + { logger } + ); + } catch (err) { + logger.error(`Cannot start entity transform [${transformId}]`); + throw err; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts new file mode 100644 index 00000000000000..60a250a33f0d9e --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/stop_and_delete_transform.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; +import { generateTransformId } from './transform/generate_transform_id'; +import { retryTransientEsErrors } from './helpers/retry'; + +export async function stopAndDeleteTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + const transformId = generateTransformId(definition); + try { + await retryTransientEsErrors( + async () => { + await esClient.transform.stopTransform( + { transform_id: transformId, wait_for_completion: true, force: true }, + { ignore: [409] } + ); + await esClient.transform.deleteTransform( + { transform_id: transformId, force: true }, + { ignore: [404] } + ); + }, + { logger } + ); + } catch (e) { + logger.error(`Cannot stop or delete entity transform [${transformId}]`); + throw e; + } +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap new file mode 100644 index 00000000000000..e692f2068eafd6 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/__snapshots__/generate_transform.test.ts.snap @@ -0,0 +1,126 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`generateTransform(definition) should generate a valid summary transform 1`] = ` +Object { + "defer_validation": true, + "dest": Object { + "index": ".entities-observability.summary-v1.noop", + "pipeline": ".entities-observability.summary-v1.admin-console-logs-service", + }, + "frequency": "1m", + "pivot": Object { + "aggs": Object { + "_errorRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "log.level": "error", + }, + }, + ], + }, + }, + }, + "_logRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "log.level", + }, + }, + ], + }, + }, + }, + "entity.latestTimestamp": Object { + "max": Object { + "field": "@timestamp", + }, + }, + "entity.metadata.host.name": Object { + "terms": Object { + "field": "host.name", + "size": 1000, + }, + }, + "entity.metadata.kubernetes.pod.name": Object { + "terms": Object { + "field": "kubernetes.pod.name", + "size": 1000, + }, + }, + "entity.metadata.tags": Object { + "terms": Object { + "field": "tags", + "size": 1000, + }, + }, + "entity.metric.errorRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_errorRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A / 5", + }, + }, + }, + "entity.metric.logRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_logRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A / 5", + }, + }, + }, + }, + "group_by": Object { + "entity.identity.log.logger": Object { + "terms": Object { + "field": "log.logger", + "missing_bucket": false, + }, + }, + }, + }, + "settings": Object { + "deduce_mappings": false, + "unattended": true, + }, + "source": Object { + "index": Array [ + "kbn-data-forge-fake_stack.*", + ], + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-5m", + }, + }, + }, + ], + }, + }, + }, + "sync": Object { + "time": Object { + "delay": "60s", + "field": "@timestamp", + }, + }, + "transform_id": "entities-observability-summary-v1-admin-console-logs-service", +} +`; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts new file mode 100644 index 00000000000000..8c11aea1385198 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; + +export function generateMetadataAggregations(definition: EntityDefinition) { + if (!definition.metadata) { + return {}; + } + return definition.metadata.reduce( + (aggs, metadata) => ({ + ...aggs, + [`entity.metadata.${metadata.destination ?? metadata.source}`]: { + terms: { + field: metadata.source, + size: metadata.limit ?? 1000, + }, + }, + }), + {} + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts new file mode 100644 index 00000000000000..9527671768e357 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_metric_aggregations.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KeyMetric, Metric, EntityDefinition } from '@kbn/entities-schema'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { InvalidTransformError } from '../errors/invalid_transform_error'; + +function buildAggregation(metric: Metric, timestampField: string) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return {}; + case 'std_deviation': + return { + extended_stats: { field: metric.field }, + }; + case 'percentile': + if (metric.percentile == null) { + throw new InvalidTransformError( + 'You must provide a percentile value for percentile aggregations.' + ); + } + return { + percentiles: { + field: metric.field, + percents: [metric.percentile], + keyed: true, + }, + }; + case 'last_value': + return { + top_metrics: { + metrics: { field: metric.field }, + sort: { [timestampField]: 'desc' }, + }, + }; + default: + if (metric.field == null) { + throw new InvalidTransformError('You must provide a field for basic metric aggregations.'); + } + return { + [aggregation]: { field: metric.field }, + }; + } +} + +function buildMetricAggregations(keyMetric: KeyMetric, timestampField: string) { + return keyMetric.metrics.reduce((acc, metric) => { + const filter = metric.filter ? getElasticsearchQueryOrThrow(metric.filter) : { match_all: {} }; + const aggs = { metric: buildAggregation(metric, timestampField) }; + return { + ...acc, + [`_${keyMetric.name}_${metric.name}`]: { + filter, + ...(metric.aggregation !== 'doc_count' ? { aggs } : {}), + }, + }; + }, {}); +} + +function buildBucketPath(prefix: string, metric: Metric) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return `${prefix}>_count`; + case 'std_deviation': + return `${prefix}>metric[std_deviation]`; + case 'percentile': + return `${prefix}>metric[${metric.percentile}]`; + case 'last_value': + return `${prefix}>metric[${metric.field}]`; + default: + return `${prefix}>metric`; + } +} + +function convertEquationToPainless(bucketsPath: Record, equation: string) { + const workingEquation = equation || Object.keys(bucketsPath).join(' + '); + return Object.keys(bucketsPath).reduce((acc, key) => { + return acc.replaceAll(key, `params.${key}`); + }, workingEquation); +} + +function buildMetricEquation(keyMetric: KeyMetric) { + const bucketsPath = keyMetric.metrics.reduce( + (acc, metric) => ({ + ...acc, + [metric.name]: buildBucketPath(`_${keyMetric.name}_${metric.name}`, metric), + }), + {} + ); + return { + bucket_script: { + buckets_path: bucketsPath, + script: { + source: convertEquationToPainless(bucketsPath, keyMetric.equation), + lang: 'painless', + }, + }, + }; +} + +export function generateMetricAggregations(definition: EntityDefinition) { + if (!definition.metrics) { + return {}; + } + return definition.metrics.reduce((aggs, keyMetric) => { + return { + ...aggs, + ...buildMetricAggregations(keyMetric, definition.timestampField), + [`entity.metric.${keyMetric.name}`]: buildMetricEquation(keyMetric), + }; + }, {}); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts new file mode 100644 index 00000000000000..e97293b77dd4fe --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { entityDefinition } from '../helpers/fixtures/entity_definition'; +import { generateTransform } from './generate_transform'; + +describe('generateTransform(definition)', () => { + it('should generate a valid summary transform', () => { + const transform = generateTransform(entityDefinition); + expect(transform).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts new file mode 100644 index 00000000000000..6a8c0bd6377150 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { + QueryDslQueryContainer, + TransformPutTransformRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw'; +import { generateMetricAggregations } from './generate_metric_aggregations'; +import { + ENTITY_BASE_PREFIX, + ENTITY_DEFAULT_FREQUENCY, + ENTITY_DEFAULT_SYNC_DELAY, +} from '../../../../common/constants_entities'; +import { generateMetadataAggregations } from './generate_metadata_aggregations'; +import { generateTransformId } from './generate_transform_id'; +import { generateIngestPipelineId } from '../ingest_pipeline/generate_ingest_pipeline_id'; + +export function generateTransform(definition: EntityDefinition): TransformPutTransformRequest { + const filter: QueryDslQueryContainer[] = [ + { + range: { + [definition.timestampField]: { + gte: `now-${definition.lookback.toJSON()}`, + }, + }, + }, + ]; + + if (definition.filter) { + filter.push(getElasticsearchQueryOrThrow(definition.filter)); + } + + return { + transform_id: generateTransformId(definition), + defer_validation: true, + source: { + index: definition.indexPatterns, + query: { + bool: { + filter, + }, + }, + }, + dest: { + index: `${ENTITY_BASE_PREFIX}.noop`, + pipeline: generateIngestPipelineId(definition), + }, + frequency: definition.settings?.frequency || ENTITY_DEFAULT_FREQUENCY, + sync: { + time: { + field: definition.settings?.syncField ?? definition.timestampField, + delay: definition.settings?.syncDelay ?? ENTITY_DEFAULT_SYNC_DELAY, + }, + }, + settings: { + deduce_mappings: false, + unattended: true, + }, + pivot: { + group_by: definition.identityFields.reduce( + (acc, id) => ({ + ...acc, + [`entity.identity.${id.field}`]: { + terms: { field: id.field, missing_bucket: id.optional }, + }, + }), + {} + ), + aggs: { + ...generateMetricAggregations(definition), + ...generateMetadataAggregations(definition), + 'entity.latestTimestamp': { + max: { + field: definition.timestampField, + }, + }, + }, + }, + }; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts new file mode 100644 index 00000000000000..06faedb916774d --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/entities/transform/generate_transform_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { ENTITY_TRANSFORM_PREFIX } from '../../../../common/constants_entities'; + +export function generateTransformId(definition: EntityDefinition) { + return `${ENTITY_TRANSFORM_PREFIX}-${definition.id}`; +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts b/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts index b853d7a360e1d1..b364e63ff9a1f3 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts @@ -6,6 +6,7 @@ */ import { + ClusterPutComponentTemplateRequest, IndicesGetIndexTemplateResponse, IndicesPutIndexTemplateRequest, } from '@elastic/elasticsearch/lib/api/types'; @@ -47,21 +48,18 @@ function templateExists( }); } -// interface IndexPatternJson { -// index_patterns: string[]; -// name: string; -// template: { -// mappings: Record; -// settings: Record; -// }; -// } - interface TemplateManagementOptions { esClient: ElasticsearchClient; template: IndicesPutIndexTemplateRequest; logger: Logger; } +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + export async function maybeCreateTemplate({ esClient, template, @@ -93,9 +91,6 @@ export async function maybeCreateTemplate({ } export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { - const pattern = ASSETS_INDEX_PREFIX + '*'; - template.index_patterns = [pattern]; - try { await esClient.indices.putIndexTemplate(template); } catch (error: any) { @@ -108,3 +103,17 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan ); logger.debug(`Asset manager index template: ${JSON.stringify(template)}`); } + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await esClient.cluster.putComponentTemplate(component); + } catch (error: any) { + logger.error(`Error updating asset manager component template: ${error.message}`); + return; + } + + logger.info( + `Asset manager component template is up to date (use debug logging to see what was installed)` + ); + logger.debug(`Asset manager component template: ${JSON.stringify(component)}`); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts index 73b2cbe87c1386..4c947926cf571b 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/plugin.ts @@ -15,12 +15,17 @@ import { Logger, } from '@kbn/core/server'; -import { upsertTemplate } from './lib/manage_index_templates'; +import { upsertComponent, upsertTemplate } from './lib/manage_index_templates'; import { setupRoutes } from './routes'; import { assetsIndexTemplateConfig } from './templates/assets_template'; import { AssetClient } from './lib/asset_client'; import { AssetManagerPluginSetupDependencies, AssetManagerPluginStartDependencies } from './types'; import { AssetManagerConfig, configSchema, exposeToBrowserConfig } from '../common/config'; +import { entitiesBaseComponentTemplateConfig } from './templates/components/base'; +import { entitiesEventComponentTemplateConfig } from './templates/components/event'; +import { entitiesIndexTemplateConfig } from './templates/entities_template'; +import { entityDefinition } from './saved_objects'; +import { entitiesEntityComponentTemplateConfig } from './templates/components/entity'; export type AssetManagerServerPluginSetup = ReturnType; export type AssetManagerServerPluginStart = ReturnType; @@ -56,6 +61,8 @@ export class AssetManagerServerPlugin this.logger.info('Server is enabled'); + core.savedObjects.registerType(entityDefinition); + const assetClient = new AssetClient({ sourceIndices: this.config.sourceIndices, getApmIndices: plugins.apmDataAccess.getApmIndices, @@ -63,7 +70,7 @@ export class AssetManagerServerPlugin }); const router = core.http.createRouter(); - setupRoutes({ router, assetClient }); + setupRoutes({ router, assetClient, logger: this.logger }); return { assetClient, @@ -76,12 +83,36 @@ export class AssetManagerServerPlugin return; } + const esClient = core.elasticsearch.client.asInternalUser; upsertTemplate({ - esClient: core.elasticsearch.client.asInternalUser, + esClient, template: assetsIndexTemplateConfig, logger: this.logger, }).catch(() => {}); // it shouldn't reject, but just in case + // Install entities compoent templates and index template + Promise.all([ + upsertComponent({ + esClient, + logger: this.logger, + component: entitiesBaseComponentTemplateConfig, + }), + upsertComponent({ + esClient, + logger: this.logger, + component: entitiesEventComponentTemplateConfig, + }), + upsertComponent({ + esClient, + logger: this.logger, + component: entitiesEntityComponentTemplateConfig, + }), + ]) + .then(() => + upsertTemplate({ esClient, logger: this.logger, template: entitiesIndexTemplateConfig }) + ) + .catch(() => {}); + return {}; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts new file mode 100644 index 00000000000000..d403c39ae0ed12 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/create.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { stringifyZodError } from '@kbn/zod-helpers'; +import { SetupRouteOptions } from '../types'; +import { saveEntityDefinition } from '../../lib/entities/save_entity_definition'; +import { createAndInstallIngestPipeline } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { EntityIdConflict } from '../../lib/entities/errors/entity_id_conflict_error'; +import { createAndInstallTransform } from '../../lib/entities/create_and_install_transform'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { startTransform } from '../../lib/entities/start_transform'; +import { deleteEntityDefinition } from '../../lib/entities/delete_entity_definition'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; + +export function createEntityDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.post( + { + path: `${ENTITY_API_PREFIX}/definition`, + validate: { + body: (body, res) => { + try { + return res.ok(entityDefinitionSchema.parse(body)); + } catch (e) { + return res.badRequest(stringifyZodError(e)); + } + }, + }, + }, + async (context, req, res) => { + let definitionCreated = false; + let ingestPipelineCreated = false; + let transformCreated = false; + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + try { + const definition = await saveEntityDefinition(soClient, req.body); + definitionCreated = true; + await createAndInstallIngestPipeline(esClient, definition, logger); + ingestPipelineCreated = true; + await createAndInstallTransform(esClient, definition, logger); + transformCreated = true; + await startTransform(esClient, definition, logger); + + return res.ok({ body: definition }); + } catch (e) { + // Clean up anything that was successful. + if (definitionCreated) { + await deleteEntityDefinition(soClient, req.body, logger); + } + if (ingestPipelineCreated) { + await deleteIngestPipeline(esClient, req.body, logger); + } + if (transformCreated) { + await stopAndDeleteTransform(esClient, req.body, logger); + } + if (e instanceof EntityIdConflict) { + return res.conflict({ body: e }); + } + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts new file mode 100644 index 00000000000000..e1b273780a64fb --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/delete.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { schema } from '@kbn/config-schema'; +import { SetupRouteOptions } from '../types'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { deleteEntityDefinition } from '../../lib/entities/delete_entity_definition'; +import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; + +export function deleteEntityDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.delete<{ id: string }, unknown, unknown>( + { + path: `${ENTITY_API_PREFIX}/definition/{id}`, + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, req, res) => { + try { + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const definition = await readEntityDefinition(soClient, req.params.id, logger); + await stopAndDeleteTransform(esClient, definition, logger); + await deleteIngestPipeline(esClient, definition, logger); + await deleteEntityDefinition(soClient, definition, logger); + + return res.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof EntityDefinitionNotFound) { + return res.notFound({ body: e }); + } + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts new file mode 100644 index 00000000000000..3109ffc44520f5 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/entities/reset.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { schema } from '@kbn/config-schema'; +import { SetupRouteOptions } from '../types'; +import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; +import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; +import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; +import { stopAndDeleteTransform } from '../../lib/entities/stop_and_delete_transform'; +import { deleteIngestPipeline } from '../../lib/entities/delete_ingest_pipeline'; +import { deleteIndex } from '../../lib/entities/delete_index'; +import { createAndInstallIngestPipeline } from '../../lib/entities/create_and_install_ingest_pipeline'; +import { createAndInstallTransform } from '../../lib/entities/create_and_install_transform'; +import { startTransform } from '../../lib/entities/start_transform'; +import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; +import { ENTITY_API_PREFIX } from '../../../common/constants_entities'; + +export function resetEntityDefinitionRoute({ + router, + logger, +}: SetupRouteOptions) { + router.post<{ id: string }, unknown, unknown>( + { + path: `${ENTITY_API_PREFIX}/definition/{id}/_reset`, + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, req, res) => { + try { + const soClient = (await context.core).savedObjects.client; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const definition = await readEntityDefinition(soClient, req.params.id, logger); + + // Delete the transform and ingest pipeline + await stopAndDeleteTransform(esClient, definition, logger); + await deleteIngestPipeline(esClient, definition, logger); + await deleteIndex(esClient, definition, logger); + + // Recreate everything + await createAndInstallIngestPipeline(esClient, definition, logger); + await createAndInstallTransform(esClient, definition, logger); + await startTransform(esClient, definition, logger); + + return res.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof EntityDefinitionNotFound) { + return res.notFound({ body: e }); + } + if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) { + return res.customError({ body: e, statusCode: 400 }); + } + return res.customError({ body: e, statusCode: 500 }); + } + } + ); +} diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts index 52d3198bb8a9f9..d0b6c9f7ff0f11 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/index.ts @@ -14,16 +14,23 @@ import { hostsRoutes } from './assets/hosts'; import { servicesRoutes } from './assets/services'; import { containersRoutes } from './assets/containers'; import { podsRoutes } from './assets/pods'; +import { createEntityDefinitionRoute } from './entities/create'; +import { deleteEntityDefinitionRoute } from './entities/delete'; +import { resetEntityDefinitionRoute } from './entities/reset'; export function setupRoutes({ router, assetClient, + logger, }: SetupRouteOptions) { - pingRoute({ router, assetClient }); - sampleAssetsRoutes({ router, assetClient }); - assetsRoutes({ router, assetClient }); - hostsRoutes({ router, assetClient }); - servicesRoutes({ router, assetClient }); - containersRoutes({ router, assetClient }); - podsRoutes({ router, assetClient }); + pingRoute({ router, assetClient, logger }); + sampleAssetsRoutes({ router, assetClient, logger }); + assetsRoutes({ router, assetClient, logger }); + hostsRoutes({ router, assetClient, logger }); + servicesRoutes({ router, assetClient, logger }); + containersRoutes({ router, assetClient, logger }); + podsRoutes({ router, assetClient, logger }); + createEntityDefinitionRoute({ router, assetClient, logger }); + deleteEntityDefinitionRoute({ router, assetClient, logger }); + resetEntityDefinitionRoute({ router, assetClient, logger }); } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts b/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts index ae1b967a5b5967..561819d18cdae2 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/routes/types.ts @@ -6,9 +6,11 @@ */ import { IRouter, RequestHandlerContextBase } from '@kbn/core-http-server'; +import { Logger } from '@kbn/core/server'; import { AssetClient } from '../lib/asset_client'; export interface SetupRouteOptions { router: IRouter; assetClient: AssetClient; + logger: Logger; } diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts new file mode 100644 index 00000000000000..5a63444974ad79 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/entity_definition.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject, SavedObjectsType } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; + +export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition'; + +export const entityDefinition: SavedObjectsType = { + name: SO_ENTITY_DEFINITION_TYPE, + hidden: false, + namespaceType: 'multiple-isolated', + mappings: { + dynamic: false, + properties: { + id: { type: 'keyword' }, + name: { type: 'text' }, + description: { type: 'text' }, + type: { type: 'keyword' }, + filter: { type: 'keyword' }, + indexPatterns: { type: 'keyword' }, + identityFields: { type: 'object' }, + categories: { type: 'keyword' }, + metadata: { type: 'object' }, + metrics: { type: 'object' }, + staticFields: { type: 'object' }, + }, + }, + management: { + displayName: 'Entity Definition', + importableAndExportable: false, + getTitle(sloSavedObject: SavedObject) { + return `EntityDefinition: [${sloSavedObject.attributes.name}]`; + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts new file mode 100644 index 00000000000000..6145b05438bb24 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/saved_objects/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { entityDefinition, SO_ENTITY_DEFINITION_TYPE } from './entity_definition'; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts index 71d4058eba4c0e..b99ecc4559187e 100644 --- a/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/assets_template.ts @@ -6,11 +6,13 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSETS_INDEX_PREFIX } from '../constants'; export const assetsIndexTemplateConfig: IndicesPutIndexTemplateRequest = { name: 'assets', priority: 100, data_stream: {}, + index_patterns: [`${ASSETS_INDEX_PREFIX}*`], template: { settings: {}, mappings: { diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts new file mode 100644 index 00000000000000..adf527d653d9c7 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/base.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const entitiesBaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_base', + _meta: { + documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-base.html', + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + labels: { + type: 'object', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts new file mode 100644 index 00000000000000..e696d32e0dfb21 --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/entity.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_entity', + _meta: { + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + entity: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + indexPatterns: { + ignore_above: 1024, + type: 'keyword', + }, + defintionId: { + ignore_above: 1024, + type: 'keyword', + }, + latestTimestamp: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts new file mode 100644 index 00000000000000..6ad4b628fdf36a --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/components/event.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const entitiesEventComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: 'entities_v1_event', + _meta: { + documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-event.html', + ecs_version: '8.0.0', + }, + template: { + mappings: { + properties: { + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts new file mode 100644 index 00000000000000..d3934b24ea82cc --- /dev/null +++ b/x-pack/plugins/observability_solution/asset_manager/server/templates/entities_template.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ENTITY_BASE_PREFIX } from '../../common/constants_entities'; + +export const entitiesIndexTemplateConfig: IndicesPutIndexTemplateRequest = { + name: 'entities_v1_index_template', + _meta: { + description: 'The entities index template', + ecs_version: '8.0.0', + }, + composed_of: ['entities_v1_base', 'entities_v1_event', 'entities_v1_entity'], + index_patterns: [`${ENTITY_BASE_PREFIX}.*`], + priority: 1, + template: { + mappings: { + _meta: { + version: '1.6.0', + }, + date_detection: false, + dynamic_templates: [ + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + { + entity_metrics: { + mapping: { + // @ts-expect-error this should work per: https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html#match-mapping-type + type: '{dynamic_type}', + }, + // @ts-expect-error this should work per: https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html#match-mapping-type + match_mapping_type: ['long', 'double'], + path_match: 'entity.metric.*', + }, + }, + ], + }, + settings: { + index: { + codec: 'best_compression', + mapping: { + total_fields: { + limit: 2000, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json index da1095d989afb0..dbbc36252bfb19 100644 --- a/x-pack/plugins/observability_solution/asset_manager/tsconfig.json +++ b/x-pack/plugins/observability_solution/asset_manager/tsconfig.json @@ -26,6 +26,9 @@ "@kbn/metrics-data-access-plugin", "@kbn/core-elasticsearch-server", "@kbn/core-saved-objects-api-server", - "@kbn/core-saved-objects-api-server-mocks" + "@kbn/core-saved-objects-api-server-mocks", + "@kbn/entities-schema", + "@kbn/es-query", + "@kbn/zod-helpers" ] } diff --git a/yarn.lock b/yarn.lock index 98ce6357eebcda..63f50a8f16e937 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4583,6 +4583,10 @@ version "0.0.0" uid "" +"@kbn/entities-schema@link:x-pack/packages/kbn-entities-schema": + version "0.0.0" + uid "" + "@kbn/error-boundary-example-plugin@link:examples/error_boundary": version "0.0.0" uid ""