[EEM][POC] The POC for creating entity-centric indices using entity definitions

.github/CODEOWNERS

@@ -390,6 +390,7 @@ src/plugins/embeddable @elastic/kibana-presentation
 x-pack/examples/embedded_lens_example @elastic/kibana-visualizations
 x-pack/plugins/encrypted_saved_objects @elastic/kibana-security
 x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend
+x-pack/packages/kbn-entities-schema @elastic/obs-knowledge-team
 examples/error_boundary @elastic/appex-sharedux
 packages/kbn-es @elastic/kibana-operations
 packages/kbn-es-archiver @elastic/kibana-operations @elastic/appex-qa

package.json

@@ -441,6 +441,7 @@
     "@kbn/embedded-lens-example-plugin": "link:x-pack/examples/embedded_lens_example",
     "@kbn/encrypted-saved-objects-plugin": "link:x-pack/plugins/encrypted_saved_objects",
     "@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search",
+    "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",
     "@kbn/es-query": "link:packages/kbn-es-query",

tsconfig.base.json

@@ -774,6 +774,8 @@
       "@kbn/encrypted-saved-objects-plugin/*": ["x-pack/plugins/encrypted_saved_objects/*"],
       "@kbn/enterprise-search-plugin": ["x-pack/plugins/enterprise_search"],
       "@kbn/enterprise-search-plugin/*": ["x-pack/plugins/enterprise_search/*"],
+      "@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"],
+      "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],
       "@kbn/error-boundary-example-plugin/*": ["examples/error_boundary/*"],
       "@kbn/es": ["packages/kbn-es"],

x-pack/packages/kbn-entities-schema/README.md

@@ -0,0 +1,3 @@
+# @kbn/entities-schema
+
+The entities schema for the asset model for Observability

x-pack/packages/kbn-entities-schema/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './src/schema/entity_definition';
+export * from './src/schema/entity';
+export * from './src/schema/common';

x-pack/packages/kbn-entities-schema/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-entities-schema'],
+};

x-pack/packages/kbn-entities-schema/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/entities-schema",
+  "owner": "@elastic/obs-knowledge-team"
+}

x-pack/packages/kbn-entities-schema/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/entities-schema",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-entities-schema/src/schema/common.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import moment from 'moment';
+
+export enum EntityType {
+  service = 'service',
+  host = 'host',
+  pod = 'pod',
+  node = 'node',
+}
+
+export const arrayOfStringsSchema = z.array(z.string());
+
+export const entityTypeSchema = z.nativeEnum(EntityType);
+
+export enum BasicAggregations {
+  avg = 'avg',
+  max = 'max',
+  min = 'min',
+  sum = 'sum',
+  cardinality = 'cardinality',
+  last_value = 'last_value',
+  std_deviation = 'std_deviation',
+}
+
+export const basicAggregationsSchema = z.nativeEnum(BasicAggregations);
+
+const metricNameSchema = z
+  .string()
+  .length(1)
+  .regex(/[a-zA-Z]/)
+  .toUpperCase();
+
+export const filterSchema = z.optional(z.string());
+
+export const basicMetricWithFieldSchema = z.object({
+  name: metricNameSchema,
+  aggregation: basicAggregationsSchema,
+  field: z.string(),
+  filter: filterSchema,
+});
+
+export const docCountMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('doc_count'),
+  filter: filterSchema,
+});
+
+export const durationSchema = z
+  .string()
+  .regex(/\d+[m|d|s|h]/)
+  .transform((val: string) => {
+    const parts = val.match(/(\d+)([m|s|h|d])/);
+    if (parts === null) {
+      throw new Error('Unable to parse duration');
+    }
+    const value = parseInt(parts[1], 10);
+    const unit = parts[2] as 'm' | 's' | 'h' | 'd';
+    const duration = moment.duration(value, unit);
+    return { ...duration, toJSON: () => val };
+  });
+
+export const percentileMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('percentile'),
+  field: z.string(),
+  percentile: z.number(),
+  filter: filterSchema,
+});
+
+export const metricSchema = z.discriminatedUnion('aggregation', [
+  basicMetricWithFieldSchema,
+  docCountMetricSchema,
+  percentileMetricSchema,
+]);
+
+export type Metric = z.infer<typeof metricSchema>;
+
+export const keyMetricSchema = z.object({
+  name: z.string(),
+  metrics: z.array(metricSchema),
+  equation: z.string(),
+});
+
+export type KeyMetric = z.infer<typeof keyMetricSchema>;
+
+export const metadataSchema = z
+  .object({
+    source: z.string(),
+    destination: z.optional(z.string()),
+    limit: z.optional(z.number().default(1000)),
+  })
+  .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 })));
+
+export const identityFieldsSchema = z
+  .object({
+    field: z.string(),
+    optional: z.boolean(),
+  })
+  .or(z.string().transform((value) => ({ field: value, optional: false })));

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import { arrayOfStringsSchema } from './common';
+
+export const entitySchema = z.intersection(
+  z.object({
+    entity: z.object({
+      id: z.string(),
+      indexPatterns: arrayOfStringsSchema,
+      identityFields: arrayOfStringsSchema,
+      metric: z.record(z.string(), z.number()),
+    }),
+  }),
+  z.record(z.string(), z.string().or(z.number()))
+);

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import {
+  arrayOfStringsSchema,
+  entityTypeSchema,
+  keyMetricSchema,
+  metadataSchema,
+  filterSchema,
+  durationSchema,
+  identityFieldsSchema,
+} from './common';
+
+export const entityDefinitionSchema = z.object({
+  id: z.string().regex(/^[\w-]+$/),
+  name: z.string(),
+  description: z.optional(z.string()),
+  type: entityTypeSchema,
+  filter: filterSchema,
+  indexPatterns: arrayOfStringsSchema,
+  identityFields: z.array(identityFieldsSchema),
+  identityTemplate: z.string(),
+  metadata: z.optional(z.array(metadataSchema)),
+  metrics: z.optional(z.array(keyMetricSchema)),
+  staticFields: z.optional(z.record(z.string(), z.string())),
+  lookback: durationSchema,
+  timestampField: z.string(),
+  settings: z.optional(
+    z.object({
+      syncField: z.optional(z.string()),
+      syncDelay: z.optional(z.string()),
+      frequency: z.optional(z.string()),
+    })
+  ),
+});
+
+export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

x-pack/packages/kbn-entities-schema/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+  ]
+}

x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ENTITY_VERSION = 'v1';
+export const ENTITY_BASE_PREFIX = `.entities-observability.summary-${ENTITY_VERSION}`;
+export const ENTITY_TRANSFORM_PREFIX = `entities-observability-summary-${ENTITY_VERSION}`;
+export const ENTITY_DEFAULT_FREQUENCY = '1m';
+export const ENTITY_DEFAULT_SYNC_DELAY = '60s';
+export const ENTITY_API_PREFIX = '/api/entities';

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/entities-schema';
+import { generateProcessors } from './ingest_pipeline/generate_processors';
+import { retryTransientEsErrors } from './helpers/retry';
+import { EntitySecurityException } from './errors/entity_security_exception';
+import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id';
+
+export async function createAndInstallIngestPipeline(
+  esClient: ElasticsearchClient,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const processors = generateProcessors(definition);
+  const id = generateIngestPipelineId(definition);
+  try {
+    await retryTransientEsErrors(
+      () =>
+        esClient.ingest.putPipeline({
+          id,
+          processors,
+        }),
+      { logger }
+    );
+  } catch (e) {
+    logger.error(`Cannot create entity ingest pipeline for [${definition.id}] entity defintion`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return id;
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/entities-schema';
+import { generateTransform } from './transform/generate_transform';
+import { retryTransientEsErrors } from './helpers/retry';
+import { EntitySecurityException } from './errors/entity_security_exception';
+
+export async function createAndInstallTransform(
+  esClient: ElasticsearchClient,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const transform = generateTransform(definition);
+  try {
+    await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger });
+  } catch (e) {
+    logger.error(`Cannot create entity transform for [${definition.id}] entity definition`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return transform.transform_id;
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/entities-schema';
+import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
+import { EntityDefinitionNotFound } from './errors/entity_not_found';
+
+export async function deleteEntityDefinition(
+  soClient: SavedObjectsClientContract,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const response = await soClient.find<EntityDefinition>({
+    type: SO_ENTITY_DEFINITION_TYPE,
+    page: 1,
+    perPage: 1,
+    filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
+  });
+
+  if (response.total === 0) {
+    logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
+    throw new EntityDefinitionNotFound(`Entity defintion with [${definition.id}] not found.`);
+  }
+
+  await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
+}