[EEM][POC] The POC for creating entity-centric indices using entity definitions

.github/CODEOWNERS

@@ -600,6 +600,7 @@ src/plugins/newsfeed @elastic/kibana-core
 test/common/plugins/newsfeed @elastic/kibana-core
 src/plugins/no_data_page @elastic/appex-sharedux
 x-pack/plugins/notifications @elastic/appex-sharedux
+x-pack/packages/kbn-oam-schema @elastic/obs-knowledge-team
 packages/kbn-object-versioning @elastic/appex-sharedux
 x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant
 x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant

package.json

@@ -621,6 +621,7 @@
     "@kbn/newsfeed-test-plugin": "link:test/common/plugins/newsfeed",
     "@kbn/no-data-page-plugin": "link:src/plugins/no_data_page",
     "@kbn/notifications-plugin": "link:x-pack/plugins/notifications",
+    "@kbn/oam-schema": "link:x-pack/packages/kbn-oam-schema",
     "@kbn/object-versioning": "link:packages/kbn-object-versioning",
     "@kbn/observability-ai-assistant-app-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_app",
     "@kbn/observability-ai-assistant-management-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_management",

tsconfig.base.json

@@ -1194,6 +1194,8 @@
       "@kbn/no-data-page-plugin/*": ["src/plugins/no_data_page/*"],
       "@kbn/notifications-plugin": ["x-pack/plugins/notifications"],
       "@kbn/notifications-plugin/*": ["x-pack/plugins/notifications/*"],
+      "@kbn/oam-schema": ["x-pack/packages/kbn-oam-schema"],
+      "@kbn/oam-schema/*": ["x-pack/packages/kbn-oam-schema/*"],
       "@kbn/object-versioning": ["packages/kbn-object-versioning"],
       "@kbn/object-versioning/*": ["packages/kbn-object-versioning/*"],
       "@kbn/observability-ai-assistant-app-plugin": ["x-pack/plugins/observability_solution/observability_ai_assistant_app"],

x-pack/packages/kbn-oam-schema/README.md

@@ -0,0 +1,3 @@
+# @kbn/oam-schema
+
+Empty package generated by @kbn/generate

x-pack/packages/kbn-oam-schema/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './src/schema/entity_definition';
+export * from './src/schema/entity';
+export * from './src/schema/common';

x-pack/packages/kbn-oam-schema/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-oam-schema'],
+};

x-pack/packages/kbn-oam-schema/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/oam-schema",
+  "owner": "@elastic/obs-knowledge-team"
+}

x-pack/packages/kbn-oam-schema/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/oam-schema",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-oam-schema/src/schema/common.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import moment from 'moment';
+
+export enum EntityType {
+  service = 'service',
+  host = 'host',
+  pod = 'pod',
+  node = 'node',
+}
+
+export const arrayOfStringsSchema = z.array(z.string());
+
+export const entityTypeSchema = z.nativeEnum(EntityType);
+
+export enum BasicAggregations {
+  avg = 'avg',
+  max = 'max',
+  min = 'min',
+  sum = 'sum',
+  cardinality = 'cardinality',
+  last_value = 'last_value',
+  std_deviation = 'std_deviation',
+}
+
+export const basicAggregationsSchema = z.nativeEnum(BasicAggregations);
+
+const metricNameSchema = z
+  .string()
+  .length(1)
+  .regex(/[a-zA-Z]/)
+  .toUpperCase();
+
+export const filterSchema = z.optional(z.string());
+
+export const basicMetricWithFieldSchema = z.object({
+  name: metricNameSchema,
+  aggregation: basicAggregationsSchema,
+  field: z.string(),
+  filter: filterSchema,
+});
+
+export const docCountMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('doc_count'),
+  filter: filterSchema,
+});
+
+export const durationSchema = z
+  .string()
+  .regex(/\d+[m|d|s|h]/)
+  .transform((val: string) => {
+    const parts = val.match(/(\d+)([m|s|h|d])/);
+    if (parts === null) {
+      throw new Error('Unable to parse duration');
+    }
+    const value = parseInt(parts[1], 10);
+    const unit = parts[2] as 'm' | 's' | 'h' | 'd';
+    const duration = moment.duration(value, unit);
+    return { ...duration, toJSON: () => val };
+  });
+
+export const percentileMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('percentile'),
+  field: z.string(),
+  percentile: z.number(),
+  filter: filterSchema,
+});
+
+export const metricSchema = z.discriminatedUnion('aggregation', [
+  basicMetricWithFieldSchema,
+  docCountMetricSchema,
+  percentileMetricSchema,
+]);
+
+export type Metric = z.infer<typeof metricSchema>;
+
+export const keyMetricSchema = z.object({
+  name: z.string(),
+  metrics: z.array(metricSchema),
+  equation: z.string(),
+});
+
+export type KeyMetric = z.infer<typeof keyMetricSchema>;
+
+export const metadataSchema = z
+  .object({
+    source: z.string(),
+    destination: z.optional(z.string()),
+    limit: z.optional(z.number().default(1000)),
+  })
+  .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 })));

x-pack/packages/kbn-oam-schema/src/schema/entity.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import { arrayOfStringsSchema } from './common';
+
+export const entitySchema = z.intersection(
+  z.object({
+    entity: z.object({
+      id: z.string(),
+      indexPatterns: arrayOfStringsSchema,
+      identityFields: arrayOfStringsSchema,
+      metric: z.record(z.string(), z.number()),
+    }),
+  }),
+  z.record(z.string(), z.string().or(z.number()))
+);

x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import {
+  arrayOfStringsSchema,
+  entityTypeSchema,
+  keyMetricSchema,
+  metadataSchema,
+  filterSchema,
+  durationSchema,
+} from './common';
+
+export const entityDefinitionSchema = z.object({
+  id: z.string().regex(/^[\w-]+$/),
+  name: z.string(),
+  description: z.optional(z.string()),
+  type: entityTypeSchema,
+  filter: filterSchema,
+  indexPatterns: arrayOfStringsSchema,
+  identityFields: arrayOfStringsSchema,
+  identityTemplate: z.string(),
+  metadata: z.optional(z.array(metadataSchema)),
+  metrics: z.optional(z.array(keyMetricSchema)),
+  staticFields: z.optional(z.record(z.string(), z.string())),
+  lookback: durationSchema,
+  timestampField: z.string(),
+  settings: z.optional(
+    z.object({
+      syncField: z.optional(z.string()),
+      syncDelay: z.optional(z.string()),
+      frequency: z.optional(z.string()),
+    })
+  ),
+});
+
+export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

x-pack/packages/kbn-oam-schema/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+  ]
+}

x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ENTITY_VERSION = 'v1';
+export const ENTITY_BASE_PREFIX = `.entities-observability.entity-${ENTITY_VERSION}`;
+export const ENTITY_TRANSFORM_PREFIX = `entities-observability-entity-${ENTITY_VERSION}`;
+export const ENTITY_DEFAULT_FREQUENCY = '1m';
+export const ENTITY_DEFAULT_SYNC_DELAY = '60s';
+export const ENTITY_API_PREFIX = '/api/entities';

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { generateProcessors } from './ingest_pipeline/generate_processors';
+import { retryTransientEsErrors } from './helpers/retry';
+import { EntitySecurityException } from './errors/entity_security_exception';
+import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id';
+
+export async function createAndInstallIngestPipeline(
+  esClient: ElasticsearchClient,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const processors = generateProcessors(definition);
+  const id = generateIngestPipelineId(definition);
+  try {
+    await retryTransientEsErrors(
+      () =>
+        esClient.ingest.putPipeline({
+          id,
+          processors,
+        }),
+      { logger }
+    );
+  } catch (e) {
+    logger.error(`Cannot create entity ingest pipeline for [${definition.id}] entity defintion`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return id;
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { generateTransform } from './transform/generate_transform';
+import { retryTransientEsErrors } from './helpers/retry';
+import { EntitySecurityException } from './errors/entity_security_exception';
+
+export async function createAndInstallTransform(
+  esClient: ElasticsearchClient,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const transform = generateTransform(definition);
+  try {
+    await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger });
+  } catch (e) {
+    logger.error(`Cannot create entity transform for [${definition.id}] entity definition`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return transform.transform_id;
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
+import { EntityDefinitionNotFound } from './errors/entity_not_found';
+
+export async function deleteEntityDefinition(
+  soClient: SavedObjectsClientContract,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const response = await soClient.find<EntityDefinition>({
+    type: SO_ENTITY_DEFINITION_TYPE,
+    page: 1,
+    perPage: 1,
+    filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
+  });
+
+  if (response.total === 0) {
+    logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
+    throw new EntityDefinitionNotFound(`Entity defintion with [${definition.id}] not found.`);
+  }
+
+  await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { generateIndexName } from './helpers/generate_index_name';
+
+export async function deleteIndex(
+  esClient: ElasticsearchClient,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const indexName = generateIndexName(definition);
+  try {
+    await esClient.indices.delete({ index: indexName, ignore_unavailable: true });
+  } catch (e) {
+    logger.error(`Unable to remove entity defintion index [${definition.id}}]`);
+    throw e;
+  }
+}