[EEM][POC] The POC for creating entity-centric indices using entity definitions

.github/CODEOWNERS

@@ -600,6 +600,7 @@ src/plugins/newsfeed @elastic/kibana-core
 test/common/plugins/newsfeed @elastic/kibana-core
 src/plugins/no_data_page @elastic/appex-sharedux
 x-pack/plugins/notifications @elastic/appex-sharedux
+x-pack/packages/kbn-oam-schema @elastic/obs-knowledge-team
 packages/kbn-object-versioning @elastic/appex-sharedux
 x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant
 x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant

package.json

@@ -621,6 +621,7 @@
     "@kbn/newsfeed-test-plugin": "link:test/common/plugins/newsfeed",
     "@kbn/no-data-page-plugin": "link:src/plugins/no_data_page",
     "@kbn/notifications-plugin": "link:x-pack/plugins/notifications",
+    "@kbn/oam-schema": "link:x-pack/packages/kbn-oam-schema",
     "@kbn/object-versioning": "link:packages/kbn-object-versioning",
     "@kbn/observability-ai-assistant-app-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_app",
     "@kbn/observability-ai-assistant-management-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_management",

tsconfig.base.json

@@ -1194,6 +1194,8 @@
       "@kbn/no-data-page-plugin/*": ["src/plugins/no_data_page/*"],
       "@kbn/notifications-plugin": ["x-pack/plugins/notifications"],
       "@kbn/notifications-plugin/*": ["x-pack/plugins/notifications/*"],
+      "@kbn/oam-schema": ["x-pack/packages/kbn-oam-schema"],
+      "@kbn/oam-schema/*": ["x-pack/packages/kbn-oam-schema/*"],
       "@kbn/object-versioning": ["packages/kbn-object-versioning"],
       "@kbn/object-versioning/*": ["packages/kbn-object-versioning/*"],
       "@kbn/observability-ai-assistant-app-plugin": ["x-pack/plugins/observability_solution/observability_ai_assistant_app"],

x-pack/packages/kbn-oam-schema/README.md

@@ -0,0 +1,3 @@
+# @kbn/oam-schema
+
+Empty package generated by @kbn/generate

x-pack/packages/kbn-oam-schema/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './src/schema/oam_definition';
+export * from './src/schema/asset';
+export * from './src/schema/common';

x-pack/packages/kbn-oam-schema/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-oam-schema'],
+};

x-pack/packages/kbn-oam-schema/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/oam-schema",
+  "owner": "@elastic/obs-knowledge-team"
+}

x-pack/packages/kbn-oam-schema/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/oam-schema",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-oam-schema/src/schema/asset.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import { arrayOfStringsSchema } from './common';
+
+export const assetSchema = z.intersection(
+  z.object({
+    asset: z.object({
+      id: z.string(),
+      indexPattern: arrayOfStringsSchema,
+      category: arrayOfStringsSchema,
+      identityField: arrayOfStringsSchema,
+      metric: z.record(z.string(), z.number()),
+    }),
+  }),
+  z.record(z.string(), z.string().or(z.number()))
+);

x-pack/packages/kbn-oam-schema/src/schema/common.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import moment from 'moment';
+
+export enum AssetType {
+  service = 'service',
+  host = 'host',
+  pod = 'pod',
+  node = 'node',
+}
+
+export const arrayOfStringsSchema = z.array(z.string());
+
+export const assetTypeSchema = z.nativeEnum(AssetType);
+
+export enum BasicAggregations {
+  avg = 'avg',
+  max = 'max',
+  min = 'min',
+  sum = 'sum',
+  cardinality = 'cardinality',
+  last_value = 'last_value',
+  std_deviation = 'std_deviation',
+}
+
+export const basicAggregationsSchema = z.nativeEnum(BasicAggregations);
+
+const metricNameSchema = z
+  .string()
+  .length(1)
+  .regex(/[a-zA-Z]/)
+  .toUpperCase();
+
+export const filterSchema = z.optional(z.string());
+
+export const basicMetricWithFieldSchema = z.object({
+  name: metricNameSchema,
+  aggregation: basicAggregationsSchema,
+  field: z.string(),
+  filter: filterSchema,
+});
+
+export const docCountMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('doc_count'),
+  filter: filterSchema,
+});
+
+export const durationSchema = z
+  .string()
+  .regex(/\d+[m|d|s|h]/)
+  .transform((val: string) => {
+    const parts = val.match(/(\d+)([m|s|h|d])/);
+    if (parts === null) {
+      throw new Error('Unable to parse duration');
+    }
+    const value = parseInt(parts[1], 10);
+    const unit = parts[2] as 'm' | 's' | 'h' | 'd';
+    const duration = moment.duration(value, unit);
+    return { ...duration, toJSON: () => val };
+  });
+
+export const percentileMetricSchema = z.object({
+  name: metricNameSchema,
+  aggregation: z.literal('percentile'),
+  field: z.string(),
+  percentile: z.number(),
+  filter: filterSchema,
+});
+
+export const metricSchema = z.discriminatedUnion('aggregation', [
+  basicMetricWithFieldSchema,
+  docCountMetricSchema,
+  percentileMetricSchema,
+]);
+
+export type Metric = z.infer<typeof metricSchema>;
+
+export const keyMetricSchema = z.object({
+  name: z.string(),
+  metrics: z.array(metricSchema),
+  equation: z.string(),
+});
+
+export type KeyMetric = z.infer<typeof keyMetricSchema>;
+
+export const metadataSchema = z
+  .object({
+    source: z.string(),
+    destination: z.optional(z.string()),
+    limit: z.optional(z.number().default(1000)),
+  })
+  .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 })));

x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+import {
+  arrayOfStringsSchema,
+  assetTypeSchema,
+  keyMetricSchema,
+  metadataSchema,
+  filterSchema,
+  durationSchema,
+} from './common';
+
+export const oamDefinitionSchema = z.object({
+  id: z.string().regex(/^[\w-]+$/),
+  name: z.string(),
+  description: z.optional(z.string()),
+  type: assetTypeSchema,
+  filter: filterSchema,
+  indexPatterns: arrayOfStringsSchema,
+  identityFields: arrayOfStringsSchema,
+  identityTemplate: z.string(),
+  categories: arrayOfStringsSchema,
+  metadata: z.optional(z.array(metadataSchema)),
+  metrics: z.optional(z.array(keyMetricSchema)),
+  staticFields: z.optional(z.record(z.string(), z.string())),
+  lookback: durationSchema,
+  timestampField: z.string(),
+  settings: z.optional(
+    z.object({
+      syncField: z.optional(z.string()),
+      syncDelay: z.optional(z.string()),
+      frequency: z.optional(z.string()),
+    })
+  ),
+});
+
+export type OAMDefinition = z.infer<typeof oamDefinitionSchema>;

x-pack/packages/kbn-oam-schema/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+  ]
+}

x-pack/plugins/observability_solution/asset_manager/common/constants_oam.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const OAM_VERSION = 'v1';
+export const OAM_BASE_PREFIX = `.oam-observability.asset-${OAM_VERSION}`;
+export const OAM_TRANSFORM_PREFIX = `oam-observability-asset-${OAM_VERSION}`;

x-pack/plugins/observability_solution/asset_manager/server/lib/manage_index_templates.ts

@@ -6,6 +6,7 @@
  */
 
 import {
+  ClusterPutComponentTemplateRequest,
   IndicesGetIndexTemplateResponse,
   IndicesPutIndexTemplateRequest,
 } from '@elastic/elasticsearch/lib/api/types';
@@ -47,21 +48,18 @@ function templateExists(
   });
 }
 
-// interface IndexPatternJson {
-//   index_patterns: string[];
-//   name: string;
-//   template: {
-//     mappings: Record<string, any>;
-//     settings: Record<string, any>;
-//   };
-// }
-
 interface TemplateManagementOptions {
   esClient: ElasticsearchClient;
   template: IndicesPutIndexTemplateRequest;
   logger: Logger;
 }
 
+interface ComponentManagementOptions {
+  esClient: ElasticsearchClient;
+  component: ClusterPutComponentTemplateRequest;
+  logger: Logger;
+}
+
 export async function maybeCreateTemplate({
   esClient,
   template,
@@ -93,9 +91,6 @@ export async function maybeCreateTemplate({
 }
 
 export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) {
-  const pattern = ASSETS_INDEX_PREFIX + '*';
-  template.index_patterns = [pattern];
-
   try {
     await esClient.indices.putIndexTemplate(template);
   } catch (error: any) {
@@ -108,3 +103,17 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan
   );
   logger.debug(`Asset manager index template: ${JSON.stringify(template)}`);
 }
+
+export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) {
+  try {
+    await esClient.cluster.putComponentTemplate(component);
+  } catch (error: any) {
+    logger.error(`Error updating asset manager component template: ${error.message}`);
+    return;
+  }
+
+  logger.info(
+    `Asset manager component template is up to date (use debug logging to see what was installed)`
+  );
+  logger.debug(`Asset manager component template: ${JSON.stringify(component)}`);
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { OAMDefinition } from '@kbn/oam-schema';
+import { generateProcessors } from './ingest_pipeline/generate_processors';
+import { retryTransientEsErrors } from './helpers/retry';
+import { OAMSecurityException } from './errors/oam_security_exception';
+import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id';
+
+export async function createAndInstallIngestPipeline(
+  esClient: ElasticsearchClient,
+  definition: OAMDefinition,
+  logger: Logger
+) {
+  const processors = generateProcessors(definition);
+  const id = generateIngestPipelineId(definition);
+  try {
+    await retryTransientEsErrors(
+      () =>
+        esClient.ingest.putPipeline({
+          id,
+          processors,
+        }),
+      { logger }
+    );
+  } catch (e) {
+    logger.error(`Cannot create OAM tranform for [${definition.id}] asset defintion`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new OAMSecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return id;
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { OAMDefinition } from '@kbn/oam-schema';
+import { generateTransform } from './transform/generate_transform';
+import { retryTransientEsErrors } from './helpers/retry';
+import { OAMSecurityException } from './errors/oam_security_exception';
+
+export async function createAndInstallTransform(
+  esClient: ElasticsearchClient,
+  definition: OAMDefinition,
+  logger: Logger
+) {
+  const transform = generateTransform(definition);
+  try {
+    await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger });
+  } catch (e) {
+    logger.error(`Cannot create OAM transform for [${definition.id}] asset definition`);
+    if (e.meta?.body?.error?.type === 'security_exception') {
+      throw new OAMSecurityException(e.meta.body.error.reason, definition);
+    }
+    throw e;
+  }
+  return transform.transform_id;
+}