[EEM][POC] The POC for creating entity-centric indices using entity definitions

x-pack/packages/kbn-oam-schema/index.ts

@@ -5,6 +5,6 @@
  * 2.0.
  */
 
-export * from './src/schema/oam_definition';
-export * from './src/schema/asset';
+export * from './src/schema/entity_definition';
+export * from './src/schema/entity';
 export * from './src/schema/common';

x-pack/packages/kbn-oam-schema/src/schema/common.ts

@@ -8,7 +8,7 @@
 import { z } from 'zod';
 import moment from 'moment';
 
-export enum AssetType {
+export enum EntityType {
   service = 'service',
   host = 'host',
   pod = 'pod',
@@ -17,7 +17,7 @@ export enum AssetType {
 
 export const arrayOfStringsSchema = z.array(z.string());
 
-export const assetTypeSchema = z.nativeEnum(AssetType);
+export const entityTypeSchema = z.nativeEnum(EntityType);
 
 export enum BasicAggregations {
   avg = 'avg',

x-pack/packages/kbn-oam-schema/src/schema/asset.ts → x-pack/packages/kbn-oam-schema/src/schema/entity.ts

@@ -8,13 +8,12 @@
 import { z } from 'zod';
 import { arrayOfStringsSchema } from './common';
 
-export const assetSchema = z.intersection(
+export const entitySchema = z.intersection(
   z.object({
-    asset: z.object({
+    entity: z.object({
       id: z.string(),
-      indexPattern: arrayOfStringsSchema,
-      category: arrayOfStringsSchema,
-      identityField: arrayOfStringsSchema,
+      indexPatterns: arrayOfStringsSchema,
+      identityFields: arrayOfStringsSchema,
       metric: z.record(z.string(), z.number()),
     }),
   }),

x-pack/packages/kbn-oam-schema/src/schema/oam_definition.ts → x-pack/packages/kbn-oam-schema/src/schema/entity_definition.ts

@@ -8,23 +8,22 @@
 import { z } from 'zod';
 import {
   arrayOfStringsSchema,
-  assetTypeSchema,
+  entityTypeSchema,
   keyMetricSchema,
   metadataSchema,
   filterSchema,
   durationSchema,
 } from './common';
 
-export const oamDefinitionSchema = z.object({
+export const entityDefinitionSchema = z.object({
   id: z.string().regex(/^[\w-]+$/),
   name: z.string(),
   description: z.optional(z.string()),
-  type: assetTypeSchema,
+  type: entityTypeSchema,
   filter: filterSchema,
   indexPatterns: arrayOfStringsSchema,
   identityFields: arrayOfStringsSchema,
   identityTemplate: z.string(),
-  categories: arrayOfStringsSchema,
   metadata: z.optional(z.array(metadataSchema)),
   metrics: z.optional(z.array(keyMetricSchema)),
   staticFields: z.optional(z.record(z.string(), z.string())),
@@ -39,4 +38,4 @@ export const oamDefinitionSchema = z.object({
   ),
 });
 
-export type OAMDefinition = z.infer<typeof oamDefinitionSchema>;
+export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

x-pack/plugins/observability_solution/asset_manager/common/constants_entities.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ENTITY_VERSION = 'v1';
+export const ENTITY_BASE_PREFIX = `.entities-observability.entity-${ENTITY_VERSION}`;
+export const ENTITY_TRANSFORM_PREFIX = `entities-observability-entity-${ENTITY_VERSION}`;
+export const ENTITY_DEFAULT_FREQUENCY = '1m';
+export const ENTITY_DEFAULT_SYNC_DELAY = '60s';
+export const ENTITY_API_PREFIX = '/api/entities';

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_ingest_pipeline.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_ingest_pipeline.ts

@@ -6,15 +6,15 @@
  */
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 import { generateProcessors } from './ingest_pipeline/generate_processors';
 import { retryTransientEsErrors } from './helpers/retry';
-import { OAMSecurityException } from './errors/oam_security_exception';
+import { EntitySecurityException } from './errors/entity_security_exception';
 import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id';
 
 export async function createAndInstallIngestPipeline(
   esClient: ElasticsearchClient,
-  definition: OAMDefinition,
+  definition: EntityDefinition,
   logger: Logger
 ) {
   const processors = generateProcessors(definition);
@@ -29,9 +29,9 @@ export async function createAndInstallIngestPipeline(
       { logger }
     );
   } catch (e) {
-    logger.error(`Cannot create OAM tranform for [${definition.id}] asset defintion`);
+    logger.error(`Cannot create entity ingest pipeline for [${definition.id}] entity defintion`);
     if (e.meta?.body?.error?.type === 'security_exception') {
-      throw new OAMSecurityException(e.meta.body.error.reason, definition);
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
     }
     throw e;
   }

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/create_and_install_transform.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/create_and_install_transform.ts

@@ -6,23 +6,23 @@
  */
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 import { generateTransform } from './transform/generate_transform';
 import { retryTransientEsErrors } from './helpers/retry';
-import { OAMSecurityException } from './errors/oam_security_exception';
+import { EntitySecurityException } from './errors/entity_security_exception';
 
 export async function createAndInstallTransform(
   esClient: ElasticsearchClient,
-  definition: OAMDefinition,
+  definition: EntityDefinition,
   logger: Logger
 ) {
   const transform = generateTransform(definition);
   try {
     await retryTransientEsErrors(() => esClient.transform.putTransform(transform), { logger });
   } catch (e) {
-    logger.error(`Cannot create OAM transform for [${definition.id}] asset definition`);
+    logger.error(`Cannot create entity transform for [${definition.id}] entity definition`);
     if (e.meta?.body?.error?.type === 'security_exception') {
-      throw new OAMSecurityException(e.meta.body.error.reason, definition);
+      throw new EntitySecurityException(e.meta.body.error.reason, definition);
     }
     throw e;
   }

x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_entity_definition.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
+import { EntityDefinitionNotFound } from './errors/entity_not_found';
+
+export async function deleteEntityDefinition(
+  soClient: SavedObjectsClientContract,
+  definition: EntityDefinition,
+  logger: Logger
+) {
+  const response = await soClient.find<EntityDefinition>({
+    type: SO_ENTITY_DEFINITION_TYPE,
+    page: 1,
+    perPage: 1,
+    filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
+  });
+
+  if (response.total === 0) {
+    logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
+    throw new EntityDefinitionNotFound(`Entity defintion with [${definition.id}] not found.`);
+  }
+
+  await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
+}

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_index.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_index.ts

@@ -6,19 +6,19 @@
  */
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 import { generateIndexName } from './helpers/generate_index_name';
 
 export async function deleteIndex(
   esClient: ElasticsearchClient,
-  definition: OAMDefinition,
+  definition: EntityDefinition,
   logger: Logger
 ) {
   const indexName = generateIndexName(definition);
   try {
-    esClient.indices.delete({ index: indexName, ignore_unavailable: true });
+    await esClient.indices.delete({ index: indexName, ignore_unavailable: true });
   } catch (e) {
-    logger.error(`Unable to remove OAM Defintion index [${definition.id}}]`);
+    logger.error(`Unable to remove entity defintion index [${definition.id}}]`);
     throw e;
   }
 }

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/delete_ingest_pipeline.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/delete_ingest_pipeline.ts

@@ -6,13 +6,13 @@
  */
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 import { generateIngestPipelineId } from './ingest_pipeline/generate_ingest_pipeline_id';
 import { retryTransientEsErrors } from './helpers/retry';
 
 export async function deleteIngestPipeline(
   esClient: ElasticsearchClient,
-  definition: OAMDefinition,
+  definition: EntityDefinition,
   logger: Logger
 ) {
   const pipelineId = generateIngestPipelineId(definition);

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_security_exception.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_id_conflict_error.ts

@@ -5,14 +5,14 @@
  * 2.0.
  */
 
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 
-export class OAMSecurityException extends Error {
-  public defintion: OAMDefinition;
+export class EntityIdConflict extends Error {
+  public defintion: EntityDefinition;
 
-  constructor(message: string, def: OAMDefinition) {
+  constructor(message: string, def: EntityDefinition) {
     super(message);
-    this.name = 'OAMSecurityException';
+    this.name = 'EntityIdConflict';
     this.defintion = def;
   }
 }

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_not_found.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_not_found.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-export class OAMNotFound extends Error {
+export class EntityDefinitionNotFound extends Error {
   constructor(message: string) {
     super(message);
-    this.name = 'OAMNotFound';
+    this.name = 'EntityDefinitionNotFound';
   }
 }

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/errors/oam_id_conflict_error.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/errors/entity_security_exception.ts

@@ -5,14 +5,14 @@
  * 2.0.
  */
 
-import { OAMDefinition } from '@kbn/oam-schema';
+import { EntityDefinition } from '@kbn/oam-schema';
 
-export class OAMIdConflict extends Error {
-  public defintion: OAMDefinition;
+export class EntitySecurityException extends Error {
+  public defintion: EntityDefinition;
 
-  constructor(message: string, def: OAMDefinition) {
+  constructor(message: string, def: EntityDefinition) {
     super(message);
-    this.name = 'OAMIdConflict';
+    this.name = 'EntitySecurityException';
     this.defintion = def;
   }
 }

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/fixtures/oam_definition.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -5,14 +5,13 @@
  * 2.0.
  */
 
-import { oamDefinitionSchema } from '@kbn/oam-schema';
-export const oamDefinition = oamDefinitionSchema.parse({
+import { entityDefinitionSchema } from '@kbn/oam-schema';
+export const entityDefinition = entityDefinitionSchema.parse({
   id: 'admin-console-logs-service',
   name: 'Services for Admin Console',
   type: 'service',
   indexPatterns: ['kbn-data-forge-fake_stack.*'],
   timestampField: '@timestamp',
-  categories: ['service', 'logs'],
   identityFields: ['log.logger'],
   identityTemplate: 'service:{{log.logger}}',
   metadata: ['tags', 'host.name', 'kubernetes.pod.name'],

x-pack/plugins/observability_solution/asset_manager/server/lib/oam/helpers/generate_index_name.ts → x-pack/plugins/observability_solution/asset_manager/server/lib/entities/helpers/generate_index_name.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { OAMDefinition } from '@kbn/oam-schema';
-import { OAM_BASE_PREFIX } from '../../../../common/constants_oam';
+import { EntityDefinition } from '@kbn/oam-schema';
+import { ENTITY_BASE_PREFIX } from '../../../../common/constants_entities';
 
-export function generateIndexName(definition: OAMDefinition) {
-  return `${OAM_BASE_PREFIX}.${definition.id}`;
+export function generateIndexName(definition: EntityDefinition) {
+  return `${ENTITY_BASE_PREFIX}.${definition.id}`;
 }