Revert "[EventLog] change to use Data stream lifecycle instead of ILM" (

#164614)

Resolves #164581.

This reverts commit 7e234b1

The commit came from PR #163210

docs/user/production-considerations/alerting-production-considerations.asciidoc

@@ -56,10 +56,14 @@ Predicting the buffer required to account for actions depends heavily on the rul
 
 experimental[]
 
-Alerts and actions log activity in a set of "event log" data streams, one per Kibana version, named `.kibana-event-log-{VERSION}`.  These data streams are configured with a lifecycle data retention of 90 days. This can be updated to other values via the standard data stream lifecycle APIs.  Note that the event log data contains the data shown in the alerting pages in {kib}, so reducing the data retention period will result in less data being available to view.
+Alerts and actions log activity in a set of "event log" indices.  These indices are configured with an index lifecycle management (ILM) policy, which you can customize.  The default policy rolls over the index when it reaches 50GB, or after 30 days.  Indices over 90 days old are deleted.
 
-For more information on data stream lifecycle management, see:
-{ref}/data-stream-lifecycle.html[Data stream lifecycle].
+The name of the index policy is `kibana-event-log-policy`.  {kib} creates the index policy on startup, if it doesn't already exist.  The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it.
+
+Because {kib} uses the documents to display historic data, you should set the delete phase longer than you would like the historic data to be shown. For example, if you would like to see one month's worth of historic data, you should set the delete phase to at least one month.
+
+For more information on index lifecycle management, see:
+{ref}/index-lifecycle-management.html[Index Lifecycle Policies].
 
 [float]
 [[alerting-circuit-breakers]]

x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts

@@ -11,6 +11,8 @@ const createClusterClientMock = () => {
   const mock: jest.Mocked<IClusterClientAdapter> = {
     indexDocument: jest.fn(),
     indexDocuments: jest.fn(),
+    doesIlmPolicyExist: jest.fn(),
+    createIlmPolicy: jest.fn(),
     doesIndexTemplateExist: jest.fn(),
     createIndexTemplate: jest.fn(),
     doesDataStreamExist: jest.fn(),

x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts

@@ -165,6 +165,56 @@ describe('buffering documents', () => {
   });
 });
 
+describe('doesIlmPolicyExist', () => {
+  // ElasticsearchError can be a bit random in shape, we need an any here
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const notFoundError = new Error('Not found') as any;
+  notFoundError.statusCode = 404;
+
+  test('should call cluster with proper arguments', async () => {
+    await clusterClientAdapter.doesIlmPolicyExist('foo');
+    expect(clusterClient.transport.request).toHaveBeenCalledWith({
+      method: 'GET',
+      path: '/_ilm/policy/foo',
+    });
+  });
+
+  test('should return false when 404 error is returned by Elasticsearch', async () => {
+    clusterClient.transport.request.mockRejectedValue(notFoundError);
+    await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(false);
+  });
+
+  test('should throw error when error is not 404', async () => {
+    clusterClient.transport.request.mockRejectedValue(new Error('Fail'));
+    await expect(
+      clusterClientAdapter.doesIlmPolicyExist('foo')
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`"error checking existance of ilm policy: Fail"`);
+  });
+
+  test('should return true when no error is thrown', async () => {
+    await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(true);
+  });
+});
+
+describe('createIlmPolicy', () => {
+  test('should call cluster client with given policy', async () => {
+    clusterClient.transport.request.mockResolvedValue({ success: true });
+    await clusterClientAdapter.createIlmPolicy('foo', { args: true });
+    expect(clusterClient.transport.request).toHaveBeenCalledWith({
+      method: 'PUT',
+      path: '/_ilm/policy/foo',
+      body: { args: true },
+    });
+  });
+
+  test('should throw error when call cluster client throws', async () => {
+    clusterClient.transport.request.mockRejectedValue(new Error('Fail'));
+    await expect(
+      clusterClientAdapter.createIlmPolicy('foo', { args: true })
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`"error creating ilm policy: Fail"`);
+  });
+});
+
 describe('doesIndexTemplateExist', () => {
   test('should call cluster with proper arguments', async () => {
     await clusterClientAdapter.doesIndexTemplateExist('foo');

x-pack/plugins/event_log/server/es/cluster_client_adapter.ts

@@ -178,6 +178,36 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
     }
   }
 
+  public async doesIlmPolicyExist(policyName: string): Promise<boolean> {
+    const request = {
+      method: 'GET',
+      path: `/_ilm/policy/${policyName}`,
+    };
+    try {
+      const esClient = await this.elasticsearchClientPromise;
+      await esClient.transport.request(request);
+    } catch (err) {
+      if (err.statusCode === 404) return false;
+      throw new Error(`error checking existance of ilm policy: ${err.message}`);
+    }
+    return true;
+  }
+
+  public async createIlmPolicy(policyName: string, policy: Record<string, unknown>): Promise<void> {
+    this.logger.info(`Installing ILM policy ${policyName}`);
+    const request = {
+      method: 'PUT',
+      path: `/_ilm/policy/${policyName}`,
+      body: policy,
+    };
+    try {
+      const esClient = await this.elasticsearchClientPromise;
+      await esClient.transport.request(request);
+    } catch (err) {
+      throw new Error(`error creating ilm policy: ${err.message}`);
+    }
+  }
+
   public async doesIndexTemplateExist(name: string): Promise<boolean> {
     try {
       const esClient = await this.elasticsearchClientPromise;

x-pack/plugins/event_log/server/es/context.test.ts

@@ -57,12 +57,13 @@ describe('createEsContext', () => {
     expect(esNames).toStrictEqual({
       base: 'test-index',
       dataStream: 'test-index-event-log-1.2.3',
+      ilmPolicy: 'test-index-event-log-policy',
       indexPattern: 'test-index-event-log-*',
       indexTemplate: 'test-index-event-log-1.2.3-template',
     });
   });
 
-  test('should return exist false for esAdapter index template and data stream before initialize', async () => {
+  test('should return exist false for esAdapter ilm policy, index template and data stream before initialize', async () => {
     const context = createEsContext({
       logger,
       indexNameRoot: 'test1',
@@ -83,7 +84,7 @@ describe('createEsContext', () => {
     expect(doesIndexTemplateExist).toBeFalsy();
   });
 
-  test('should return exist true for esAdapter index template and data stream after initialize', async () => {
+  test('should return exist true for esAdapter ilm policy, index template and data stream after initialize', async () => {
     const context = createEsContext({
       logger,
       indexNameRoot: 'test2',
@@ -93,6 +94,11 @@ describe('createEsContext', () => {
     elasticsearchClient.indices.existsTemplate.mockResponse(true);
     context.initialize();
 
+    const doesIlmPolicyExist = await context.esAdapter.doesIlmPolicyExist(
+      context.esNames.ilmPolicy
+    );
+    expect(doesIlmPolicyExist).toBeTruthy();
+
     elasticsearchClient.indices.getDataStream.mockResolvedValue(GetDataStreamsResponse);
     const doesDataStreamExist = await context.esAdapter.doesDataStreamExist(
       context.esNames.dataStream

x-pack/plugins/event_log/server/es/documents.test.ts

@@ -5,9 +5,19 @@
  * 2.0.
  */
 
-import { getIndexTemplate } from './documents';
+import { getIndexTemplate, getIlmPolicy } from './documents';
 import { getEsNames } from './names';
 
+describe('getIlmPolicy()', () => {
+  test('returns the basic structure of an ilm policy', () => {
+    expect(getIlmPolicy()).toMatchObject({
+      policy: {
+        phases: {},
+      },
+    });
+  });
+});
+
 describe('getIndexTemplate()', () => {
   const kibanaVersion = '1.2.3';
   const esNames = getEsNames('XYZ', kibanaVersion);
@@ -17,6 +27,7 @@ describe('getIndexTemplate()', () => {
     expect(indexTemplate.index_patterns).toEqual([esNames.dataStream]);
     expect(indexTemplate.template.settings.number_of_shards).toBeGreaterThanOrEqual(0);
     expect(indexTemplate.template.settings.auto_expand_replicas).toBe('0-1');
+    expect(indexTemplate.template.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy);
     expect(indexTemplate.template.mappings).toMatchObject({});
   });
 });

x-pack/plugins/event_log/server/es/documents.ts

@@ -25,13 +25,41 @@ export function getIndexTemplate(esNames: EsNames) {
         hidden: true,
         number_of_shards: 1,
         auto_expand_replicas: '0-1',
-      },
-      lifecycle: {
-        data_retention: '90d',
+        'index.lifecycle.name': esNames.ilmPolicy,
       },
       mappings,
     },
   };
 
   return indexTemplateBody;
 }
+
+// returns the body of an ilm policy used in an ES PUT _ilm/policy call
+export function getIlmPolicy() {
+  return {
+    policy: {
+      _meta: {
+        description:
+          'ilm policy the Kibana event log, created initially by Kibana, but updated by the user, not Kibana',
+        managed: false,
+      },
+      phases: {
+        hot: {
+          actions: {
+            rollover: {
+              max_size: '50GB',
+              max_age: '30d',
+              // max_docs: 1, // you know, for testing
+            },
+          },
+        },
+        delete: {
+          min_age: '90d',
+          actions: {
+            delete: {},
+          },
+        },
+      },
+    },
+  };
+}

x-pack/plugins/event_log/server/es/init.test.ts

@@ -83,6 +83,7 @@ describe('initializeEs', () => {
       `error getting existing index templates - Fail`
     );
     expect(esContext.esAdapter.setLegacyIndexTemplateToHidden).not.toHaveBeenCalled();
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
   });
 
   test(`should continue initialization if updating existing index templates throws an error`, async () => {
@@ -123,6 +124,7 @@ describe('initializeEs', () => {
     expect(esContext.logger.error).toHaveBeenCalledWith(
       `error setting existing \"foo-bar-template\" index template to hidden - Fail`
     );
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
   });
 
   test(`should update existing index settings if any exist and are not hidden`, async () => {
@@ -205,6 +207,7 @@ describe('initializeEs', () => {
     expect(esContext.esAdapter.getExistingIndices).toHaveBeenCalled();
     expect(esContext.logger.error).toHaveBeenCalledWith(`error getting existing indices - Fail`);
     expect(esContext.esAdapter.setIndexToHidden).not.toHaveBeenCalled();
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
   });
 
   test(`should continue initialization if updating existing index settings throws an error`, async () => {
@@ -248,6 +251,7 @@ describe('initializeEs', () => {
     expect(esContext.logger.error).toHaveBeenCalledWith(
       `error setting existing \"foo-bar-000001\" index to hidden - Fail`
     );
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
   });
 
   test(`should update existing index aliases if any exist and are not hidden`, async () => {
@@ -296,6 +300,7 @@ describe('initializeEs', () => {
       `error getting existing index aliases - Fail`
     );
     expect(esContext.esAdapter.setIndexAliasToHidden).not.toHaveBeenCalled();
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
   });
 
   test(`should continue initialization if updating existing index aliases throws an error`, async () => {
@@ -331,6 +336,23 @@ describe('initializeEs', () => {
     expect(esContext.logger.error).toHaveBeenCalledWith(
       `error setting existing \"foo-bar\" index aliases - Fail`
     );
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
+  });
+
+  test(`should create ILM policy if it doesn't exist`, async () => {
+    esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(false);
+
+    await initializeEs(esContext);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
+    expect(esContext.esAdapter.createIlmPolicy).toHaveBeenCalled();
+  });
+
+  test(`shouldn't create ILM policy if it exists`, async () => {
+    esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true);
+
+    await initializeEs(esContext);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
+    expect(esContext.esAdapter.createIlmPolicy).not.toHaveBeenCalled();
   });
 
   test(`should create index template if it doesn't exist`, async () => {
@@ -441,10 +463,30 @@ describe('retries', () => {
     esContext.esAdapter.getExistingLegacyIndexTemplates.mockResolvedValue({});
     esContext.esAdapter.getExistingIndices.mockResolvedValue({});
     esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({});
+    esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true);
     esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true);
     esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true);
   });
 
+  test('createIlmPolicyIfNotExists with 1 retry', async () => {
+    esContext.esAdapter.doesIlmPolicyExist.mockRejectedValueOnce(new Error('retry 1'));
+
+    const timeStart = performance.now();
+    await initializeEs(esContext);
+    const timeElapsed = Math.ceil(performance.now() - timeStart);
+
+    expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY);
+
+    expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(2);
+    expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1);
+
+    const prefix = `eventLog initialization operation failed and will be retried: createIlmPolicyIfNotExists`;
+    expect(esContext.logger.warn).toHaveBeenCalledTimes(1);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 1`);
+  });
+
   test('createIndexTemplateIfNotExists with 2 retries', async () => {
     esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2a'));
     esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2b'));
@@ -456,6 +498,7 @@ describe('retries', () => {
     expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2));
 
     expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
     expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(3);
     expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1);
 
@@ -481,6 +524,7 @@ describe('retries', () => {
     expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2 + 4 + 8));
 
     expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
     expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
     expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(5);
     expect(esContext.esAdapter.createDataStream).toHaveBeenCalledTimes(0);

x-pack/plugins/event_log/server/es/init.ts

@@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { asyncForEach } from '@kbn/std';
 import { groupBy } from 'lodash';
 import pRetry, { FailedAttemptError } from 'p-retry';
-import { getIndexTemplate } from './documents';
+import { getIlmPolicy, getIndexTemplate } from './documents';
 import { EsContext } from './context';
 
 const MAX_RETRY_DELAY = 30000;
@@ -33,6 +33,7 @@ async function initializeEsResources(esContext: EsContext) {
 
   // today, setExistingAssetsToHidden() never throws, but just in case ...
   await retry(steps.setExistingAssetsToHidden);
+  await retry(steps.createIlmPolicyIfNotExists);
   await retry(steps.createIndexTemplateIfNotExists);
   await retry(steps.createDataStreamIfNotExists);
 
@@ -201,6 +202,18 @@ class EsInitializationSteps {
     await this.setExistingIndexAliasesToHidden();
   }
 
+  async createIlmPolicyIfNotExists(): Promise<void> {
+    const exists = await this.esContext.esAdapter.doesIlmPolicyExist(
+      this.esContext.esNames.ilmPolicy
+    );
+    if (!exists) {
+      await this.esContext.esAdapter.createIlmPolicy(
+        this.esContext.esNames.ilmPolicy,
+        getIlmPolicy()
+      );
+    }
+  }
+
   async createIndexTemplateIfNotExists(): Promise<void> {
     const exists = await this.esContext.esAdapter.doesIndexTemplateExist(
       this.esContext.esNames.indexTemplate

x-pack/plugins/event_log/server/es/names.mock.ts

@@ -11,6 +11,7 @@ const createNamesMock = () => {
   const mock: jest.Mocked<EsNames> = {
     base: '.kibana',
     dataStream: '.kibana-event-log-8.0.0',
+    ilmPolicy: 'kibana-event-log-policy',
     indexPattern: '.kibana-event-log-*',
     indexTemplate: '.kibana-event-log-8.0.0-template',
   };