Strip field from events for event search (#149113)

Strip source, and keep only fields used for mappings for events search
of IM rule.

Tests on cloud env:
<img width="1182" alt="Screenshot 2023-01-19 at 16 42 19"
src="https://user-images.githubusercontent.com/7609147/213486861-1a0e5a20-b575-4ed6-ac9b-cf9fb8a8c14f.png">

Threat Indicators - 1.500.000 documents
Source - 1.000.000 documents.

No changes:
<img width="564" alt="Screenshot 2023-01-19 at 16 30 19"
src="https://user-images.githubusercontent.com/7609147/213484465-ad20d5b2-c7a7-42d3-9d92-478c5d046636.png">

Strip fields:

<img width="557" alt="Screenshot 2023-01-19 at 16 30 27"
src="https://user-images.githubusercontent.com/7609147/213484531-3ab68c61-c3f5-4e28-b2c4-c1e90a5b1775.png">

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts

@@ -534,6 +534,61 @@ describe('create_signals', () => {
     });
   });
 
+  test('it respects overriderBody params', () => {
+    const query = buildEventsSearchQuery({
+      index: ['auditbeat-*'],
+      from: 'now-5m',
+      to: 'today',
+      filter: {},
+      size: 100,
+      searchAfterSortIds: undefined,
+      primaryTimestamp: '@timestamp',
+      secondaryTimestamp: undefined,
+      runtimeMappings: undefined,
+      overrideBody: {
+        _source: false,
+        fields: ['@timestamp'],
+      },
+    });
+    expect(query).toEqual({
+      allow_no_indices: true,
+      index: ['auditbeat-*'],
+      size: 100,
+      runtime_mappings: undefined,
+      track_total_hits: undefined,
+      ignore_unavailable: true,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {},
+              {
+                range: {
+                  '@timestamp': {
+                    gte: 'now-5m',
+                    lte: 'today',
+                    format: 'strict_date_optional_time',
+                  },
+                },
+              },
+            ],
+          },
+        },
+        _source: false,
+        fields: ['@timestamp'],
+        runtime_mappings: undefined,
+        sort: [
+          {
+            '@timestamp': {
+              order: 'asc',
+              unmapped_type: 'date',
+            },
+          },
+        ],
+      },
+    });
+  });
+
   describe('buildEqlSearchRequest', () => {
     test('should build a basic request with time range', () => {
       const request = buildEqlSearchRequest({

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts

@@ -7,6 +7,7 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { isEmpty } from 'lodash';
 import type { Filter } from '@kbn/es-query';
+import type { OverrideBodyQuery } from './types';
 import type {
   RuleFilterArray,
   TimestampOverride,
@@ -27,6 +28,7 @@ interface BuildEventsSearchQuery {
   secondaryTimestamp: TimestampOverride | undefined;
   trackTotalHits?: boolean;
   additionalFilters?: estypes.QueryDslQueryContainer[];
+  overrideBody?: OverrideBodyQuery;
 }
 
 interface BuildEqlSearchRequestParams {
@@ -132,6 +134,7 @@ export const buildEventsSearchQuery = ({
   secondaryTimestamp,
   trackTotalHits,
   additionalFilters,
+  overrideBody,
 }: BuildEventsSearchQuery) => {
   const timestamps = secondaryTimestamp
     ? [primaryTimestamp, secondaryTimestamp]
@@ -193,6 +196,7 @@ export const buildEventsSearchQuery = ({
       ...(aggregations ? { aggregations } : {}),
       runtime_mappings: runtimeMappings,
       sort,
+      ...overrideBody,
     },
   };
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts

@@ -14,6 +14,10 @@ import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks
 import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
 import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
 import { ruleExecutionLogMock } from '../rule_monitoring/mocks';
+import { buildEventsSearchQuery } from './build_events_query';
+
+jest.mock('./build_events_query');
+const mockBuildEventsSearchQuery = buildEventsSearchQuery as jest.Mock;
 
 describe('singleSearchAfter', () => {
   const mockService: RuleExecutorServicesMock = alertsMock.createRuleExecutorServices();
@@ -157,4 +161,47 @@ describe('singleSearchAfter', () => {
       })
     ).rejects.toThrow('Fake Error');
   });
+
+  test('singleSearchAfter passes overrideBody to buildEventsSearchQuery', async () => {
+    mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
+      elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId())
+    );
+    await singleSearchAfter({
+      searchAfterSortIds: undefined,
+      index: [],
+      from: 'now-360s',
+      to: 'now',
+      services: mockService,
+      ruleExecutionLogger,
+      pageSize: 1,
+      filter: {},
+      primaryTimestamp: '@timestamp',
+      secondaryTimestamp: undefined,
+      runtimeMappings: undefined,
+      overrideBody: {
+        _source: false,
+        fields: ['@timestamp'],
+      },
+    });
+
+    expect(mockBuildEventsSearchQuery).toHaveBeenCalledWith({
+      additionalFilters: undefined,
+      aggregations: undefined,
+      filter: {},
+      from: 'now-360s',
+      index: [],
+      primaryTimestamp: '@timestamp',
+      runtimeMappings: undefined,
+      searchAfterSortIds: undefined,
+      secondaryTimestamp: undefined,
+      size: 1,
+      sortOrder: undefined,
+      to: 'now',
+      trackTotalHits: undefined,
+      overrideBody: {
+        _source: false,
+        fields: ['@timestamp'],
+      },
+    });
+  });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts

@@ -11,7 +11,7 @@ import type {
   AlertInstanceState,
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
-import type { SignalSearchResponse, SignalSource } from './types';
+import type { SignalSearchResponse, SignalSource, OverrideBodyQuery } from './types';
 import { buildEventsSearchQuery } from './build_events_query';
 import { createErrorsFromShard, makeFloatString } from './utils';
 import type { TimestampOverride } from '../../../../common/detection_engine/rule_schema';
@@ -34,6 +34,7 @@ interface SingleSearchAfterParams {
   trackTotalHits?: boolean;
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   additionalFilters?: estypes.QueryDslQueryContainer[];
+  overrideBody?: OverrideBodyQuery;
 }
 
 // utilize search_after for paging results into bulk.
@@ -55,6 +56,7 @@ export const singleSearchAfter = async <
   secondaryTimestamp,
   trackTotalHits,
   additionalFilters,
+  overrideBody,
 }: SingleSearchAfterParams): Promise<{
   searchResult: SignalSearchResponse<TAggregations>;
   searchDuration: string;
@@ -76,6 +78,11 @@ export const singleSearchAfter = async <
         secondaryTimestamp,
         trackTotalHits,
         additionalFilters,
+        /**
+         * overrideBody allows the search after to ignore the _source property of the result,
+         * thus reducing the size of the response and increasing the performance of the query.
+         */
+        overrideBody,
       });
 
       const start = performance.now();

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts

@@ -124,6 +124,11 @@ export const createThreatSignals = async ({
     _source: false,
   };
 
+  const eventListConfig = {
+    fields: threatMapping.map((mapping) => mapping.entries.map((item) => item.field)).flat(),
+    _source: false,
+  };
+
   const threatEnrichment = buildThreatEnrichment({
     ruleExecutionLogger,
     services,
@@ -197,6 +202,7 @@ export const createThreatSignals = async ({
           primaryTimestamp,
           secondaryTimestamp,
           exceptionFilter,
+          eventListConfig,
         }),
 
       createSignal: (slicedChunk) =>

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts

@@ -27,6 +27,7 @@ export const getEventList = async ({
   secondaryTimestamp,
   runtimeMappings,
   exceptionFilter,
+  eventListConfig,
 }: EventsOptions): Promise<estypes.SearchResponse<EventDoc>> => {
   const calculatedPerPage = perPage ?? MAX_PER_PAGE;
   if (calculatedPerPage > 10000) {
@@ -59,6 +60,7 @@ export const getEventList = async ({
     sortOrder: 'desc',
     trackTotalHits: false,
     runtimeMappings,
+    overrideBody: eventListConfig,
   });
 
   ruleExecutionLogger.debug(`Retrieved events items of size: ${searchResult.hits.hits.length}`);

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts

@@ -34,6 +34,7 @@ import type {
   SearchAfterAndBulkCreateReturnType,
   SignalsEnrichment,
   WrapHits,
+  OverrideBodyQuery,
 } from '../types';
 import type { CompleteRule, ThreatRuleParams } from '../../rule_schema';
 import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
@@ -265,6 +266,7 @@ export interface EventsOptions {
   tuple: RuleRangeTuple;
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   exceptionFilter: Filter | undefined;
+  eventListConfig?: OverrideBodyQuery;
 }
 
 export interface EventDoc {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -342,3 +342,9 @@ export type EventGroupingMultiBucketAggregationResult = ESSearchResponse<
     };
   }
 >;
+
+// the new fields can be added later if needed
+export interface OverrideBodyQuery {
+  _source?: estypes.SearchSourceConfig;
+  fields?: estypes.Fields;
+}