[Security Solution][Exceptions] Implement exceptions for ML rules (#8…

…4006) * Implement exceptions for ML rules * Remove unused import * Better implicit types * Retrieve ML rule index pattern for exception field suggestions and autocomplete * Add ML job logic to edit exception modal * Remove unnecessary logic change Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Dec 2, 2020 · d47c70c · d47c70c
1 parent 4f3d72b
commit d47c70c
Show file tree

Hide file tree

Showing 17 changed files with 552 additions and 179 deletions.
diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
@@ -7,6 +7,7 @@
 import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
 import { Filter, EsQueryConfig } from 'src/plugins/data/public';
 import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
+import { ExceptionListItemSchema } from '../shared_imports';
 
 describe('get_filter', () => {
   describe('getQueryFilter', () => {
@@ -919,19 +920,27 @@ describe('get_filter', () => {
       dateFormatTZ: 'Zulu',
     };
     test('it should build a filter without chunking exception items', () => {
-      const exceptionFilter = buildExceptionFilter(
-        [
-          { language: 'kuery', query: 'host.name: linux and some.field: value' },
-          { language: 'kuery', query: 'user.name: name' },
+      const exceptionItem1: ExceptionListItemSchema = {
+        ...getExceptionListItemSchemaMock(),
+        entries: [
+          { field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
+          { field: 'some.field', operator: 'included', type: 'match', value: 'value' },
         ],
-        {
+      };
+      const exceptionItem2: ExceptionListItemSchema = {
+        ...getExceptionListItemSchemaMock(),
+        entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
+      };
+      const exceptionFilter = buildExceptionFilter({
+        lists: [exceptionItem1, exceptionItem2],
+        config,
+        excludeExceptions: true,
+        chunkSize: 2,
+        indexPattern: {
           fields: [],
           title: 'auditbeat-*',
         },
-        config,
-        true,
-        2
-      );
+      });
       expect(exceptionFilter).toEqual({
         meta: {
           alias: null,
@@ -949,7 +958,7 @@ describe('get_filter', () => {
                         minimum_should_match: 1,
                         should: [
                           {
-                            match: {
+                            match_phrase: {
                               'host.name': 'linux',
                             },
                           },
@@ -961,7 +970,7 @@ describe('get_filter', () => {
                         minimum_should_match: 1,
                         should: [
                           {
-                            match: {
+                            match_phrase: {
                               'some.field': 'value',
                             },
                           },
@@ -976,7 +985,7 @@ describe('get_filter', () => {
                   minimum_should_match: 1,
                   should: [
                     {
-                      match: {
+                      match_phrase: {
                         'user.name': 'name',
                       },
                     },
@@ -990,20 +999,31 @@ describe('get_filter', () => {
     });
 
     test('it should properly chunk exception items', () => {
-      const exceptionFilter = buildExceptionFilter(
-        [
-          { language: 'kuery', query: 'host.name: linux and some.field: value' },
-          { language: 'kuery', query: 'user.name: name' },
-          { language: 'kuery', query: 'file.path: /safe/path' },
+      const exceptionItem1: ExceptionListItemSchema = {
+        ...getExceptionListItemSchemaMock(),
+        entries: [
+          { field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
+          { field: 'some.field', operator: 'included', type: 'match', value: 'value' },
         ],
-        {
+      };
+      const exceptionItem2: ExceptionListItemSchema = {
+        ...getExceptionListItemSchemaMock(),
+        entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
+      };
+      const exceptionItem3: ExceptionListItemSchema = {
+        ...getExceptionListItemSchemaMock(),
+        entries: [{ field: 'file.path', operator: 'included', type: 'match', value: '/safe/path' }],
+      };
+      const exceptionFilter = buildExceptionFilter({
+        lists: [exceptionItem1, exceptionItem2, exceptionItem3],
+        config,
+        excludeExceptions: true,
+        chunkSize: 2,
+        indexPattern: {
           fields: [],
           title: 'auditbeat-*',
         },
-        config,
-        true,
-        2
-      );
+      });
       expect(exceptionFilter).toEqual({
         meta: {
           alias: null,
@@ -1024,7 +1044,7 @@ describe('get_filter', () => {
                               minimum_should_match: 1,
                               should: [
                                 {
-                                  match: {
+                                  match_phrase: {
                                     'host.name': 'linux',
                                   },
                                 },
@@ -1036,7 +1056,7 @@ describe('get_filter', () => {
                               minimum_should_match: 1,
                               should: [
                                 {
-                                  match: {
+                                  match_phrase: {
                                     'some.field': 'value',
                                   },
                                 },
@@ -1051,7 +1071,7 @@ describe('get_filter', () => {
                         minimum_should_match: 1,
                         should: [
                           {
-                            match: {
+                            match_phrase: {
                               'user.name': 'name',
                             },
                           },
@@ -1069,7 +1089,7 @@ describe('get_filter', () => {
                         minimum_should_match: 1,
                         should: [
                           {
-                            match: {
+                            match_phrase: {
                               'file.path': '/safe/path',
                             },
                           },

diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
@@ -6,7 +6,6 @@
 
 import {
   Filter,
-  Query,
   IIndexPattern,
   isFilterDisabled,
   buildEsQuery,
@@ -18,15 +17,10 @@ import {
 } from '../../../lists/common/schemas';
 import { ESBoolQuery } from '../typed_json';
 import { buildExceptionListQueries } from './build_exceptions_query';
-import {
-  Query as QueryString,
-  Language,
-  Index,
-  TimestampOverrideOrUndefined,
-} from './schemas/common/schemas';
+import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
 
 export const getQueryFilter = (
-  query: QueryString,
+  query: Query,
   language: Language,
   filters: Array<Partial<Filter>>,
   index: Index,
@@ -53,19 +47,18 @@ export const getQueryFilter = (
    * buildEsQuery, this allows us to offer nested queries
    * regardless
    */
-  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
-  if (exceptionQueries.length > 0) {
-    // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
-    // allowing us to make 1024-item chunks of exception list items.
-    // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
-    // very conservative value.
-    const exceptionFilter = buildExceptionFilter(
-      exceptionQueries,
-      indexPattern,
-      config,
-      excludeExceptions,
-      1024
-    );
+  // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
+  // allowing us to make 1024-item chunks of exception list items.
+  // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
+  // very conservative value.
+  const exceptionFilter = buildExceptionFilter({
+    lists,
+    config,
+    excludeExceptions,
+    chunkSize: 1024,
+    indexPattern,
+  });
+  if (exceptionFilter !== undefined) {
     enabledFilters.push(exceptionFilter);
   }
   const initialQuery = { query, language };
@@ -101,15 +94,17 @@ export const buildEqlSearchRequest = (
     ignoreFilterIfFieldNotInIndex: false,
     dateFormatTZ: 'Zulu',
   };
-  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
-  let exceptionFilter: Filter | undefined;
-  if (exceptionQueries.length > 0) {
-    // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
-    // allowing us to make 1024-item chunks of exception list items.
-    // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
-    // very conservative value.
-    exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
-  }
+  // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
+  // allowing us to make 1024-item chunks of exception list items.
+  // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
+  // very conservative value.
+  const exceptionFilter = buildExceptionFilter({
+    lists: exceptionLists,
+    config,
+    excludeExceptions: true,
+    chunkSize: 1024,
+    indexPattern,
+  });
   const indexString = index.join();
   const requestFilter: unknown[] = [
     {
@@ -154,13 +149,23 @@ export const buildEqlSearchRequest = (
   }
 };
 
-export const buildExceptionFilter = (
-  exceptionQueries: Query[],
-  indexPattern: IIndexPattern,
-  config: EsQueryConfig,
-  excludeExceptions: boolean,
-  chunkSize: number
-) => {
+export const buildExceptionFilter = ({
+  lists,
+  config,
+  excludeExceptions,
+  chunkSize,
+  indexPattern,
+}: {
+  lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
+  config: EsQueryConfig;
+  excludeExceptions: boolean;
+  chunkSize: number;
+  indexPattern?: IIndexPattern;
+}) => {
+  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
+  if (exceptionQueries.length === 0) {
+    return undefined;
+  }
   const exceptionFilter: Filter = {
     meta: {
       alias: null,

diff --git a/...ugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx b/...ugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx
@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+/* eslint complexity: ["error", 30]*/
+
 import React, { memo, useEffect, useState, useCallback, useMemo } from 'react';
 import styled, { css } from 'styled-components';
 import {
@@ -53,6 +55,7 @@ import {
 import { ErrorInfo, ErrorCallout } from '../error_callout';
 import { ExceptionsBuilderExceptionItem } from '../types';
 import { useFetchIndex } from '../../../containers/source';
+import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs';
 
 export interface AddExceptionModalProps {
   ruleName: string;
@@ -108,7 +111,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({
   const { http } = useKibana().services;
   const [errorsExist, setErrorExists] = useState(false);
   const [comment, setComment] = useState('');
-  const { rule: maybeRule } = useRuleAsync(ruleId);
+  const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId);
   const [shouldCloseAlert, setShouldCloseAlert] = useState(false);
   const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false);
   const [shouldDisableBulkClose, setShouldDisableBulkClose] = useState(false);
@@ -124,8 +127,22 @@ export const AddExceptionModal = memo(function AddExceptionModal({
   const [isSignalIndexPatternLoading, { indexPatterns: signalIndexPatterns }] = useFetchIndex(
     memoSignalIndexName
   );
-  const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices);
 
+  const memoMlJobIds = useMemo(
+    () => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []),
+    [maybeRule]
+  );
+  const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds);
+
+  const memoRuleIndices = useMemo(() => {
+    if (jobs.length > 0) {
+      return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : [];
+    } else {
+      return ruleIndices;
+    }
+  }, [jobs, ruleIndices]);
+
+  const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices);
   const onError = useCallback(
     (error: Error): void => {
       addError(error, { title: i18n.ADD_EXCEPTION_ERROR });
@@ -364,6 +381,8 @@ export const AddExceptionModal = memo(function AddExceptionModal({
           !isSignalIndexPatternLoading &&
           !isLoadingExceptionList &&
           !isIndexPatternLoading &&
+          !isRuleLoading &&
+          !mlJobLoading &&
           ruleExceptionList && (
             <>
               <ModalBodySection className="builder-section">