[Security Solution][Detections] Refactor ML calls for newest ML permi…

…ssions (#74582) (#74919)

## Summary

Addresses #73567.

ML Users (role: `machine_learning_user`) were previously able to invoke the ML Recognizer API, which we use to get not-yet-installed ML Jobs relevant to our index patterns. As of #64662 this is not true, and so we receive errors from components using the underlying hook, `useSiemJobs`.

To solve this I've created two separate hooks to replace `useSiemJobs`:

* `useSecurityJobs`
  * used on ML Popover
  * includes uninstalled ML Jobs
  * checks (and returns) `isMlAdmin` before fetching data
* `useInstalledSecurityJobs`
  * used on ML Jobs Dropdown and Anomalies Table
  * includes only installed ML Jobs
  * checks (and returns) `isMlUser` before fetching data

Note that we while we now receive the knowledge to do so, we do not always inform the user in the case of invalid permissions, and instead have the following behaviors:

#### User has insufficient license
* ML Popover:  shows an upgrade CTA
* Anomalies Tables: show no data
* Rule Creation: ML Rule option is disabled, shows upgrade CTA
* Rule Details: ML Job Id is displayed as text
#### User is ML User
* ML Popover:  not shown
* Anomalies Tables: show no data
* Rule Creation: ML Rule option is disabled
* Rule Details: ML Job Id is displayed as text
#### User is ML Admin
* ML Popover:  shown
* Anomalies Tables: show data __for installed ML Jobs__
  * This is the same as previous logic, but worth calling out that you can't view historical anomalies
* Rule Creation: ML Rule option is enabled, all ML Jobs available
* Rule Details: ML Job Id is displayed as hyperlink, job status badge shown

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

### For maintainers

- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

Co-authored-by: Ryland Herrick <ryalnd@gmail.com>

x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts

@@ -36,8 +36,8 @@ export interface MlSummaryJob {
 export interface AuditMessage {
   job_id: string;
   msgTime: number;
-  level: number;
-  highestLevel: number;
+  level: string;
+  highestLevel: string;
   highestLevelText: string;
   text: string;
 }

x-pack/plugins/ml/public/shared.ts

@@ -9,6 +9,7 @@ export * from '../common/constants/anomalies';
 export * from '../common/types/data_recognizer';
 export * from '../common/types/capabilities';
 export * from '../common/types/anomalies';
+export * from '../common/types/anomaly_detection_jobs';
 export * from '../common/types/modules';
 export * from '../common/types/audit_message';
 

x-pack/plugins/security_solution/common/machine_learning/has_ml_license.test.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { emptyMlCapabilities } from './empty_ml_capabilities';
+import { hasMlLicense } from './has_ml_license';
+
+describe('hasMlLicense', () => {
+  test('it returns false when license is not platinum or trial', () => {
+    const capabilities = { ...emptyMlCapabilities, isPlatinumOrTrialLicense: false };
+    expect(hasMlLicense(capabilities)).toEqual(false);
+  });
+
+  test('it returns true when license is platinum or trial', () => {
+    const capabilities = { ...emptyMlCapabilities, isPlatinumOrTrialLicense: true };
+    expect(hasMlLicense(capabilities)).toEqual(true);
+  });
+});

x-pack/plugins/security_solution/common/machine_learning/has_ml_license.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { MlCapabilitiesResponse } from '../../../ml/common/types/capabilities';
+
+export const hasMlLicense = (capabilities: MlCapabilitiesResponse): boolean =>
+  capabilities.isPlatinumOrTrialLicense;

x-pack/plugins/security_solution/common/machine_learning/is_security_job.ts

@@ -4,8 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { MlSummaryJob } from '../../../ml/common/types/anomaly_detection_jobs';
 import { ML_GROUP_IDS } from '../constants';
 
-export const isSecurityJob = (job: MlSummaryJob): boolean =>
+export const isSecurityJob = (job: { groups: string[] }): boolean =>
   job.groups.some((group) => ML_GROUP_IDS.includes(group));

x-pack/plugins/security_solution/public/common/components/ml/anomaly/use_anomalies_table_data.ts

@@ -9,13 +9,11 @@ import { useState, useEffect, useMemo } from 'react';
 import { DEFAULT_ANOMALY_SCORE } from '../../../../../common/constants';
 import { anomaliesTableData } from '../api/anomalies_table_data';
 import { InfluencerInput, Anomalies, CriteriaFields } from '../types';
-import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions';
-import { useSiemJobs } from '../../ml_popover/hooks/use_siem_jobs';
-import { useMlCapabilities } from '../../ml_popover/hooks/use_ml_capabilities';
-import { useStateToaster, errorToToaster } from '../../toasters';
 
 import * as i18n from './translations';
 import { useTimeZone, useUiSetting$ } from '../../../lib/kibana';
+import { useAppToasts } from '../../../hooks/use_app_toasts';
+import { useInstalledSecurityJobs } from '../hooks/use_installed_security_jobs';
 
 interface Args {
   influencers?: InfluencerInput[];
@@ -58,15 +56,13 @@ export const useAnomaliesTableData = ({
   skip = false,
 }: Args): Return => {
   const [tableData, setTableData] = useState<Anomalies | null>(null);
-  const [, siemJobs] = useSiemJobs(true);
+  const { isMlUser, jobs } = useInstalledSecurityJobs();
   const [loading, setLoading] = useState(true);
-  const capabilities = useMlCapabilities();
-  const userPermissions = hasMlUserPermissions(capabilities);
-  const [, dispatchToaster] = useStateToaster();
+  const { addError } = useAppToasts();
   const timeZone = useTimeZone();
   const [anomalyScore] = useUiSetting$<number>(DEFAULT_ANOMALY_SCORE);
 
-  const siemJobIds = siemJobs.filter((job) => job.isInstalled).map((job) => job.id);
+  const jobIds = jobs.map((job) => job.id);
   const startDateMs = useMemo(() => new Date(startDate).getTime(), [startDate]);
   const endDateMs = useMemo(() => new Date(endDate).getTime(), [endDate]);
 
@@ -81,11 +77,11 @@ export const useAnomaliesTableData = ({
       earliestMs: number,
       latestMs: number
     ) {
-      if (userPermissions && !skip && siemJobIds.length > 0) {
+      if (isMlUser && !skip && jobIds.length > 0) {
         try {
           const data = await anomaliesTableData(
             {
-              jobIds: siemJobIds,
+              jobIds,
               criteriaFields: criteriaFieldsInput,
               aggregationInterval: 'auto',
               threshold: getThreshold(anomalyScore, threshold),
@@ -104,13 +100,13 @@ export const useAnomaliesTableData = ({
           }
         } catch (error) {
           if (isSubscribed) {
-            errorToToaster({ title: i18n.SIEM_TABLE_FETCH_FAILURE, error, dispatchToaster });
+            addError(error, { title: i18n.SIEM_TABLE_FETCH_FAILURE });
             setLoading(false);
           }
         }
-      } else if (!userPermissions && isSubscribed) {
+      } else if (!isMlUser && isSubscribed) {
         setLoading(false);
-      } else if (siemJobIds.length === 0 && isSubscribed) {
+      } else if (jobIds.length === 0 && isSubscribed) {
         setLoading(false);
       } else if (isSubscribed) {
         setTableData(null);
@@ -132,9 +128,9 @@ export const useAnomaliesTableData = ({
     startDateMs,
     endDateMs,
     skip,
-    userPermissions,
+    isMlUser,
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    siemJobIds.sort().join(),
+    jobIds.sort().join(),
   ]);
 
   return [loading, tableData];

x-pack/plugins/security_solution/public/common/components/ml/api/get_jobs_summary.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { HttpSetup } from '../../../../../../../../src/core/public';
+import { MlSummaryJob } from '../../../../../../ml/public';
+
+export interface GetJobsSummaryArgs {
+  http: HttpSetup;
+  jobIds?: string[];
+  signal: AbortSignal;
+}
+
+/**
+ * Fetches a summary of all ML jobs currently installed
+ *
+ * @param http HTTP Service
+ * @param jobIds Array of job IDs to filter against
+ * @param signal to cancel request
+ *
+ * @throws An error if response is not OK
+ */
+export const getJobsSummary = async ({
+  http,
+  jobIds,
+  signal,
+}: GetJobsSummaryArgs): Promise<MlSummaryJob[]> =>
+  http.fetch<MlSummaryJob[]>('/api/ml/jobs/jobs_summary', {
+    method: 'POST',
+    body: JSON.stringify({ jobIds: jobIds ?? [] }),
+    asSystemRequest: true,
+    signal,
+  });

x-pack/plugins/security_solution/public/common/components/ml/api/get_ml_capabilities.ts

@@ -4,8 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { HttpSetup } from '../../../../../../../../src/core/public';
 import { MlCapabilitiesResponse } from '../../../../../../ml/public';
-import { KibanaServices } from '../../../lib/kibana';
 import { InfluencerInput } from '../types';
 
 export interface Body {
@@ -21,10 +21,15 @@ export interface Body {
   maxExamples: number;
 }
 
-export const getMlCapabilities = async (signal: AbortSignal): Promise<MlCapabilitiesResponse> => {
-  return KibanaServices.get().http.fetch<MlCapabilitiesResponse>('/api/ml/ml_capabilities', {
+export const getMlCapabilities = async ({
+  http,
+  signal,
+}: {
+  http: HttpSetup;
+  signal: AbortSignal;
+}): Promise<MlCapabilitiesResponse> =>
+  http.fetch<MlCapabilitiesResponse>('/api/ml/ml_capabilities', {
     method: 'GET',
     asSystemRequest: true,
     signal,
   });
-};

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_jobs_summary.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useAsync, withOptionalSignal } from '../../../../shared_imports';
+import { getJobsSummary } from '../api/get_jobs_summary';
+
+const _getJobsSummary = withOptionalSignal(getJobsSummary);
+
+export const useGetJobsSummary = () => useAsync(_getJobsSummary);

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_ml_capabilities.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getMlCapabilities } from '../api/get_ml_capabilities';
+import { useAsync, withOptionalSignal } from '../../../../shared_imports';
+
+const _getMlCapabilities = withOptionalSignal(getMlCapabilities);
+
+export const useGetMlCapabilities = () => useAsync(_getMlCapabilities);

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { renderHook } from '@testing-library/react-hooks';
+
+import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions';
+import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license';
+import { isSecurityJob } from '../../../../../common/machine_learning/is_security_job';
+import { useAppToasts } from '../../../hooks/use_app_toasts';
+import { useAppToastsMock } from '../../../hooks/use_app_toasts.mock';
+import { mockJobsSummaryResponse } from '../../ml_popover/api.mock';
+import { getJobsSummary } from '../api/get_jobs_summary';
+import { useInstalledSecurityJobs } from './use_installed_security_jobs';
+
+jest.mock('../../../../../common/machine_learning/has_ml_user_permissions');
+jest.mock('../../../../../common/machine_learning/has_ml_license');
+jest.mock('../../../hooks/use_app_toasts');
+jest.mock('../api/get_jobs_summary');
+
+describe('useInstalledSecurityJobs', () => {
+  let appToastsMock: jest.Mocked<ReturnType<typeof useAppToastsMock.create>>;
+
+  beforeEach(() => {
+    appToastsMock = useAppToastsMock.create();
+    (useAppToasts as jest.Mock).mockReturnValue(appToastsMock);
+    (getJobsSummary as jest.Mock).mockResolvedValue(mockJobsSummaryResponse);
+  });
+
+  describe('when the user has permissions', () => {
+    beforeEach(() => {
+      (hasMlUserPermissions as jest.Mock).mockReturnValue(true);
+      (hasMlLicense as jest.Mock).mockReturnValue(true);
+    });
+
+    it('returns jobs and permissions', async () => {
+      const { result, waitForNextUpdate } = renderHook(() => useInstalledSecurityJobs());
+      await waitForNextUpdate();
+
+      expect(result.current.jobs).toHaveLength(3);
+      expect(result.current.jobs).toEqual(
+        expect.arrayContaining([
+          {
+            datafeedId: 'datafeed-siem-api-rare_process_linux_ecs',
+            datafeedIndices: ['auditbeat-*'],
+            datafeedState: 'stopped',
+            description: 'SIEM Auditbeat: Detect unusually rare processes on Linux (beta)',
+            earliestTimestampMs: 1557353420495,
+            groups: ['siem'],
+            hasDatafeed: true,
+            id: 'siem-api-rare_process_linux_ecs',
+            isSingleMetricViewerJob: true,
+            jobState: 'closed',
+            latestTimestampMs: 1557434782207,
+            memory_status: 'hard_limit',
+            processed_record_count: 582251,
+          },
+        ])
+      );
+      expect(result.current.isMlUser).toEqual(true);
+      expect(result.current.isLicensed).toEqual(true);
+    });
+
+    it('filters out non-security jobs', async () => {
+      const { result, waitForNextUpdate } = renderHook(() => useInstalledSecurityJobs());
+      await waitForNextUpdate();
+
+      expect(result.current.jobs.length).toBeGreaterThan(0);
+      expect(result.current.jobs.every(isSecurityJob)).toEqual(true);
+    });
+
+    it('renders a toast error if the ML call fails', async () => {
+      (getJobsSummary as jest.Mock).mockRejectedValue('whoops');
+      const { waitForNextUpdate } = renderHook(() => useInstalledSecurityJobs());
+      await waitForNextUpdate();
+
+      expect(appToastsMock.addError).toHaveBeenCalledWith('whoops', {
+        title: 'Security job fetch failure',
+      });
+    });
+  });
+
+  describe('when the user does not have valid permissions', () => {
+    beforeEach(() => {
+      (hasMlUserPermissions as jest.Mock).mockReturnValue(false);
+      (hasMlLicense as jest.Mock).mockReturnValue(false);
+    });
+
+    it('returns empty jobs and false predicates', () => {
+      const { result } = renderHook(() => useInstalledSecurityJobs());
+
+      expect(result.current.jobs).toEqual([]);
+      expect(result.current.isMlUser).toEqual(false);
+      expect(result.current.isLicensed).toEqual(false);
+    });
+  });
+});

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useEffect, useState } from 'react';
+
+import { MlSummaryJob } from '../../../../../../ml/public';
+import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions';
+import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license';
+import { isSecurityJob } from '../../../../../common/machine_learning/is_security_job';
+import { useAppToasts } from '../../../hooks/use_app_toasts';
+import { useHttp } from '../../../lib/kibana';
+import { useMlCapabilities } from './use_ml_capabilities';
+import * as i18n from '../translations';
+import { useGetJobsSummary } from './use_get_jobs_summary';
+
+export interface UseInstalledSecurityJobsReturn {
+  loading: boolean;
+  jobs: MlSummaryJob[];
+  isMlUser: boolean;
+  isLicensed: boolean;
+}
+
+/**
+ * Returns a collection of installed ML jobs (MlSummaryJob) relevant to
+ * Security Solution, i.e. all installed jobs in the `security` ML group.
+ * Use the corresponding helper functions to filter the job list as
+ * necessary (running jobs, etc).
+ *
+ */
+export const useInstalledSecurityJobs = (): UseInstalledSecurityJobsReturn => {
+  const [jobs, setJobs] = useState<MlSummaryJob[]>([]);
+  const { addError } = useAppToasts();
+  const mlCapabilities = useMlCapabilities();
+  const http = useHttp();
+  const { error, loading, result, start } = useGetJobsSummary();
+
+  const isMlUser = hasMlUserPermissions(mlCapabilities);
+  const isLicensed = hasMlLicense(mlCapabilities);
+
+  useEffect(() => {
+    if (isMlUser && isLicensed) {
+      start({ http });
+    }
+  }, [http, isMlUser, isLicensed, start]);
+
+  useEffect(() => {
+    if (result) {
+      const securityJobs = result.filter(isSecurityJob);
+      setJobs(securityJobs);
+    }
+  }, [result]);
+
+  useEffect(() => {
+    if (error) {
+      addError(error, { title: i18n.SIEM_JOB_FETCH_FAILURE });
+    }
+  }, [addError, error]);
+
+  return { isLicensed, isMlUser, jobs, loading };
+};