[ML] Inference models management (#74978)

* [ML] init tabs

* [ML] init inference API service in UI

* [ML] server-side routes

* [ML] basic table

* [ML] support deletion

* [ML] delete multiple models

* [ML] WIP expanded row

* [ML] fix types

* [ML] expanded row

* [ML] fix types

* [ML] fix i18n id

* [ML] change server-side permission check

* [ML] refactor types

* [ML] show success toast on model deletion, fix models counter

* [ML] update expanded row

* [ML] pipelines stats

* [ML] use refresh observable

* [ML] endpoint to fetch associated pipelines

* [ML] update the endpoint to fetch associated pipelines

* [ML] show pipelines definition in expanded row

* [ML] change stats layout

* [ML] fix headers

* [ML] change breadcrumb title

* [ML] fetch models config with pipelines

* [ML] change default size to 1000

* [ML] fix collections keys, fix double fetch on initial page load

* [ML] adjust models deletion text

* [ML] fix DFA jobs on the management page

* [ML] small tabs in expanded row

* [ML] fix headers text

* [ML] fix models fetching without pipelines get permissions

* [ML] stats rendering as a description list

* [ML] fix i18n id

* [ML] remove an extra copyright comment, add selectable messages

* [ML] update stats on refresh

x-pack/plugins/ml/common/types/data_frame_analytics.ts

@@ -5,7 +5,77 @@
  */
 
 import { CustomHttpResponseOptions, ResponseError } from 'kibana/server';
+
 export interface DeleteDataFrameAnalyticsWithIndexStatus {
   success: boolean;
   error?: CustomHttpResponseOptions<ResponseError>;
 }
+
+export type IndexName = string;
+export type DataFrameAnalyticsId = string;
+
+export interface OutlierAnalysis {
+  [key: string]: {};
+
+  outlier_detection: {};
+}
+
+interface Regression {
+  dependent_variable: string;
+  training_percent?: number;
+  num_top_feature_importance_values?: number;
+  prediction_field_name?: string;
+}
+
+interface Classification {
+  dependent_variable: string;
+  training_percent?: number;
+  num_top_classes?: string;
+  num_top_feature_importance_values?: number;
+  prediction_field_name?: string;
+}
+
+export interface RegressionAnalysis {
+  [key: string]: Regression;
+
+  regression: Regression;
+}
+
+export interface ClassificationAnalysis {
+  [key: string]: Classification;
+
+  classification: Classification;
+}
+
+interface GenericAnalysis {
+  [key: string]: Record<string, any>;
+}
+
+export type AnalysisConfig =
+  | OutlierAnalysis
+  | RegressionAnalysis
+  | ClassificationAnalysis
+  | GenericAnalysis;
+
+export interface DataFrameAnalyticsConfig {
+  id: DataFrameAnalyticsId;
+  description?: string;
+  dest: {
+    index: IndexName;
+    results_field: string;
+  };
+  source: {
+    index: IndexName | IndexName[];
+    query?: any;
+  };
+  analysis: AnalysisConfig;
+  analyzed_fields: {
+    includes: string[];
+    excludes: string[];
+  };
+  model_memory_limit: string;
+  max_num_threads?: number;
+  create_time: number;
+  version: string;
+  allow_lazy_start?: boolean;
+}

x-pack/plugins/ml/common/types/inference.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataFrameAnalyticsConfig } from './data_frame_analytics';
+
+export interface IngestStats {
+  count: number;
+  time_in_millis: number;
+  current: number;
+  failed: number;
+}
+
+export interface TrainedModelStat {
+  model_id: string;
+  pipeline_count: number;
+  inference_stats?: {
+    failure_count: number;
+    inference_count: number;
+    cache_miss_count: number;
+    missing_all_fields_count: number;
+    timestamp: number;
+  };
+  ingest?: {
+    total: IngestStats;
+    pipelines: Record<
+      string,
+      IngestStats & {
+        processors: Array<
+          Record<
+            string,
+            {
+              // TODO use type from ingest_pipelines plugin
+              type: string;
+              stats: IngestStats;
+            }
+          >
+        >;
+      }
+    >;
+  };
+}
+
+export interface ModelConfigResponse {
+  created_by: string;
+  create_time: string;
+  default_field_map: Record<string, string>;
+  estimated_heap_memory_usage_bytes: number;
+  estimated_operations: number;
+  license_level: string;
+  metadata?:
+    | {
+        analytics_config: DataFrameAnalyticsConfig;
+        input: any;
+      }
+    | Record<string, any>;
+  model_id: string;
+  tags: string;
+  version: string;
+  inference_config?: Record<string, any>;
+  pipelines?: Record<string, PipelineDefinition> | null;
+}
+
+export interface PipelineDefinition {
+  processors?: Array<Record<string, any>>;
+  description?: string;
+}
+
+export interface ModelPipelines {
+  model_id: string;
+  pipelines: Record<string, PipelineDefinition>;
+}
+
+/**
+ * Get inference response from the ES endpoint
+ */
+export interface InferenceConfigResponse {
+  trained_model_configs: ModelConfigResponse[];
+}

x-pack/plugins/ml/public/application/components/stats_bar/index.ts

@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats } from './stats_bar';
+export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats, ModelsBarStats } from './stats_bar';

x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx

@@ -23,7 +23,11 @@ export interface AnalyticStatsBarStats extends Stats {
   stopped: StatsBarStat;
 }
 
-export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats;
+export interface ModelsBarStats {
+  total: StatsBarStat;
+}
+
+export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats | ModelsBarStats;
 type StatsKey = keyof StatsBarStats;
 
 interface StatsBarProps {

x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts

@@ -5,18 +5,21 @@
  */
 
 import { useEffect } from 'react';
-import { BehaviorSubject } from 'rxjs';
-import { filter, distinctUntilChanged } from 'rxjs/operators';
-import { Subscription } from 'rxjs';
+import { BehaviorSubject, Subscription } from 'rxjs';
+import { distinctUntilChanged, filter } from 'rxjs/operators';
 import { cloneDeep } from 'lodash';
 import { ml } from '../../services/ml_api_service';
 import { Dictionary } from '../../../../common/types/common';
 import { getErrorMessage } from '../../../../common/util/errors';
 import { SavedSearchQuery } from '../../contexts/ml';
+import {
+  AnalysisConfig,
+  ClassificationAnalysis,
+  OutlierAnalysis,
+  RegressionAnalysis,
+} from '../../../../common/types/data_frame_analytics';
 
-export type IndexName = string;
 export type IndexPattern = string;
-export type DataFrameAnalyticsId = string;
 
 export enum ANALYSIS_CONFIG_TYPE {
   OUTLIER_DETECTION = 'outlier_detection',
@@ -46,34 +49,6 @@ export enum OUTLIER_ANALYSIS_METHOD {
   DISTANCE_KNN = 'distance_knn',
 }
 
-interface OutlierAnalysis {
-  [key: string]: {};
-  outlier_detection: {};
-}
-
-interface Regression {
-  dependent_variable: string;
-  training_percent?: number;
-  num_top_feature_importance_values?: number;
-  prediction_field_name?: string;
-}
-export interface RegressionAnalysis {
-  [key: string]: Regression;
-  regression: Regression;
-}
-
-interface Classification {
-  dependent_variable: string;
-  training_percent?: number;
-  num_top_classes?: string;
-  num_top_feature_importance_values?: number;
-  prediction_field_name?: string;
-}
-export interface ClassificationAnalysis {
-  [key: string]: Classification;
-  classification: Classification;
-}
-
 export interface LoadExploreDataArg {
   filterByIsTraining?: boolean;
   searchQuery: SavedSearchQuery;
@@ -165,22 +140,12 @@ export interface ClassificationEvaluateResponse {
   };
 }
 
-interface GenericAnalysis {
-  [key: string]: Record<string, any>;
-}
-
 interface LoadEvaluateResult {
   success: boolean;
   eval: RegressionEvaluateResponse | ClassificationEvaluateResponse | null;
   error: string | null;
 }
 
-type AnalysisConfig =
-  | OutlierAnalysis
-  | RegressionAnalysis
-  | ClassificationAnalysis
-  | GenericAnalysis;
-
 export const getAnalysisType = (analysis: AnalysisConfig): string => {
   const keys = Object.keys(analysis);
 
@@ -342,29 +307,6 @@ export interface UpdateDataFrameAnalyticsConfig {
   max_num_threads?: number;
 }
 
-export interface DataFrameAnalyticsConfig {
-  id: DataFrameAnalyticsId;
-  description?: string;
-  dest: {
-    index: IndexName;
-    results_field: string;
-  };
-  source: {
-    index: IndexName | IndexName[];
-    query?: any;
-  };
-  analysis: AnalysisConfig;
-  analyzed_fields: {
-    includes: string[];
-    excludes: string[];
-  };
-  model_memory_limit: string;
-  max_num_threads?: number;
-  create_time: number;
-  version: string;
-  allow_lazy_start?: boolean;
-}
-
 export enum REFRESH_ANALYTICS_LIST_STATE {
   ERROR = 'error',
   IDLE = 'idle',
@@ -379,21 +321,26 @@ export const useRefreshAnalyticsList = (
   callback: {
     isLoading?(d: boolean): void;
     onRefresh?(): void;
-  } = {}
+  } = {},
+  isManagementTable = false
 ) => {
   useEffect(() => {
     const distinct$ = refreshAnalyticsList$.pipe(distinctUntilChanged());
 
     const subscriptions: Subscription[] = [];
 
     if (typeof callback.onRefresh === 'function') {
-      // initial call to refresh
-      callback.onRefresh();
+      // required in order to fetch the DFA jobs on the management page
+      if (isManagementTable) callback.onRefresh();
 
       subscriptions.push(
         distinct$
           .pipe(filter((state) => state === REFRESH_ANALYTICS_LIST_STATE.REFRESH))
-          .subscribe(() => typeof callback.onRefresh === 'function' && callback.onRefresh())
+          .subscribe(() => {
+            if (typeof callback.onRefresh === 'function') {
+              callback.onRefresh();
+            }
+          })
       );
     }
 
@@ -410,7 +357,7 @@ export const useRefreshAnalyticsList = (
     return () => {
       subscriptions.map((sub) => sub.unsubscribe());
     };
-  }, []);
+  }, [callback.onRefresh]);
 
   return {
     refresh: () => {

x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts

@@ -13,13 +13,13 @@ import {
   isClassificationAnalysis,
   isOutlierAnalysis,
   isRegressionAnalysis,
-  DataFrameAnalyticsConfig,
 } from './analytics';
 import { Field } from '../../../../common/types/fields';
 import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 import { newJobCapsService } from '../../services/new_job_capabilities_service';
 
 import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants';
+import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
 
 export type EsId = string;
 export type EsDocSource = Record<string, any>;

x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts

@@ -12,7 +12,8 @@ import { ml } from '../../services/ml_api_service';
 import { isKeywordAndTextType } from '../common/fields';
 import { SavedSearchQuery } from '../../contexts/ml';
 
-import { DataFrameAnalyticsConfig, INDEX_STATUS } from './analytics';
+import { INDEX_STATUS } from './analytics';
+import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
 
 export const getIndexData = async (
   jobConfig: DataFrameAnalyticsConfig | undefined,

x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts

@@ -11,10 +11,7 @@ export {
   isOutlierAnalysis,
   refreshAnalyticsList$,
   useRefreshAnalyticsList,
-  DataFrameAnalyticsId,
-  DataFrameAnalyticsConfig,
   UpdateDataFrameAnalyticsConfig,
-  IndexName,
   IndexPattern,
   REFRESH_ANALYTICS_LIST_STATE,
   ANALYSIS_CONFIG_TYPE,
@@ -45,3 +42,6 @@ export { getIndexData } from './get_index_data';
 export { getIndexFields } from './get_index_fields';
 
 export { useResultsViewConfig } from './use_results_view_config';
+export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
+export { DataFrameAnalyticsId } from '../../../../common/types/data_frame_analytics';
+export { IndexName } from '../../../../common/types/data_frame_analytics';

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx

@@ -23,10 +23,10 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { useMlContext } from '../../../contexts/ml';
 import { newJobCapsService } from '../../../services/new_job_capabilities_service';
 import { ml } from '../../../services/ml_api_service';
-import { DataFrameAnalyticsId } from '../../common/analytics';
 import { useCreateAnalyticsForm } from '../analytics_management/hooks/use_create_analytics_form';
 import { CreateAnalyticsAdvancedEditor } from './components/create_analytics_advanced_editor';
 import { AdvancedStep, ConfigurationStep, CreateStep, DetailsStep } from './components';
+import { DataFrameAnalyticsId } from '../../../../../common/types/data_frame_analytics';
 
 export enum ANALYTICS_STEPS {
   CONFIGURATION,