[ML] Use a new ML endpoint to estimate a model memory (#60376)

* [ML] refactor calculate_model_memory_limit route, use estimateModelMemory endpoint

* [ML] refactor validate_model_memory_limit, migrate tests to jest

* [ML] fix typing issue

* [ML] start estimateModelMemory url with /

* [ML] fix typo, filter mlcategory

* [ML] extract getCardinalities function

* [ML] fields_service.ts

* [ML] wip getMaxBucketCardinality

* [ML] refactor and comments

* [ML] fix aggs keys with special characters, fix integration tests

* [ML] use pre-defined job types

* [ML] fallback to 0 in case max bucket cardinality receives null

* [ML] calculateModelMemoryLimit on influencers change

* [ML] fix maxModelMemoryLimit

* [ML] cap aggregation to max 1000 buckets

* [ML] rename intervalDuration

x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts

@@ -81,7 +81,7 @@ export interface ModelPlotConfig {
 
 // TODO, finish this when it's needed
 export interface CustomRule {
-  actions: any;
-  scope: object;
-  conditions: object;
+  actions: string[];
+  scope?: object;
+  conditions: any[];
 }

x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts

@@ -88,16 +88,13 @@ export class MultiMetricJobCreator extends JobCreator {
 
   // called externally to set the model memory limit based current detector configuration
   public async calculateModelMemoryLimit() {
-    if (this._splitField === null) {
-      // not split field, use the default
+    if (this.jobConfig.analysis_config.detectors.length === 0) {
       this.modelMemoryLimit = DEFAULT_MODEL_MEMORY_LIMIT;
     } else {
       const { modelMemoryLimit } = await ml.calculateModelMemoryLimit({
+        analysisConfig: this.jobConfig.analysis_config,
         indexPattern: this._indexPatternTitle,
-        splitFieldName: this._splitField.name,
         query: this._datafeed_config.query,
-        fieldNames: this.fields.map(f => f.id),
-        influencerNames: this._influencers,
         timeFieldName: this._job_config.data_description.time_field,
         earliestMs: this._start,
         latestMs: this._end,

x-pack/plugins/ml/public/application/services/ml_api_service/index.ts

@@ -22,6 +22,7 @@ import {
   Datafeed,
   CombinedJob,
   Detector,
+  AnalysisConfig,
 } from '../../../../common/types/anomaly_detection_jobs';
 import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types';
 import { FieldRequestConfig } from '../../datavisualizer/index_based/common';
@@ -532,30 +533,24 @@ export const ml = {
   },
 
   calculateModelMemoryLimit({
+    analysisConfig,
     indexPattern,
-    splitFieldName,
     query,
-    fieldNames,
-    influencerNames,
     timeFieldName,
     earliestMs,
     latestMs,
   }: {
+    analysisConfig: AnalysisConfig;
     indexPattern: string;
-    splitFieldName: string;
     query: any;
-    fieldNames: string[];
-    influencerNames: string[];
     timeFieldName: string;
     earliestMs: number;
     latestMs: number;
   }) {
     const body = JSON.stringify({
+      analysisConfig,
       indexPattern,
-      splitFieldName,
       query,
-      fieldNames,
-      influencerNames,
       timeFieldName,
       earliestMs,
       latestMs,

x-pack/plugins/ml/server/client/elasticsearch_ml.ts

@@ -413,6 +413,14 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any)
     method: 'POST',
   });
 
+  ml.estimateModelMemory = ca({
+    url: {
+      fmt: '/_ml/anomaly_detectors/_estimate_model_memory',
+    },
+    needBody: true,
+    method: 'POST',
+  });
+
   ml.datafeedPreview = ca({
     url: {
       fmt: '/_ml/datafeeds/<%=datafeedId%>/_preview',