[Monitoring] Handle no mappings found for sort and collapse fields

x-pack/plugins/monitoring/server/alerts/base_alert.ts

@@ -9,6 +9,7 @@ import {
   ILegacyCustomClusterClient,
   Logger,
   IUiSettingsClient,
+  LegacyCallAPIOptions,
 } from 'kibana/server';
 import { i18n } from '@kbn/i18n';
 import {
@@ -36,6 +37,8 @@ import { MonitoringConfig } from '../config';
 import { AlertSeverity } from '../../common/enums';
 import { CommonAlertFilter, CommonAlertParams, CommonBaseAlert } from '../../common/types';
 import { MonitoringLicenseService } from '../types';
+import { mbSafeQuery } from '../lib/mb_safe_query';
+import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index';
 
 export class BaseAlert {
   public type!: string;
@@ -212,13 +215,20 @@ export class BaseAlert {
       `Executing alert with params: ${JSON.stringify(params)} and state: ${JSON.stringify(state)}`
     );
 
-    const callCluster = this.monitoringCluster
+    const _callCluster = this.monitoringCluster
       ? this.monitoringCluster.callAsInternalUser
       : services.callCluster;
+    const callCluster = async (
+      endpoint: string,
+      clientParams?: Record<string, unknown>,
+      options?: LegacyCallAPIOptions
+    ) => {
+      return await mbSafeQuery(async () => _callCluster(endpoint, clientParams, options));
+    };
     const availableCcs = this.config.ui.ccs.enabled ? await fetchAvailableCcs(callCluster) : [];
     // Support CCS use cases by querying to find available remote clusters
     // and then adding those to the index pattern we are searching against
-    let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
+    let esIndexPattern = appendMetricbeatIndex(this.config, INDEX_PATTERN_ELASTICSEARCH);
     if (availableCcs) {
       esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
     }

x-pack/plugins/monitoring/server/alerts/cluster_health_alert.test.ts

@@ -66,7 +66,11 @@ describe('ClusterHealthAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts

@@ -70,7 +70,11 @@ describe('CpuUsageAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts

@@ -31,6 +31,7 @@ import {
   CommonAlertParams,
   CommonAlertParamDetail,
 } from '../../common/types';
+import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index';
 
 const RESOLVED = i18n.translate('xpack.monitoring.alerts.cpuUsage.resolved', {
   defaultMessage: 'resolved',
@@ -137,7 +138,7 @@ export class CpuUsageAlert extends BaseAlert {
     uiSettings: IUiSettingsClient,
     availableCcs: string[]
   ): Promise<AlertData[]> {
-    let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
+    let esIndexPattern = appendMetricbeatIndex(this.config, INDEX_PATTERN_ELASTICSEARCH);
     if (availableCcs) {
       esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
     }

x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_alert.test.ts

@@ -69,7 +69,11 @@ describe('ElasticsearchVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_alert.test.ts

@@ -72,7 +72,11 @@ describe('KibanaVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts

@@ -76,7 +76,11 @@ describe('LicenseExpirationAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_alert.test.ts

@@ -69,7 +69,11 @@ describe('LogstashVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/nodes_changed_alert.test.ts

@@ -82,7 +82,11 @@ describe('NodesChangedAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-*' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/lib/alerts/append_mb_index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { MonitoringConfig } from '../../config';
+
+export function appendMetricbeatIndex(config: MonitoringConfig, indexPattern: string) {
+  return `${indexPattern},${config.ui.metricbeat.index}`;
+}

x-pack/plugins/monitoring/server/lib/alerts/get_ccs_index_pattern.ts

@@ -4,10 +4,11 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 export function getCcsIndexPattern(indexPattern: string, remotes: string[]): string {
-  return `${indexPattern},${indexPattern
-    .split(',')
-    .map((pattern) => {
-      return remotes.map((remoteName) => `${remoteName}:${pattern}`).join(',');
-    })
-    .join(',')}`;
+  const patternsToAdd = [];
+  for (const index of indexPattern.split(',')) {
+    for (const remote of remotes) {
+      patternsToAdd.push(`${remote}:${index}`);
+    }
+  }
+  return [...indexPattern.split(','), ...patternsToAdd].join(',');
 }

x-pack/plugins/monitoring/server/lib/apm/_get_time_of_last_event.js

@@ -26,6 +26,7 @@ export async function getTimeOfLastEvent({
         {
           timestamp: {
             order: 'desc',
+            unmapped_type: 'long',
           },
         },
       ],

x-pack/plugins/monitoring/server/lib/apm/get_apm_info.js

@@ -73,7 +73,7 @@ export async function getApmInfo(req, apmIndexPattern, { clusterUuid, apmUuid, s
       'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
     ],
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({
         start,
         end,

x-pack/plugins/monitoring/server/lib/apm/get_apms.js

@@ -128,8 +128,8 @@ export async function getApms(req, apmIndexPattern, clusterUuid) {
         },
       },
       sort: [
-        { 'beats_stats.beat.uuid': { order: 'asc' } }, // need to keep duplicate uuids grouped
-        { timestamp: { order: 'desc' } }, // need oldest timestamp to come first for rate calcs to work
+        { 'beats_stats.beat.uuid': { order: 'asc', unmapped_type: 'long' } }, // need to keep duplicate uuids grouped
+        { timestamp: { order: 'desc', unmapped_type: 'long' } }, // need oldest timestamp to come first for rate calcs to work
       ],
     },
   };

x-pack/plugins/monitoring/server/lib/beats/get_beat_summary.js

@@ -78,7 +78,7 @@ export async function getBeatSummary(
       'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
     ],
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createBeatsQuery({
         start,
         end,

x-pack/plugins/monitoring/server/lib/beats/get_beats.js

@@ -126,8 +126,8 @@ export async function getBeats(req, beatsIndexPattern, clusterUuid) {
         },
       },
       sort: [
-        { 'beats_stats.beat.uuid': { order: 'asc' } }, // need to keep duplicate uuids grouped
-        { timestamp: { order: 'desc' } }, // need oldest timestamp to come first for rate calcs to work
+        { 'beats_stats.beat.uuid': { order: 'asc', unmapped_type: 'long' } }, // need to keep duplicate uuids grouped
+        { timestamp: { order: 'desc', unmapped_type: 'long' } }, // need oldest timestamp to come first for rate calcs to work
       ],
     },
   };

x-pack/plugins/monitoring/server/lib/ccs_utils.js

@@ -13,7 +13,7 @@ export function appendMetricbeatIndex(config, indexPattern) {
   if (isFunction(config.get)) {
     mbIndex = config.get('monitoring.ui.metricbeat.index');
   } else {
-    mbIndex = get(config, 'monitoring.ui.metricbeat.index');
+    mbIndex = get(config, 'ui.metricbeat.index');
   }
 
   const newIndexPattern = `${indexPattern},${mbIndex}`;

x-pack/plugins/monitoring/server/lib/cluster/flag_supported_clusters.js

@@ -31,7 +31,7 @@ async function findSupportedBasicLicenseCluster(
     ignoreUnavailable: true,
     filterPath: 'hits.hits._source.cluster_uuid',
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: {
         bool: {
           filter: [

x-pack/plugins/monitoring/server/lib/cluster/get_cluster_license.js

@@ -18,7 +18,7 @@ export function getClusterLicense(req, esIndexPattern, clusterUuid) {
     ignoreUnavailable: true,
     filterPath: 'hits.hits._source.license',
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({
         type: 'cluster_stats',
         clusterUuid,

x-pack/plugins/monitoring/server/lib/cluster/get_clusters_state.js

@@ -70,7 +70,7 @@ export function getClustersState(req, esIndexPattern, clusters) {
       collapse: {
         field: 'cluster_uuid',
       },
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
     },
   };
 

x-pack/plugins/monitoring/server/lib/cluster/get_clusters_stats.js

@@ -67,7 +67,7 @@ function fetchClusterStats(req, esIndexPattern, clusterUuid) {
       collapse: {
         field: 'cluster_uuid',
       },
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
     },
   };
 

x-pack/plugins/monitoring/server/lib/elasticsearch/ccr.js

@@ -37,7 +37,7 @@ export async function checkCcrEnabled(req, esIndexPattern) {
         clusterUuid,
         metric: metricFields,
       }),
-      sort: [{ timestamp: { order: 'desc' } }],
+      sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
     },
     filterPath: ['hits.hits._source.stack_stats.xpack.ccr'],
   };

x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.js

@@ -61,7 +61,7 @@ export function getLastRecovery(req, esIndexPattern) {
     ignoreUnavailable: true,
     body: {
       _source: ['index_recovery.shards'],
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({ type: 'index_recovery', start, end, clusterUuid, metric }),
     },
   };

x-pack/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.js

@@ -42,7 +42,7 @@ export function getMlJobs(req, esIndexPattern) {
       'hits.hits._source.job_stats.node.name',
     ],
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       collapse: { field: 'job_stats.job_id' },
       query: createQuery({ type: 'job_stats', start, end, clusterUuid, metric }),
     },

x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_index_summary.js

@@ -69,7 +69,7 @@ export function getIndexSummary(
     size: 1,
     ignoreUnavailable: true,
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({ type: 'index_stats', start, end, clusterUuid, metric, filters }),
     },
   };

x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js

@@ -129,7 +129,7 @@ export function buildGetIndicesQuery(
           sort: [{ timestamp: 'asc' }],
         },
       },
-      sort: [{ timestamp: { order: 'desc' } }],
+      sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
     },
   };
 }

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.js

@@ -109,7 +109,7 @@ export function getNodeSummary(
     size: 1,
     ignoreUnavailable: true,
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({ type: 'node_stats', start, end, clusterUuid, metric, filters }),
     },
   };

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js

@@ -92,7 +92,7 @@ export async function getNodes(req, esIndexPattern, pageOfNodes, clusterStats, n
           },
         },
       },
-      sort: [{ timestamp: { order: 'desc' } }],
+      sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
     },
     filterPath: [
       'hits.hits._source.source_node',

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js

@@ -20,7 +20,7 @@ async function getUnassignedShardData(req, esIndexPattern, cluster) {
     size: 0,
     ignoreUnavailable: true,
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({
         type: 'shards',
         clusterUuid: cluster.cluster_uuid,

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js

@@ -19,7 +19,7 @@ async function getShardCountPerNode(req, esIndexPattern, cluster) {
     size: 0,
     ignoreUnavailable: true,
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({
         type: 'shards',
         clusterUuid: cluster.cluster_uuid,

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.js

@@ -57,7 +57,7 @@ export function getShardStats(
     size: 0,
     ignoreUnavailable: true,
     body: {
-      sort: { timestamp: { order: 'desc' } },
+      sort: { timestamp: { order: 'desc', unmapped_type: 'long' } },
       query: createQuery({
         type: 'shards',
         clusterUuid: cluster.cluster_uuid,

x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.js

@@ -41,7 +41,7 @@ export function getKibanaInfo(req, kbnIndexPattern, { clusterUuid, kibanaUuid })
         },
       },
       collapse: { field: 'kibana_stats.kibana.uuid' },
-      sort: [{ timestamp: { order: 'desc' } }],
+      sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
     },
   };
 

x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.js

@@ -44,7 +44,7 @@ export function getKibanas(req, kbnIndexPattern, { clusterUuid }) {
       collapse: {
         field: 'kibana_stats.kibana.uuid',
       },
-      sort: [{ timestamp: { order: 'desc' } }],
+      sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
       _source: [
         'timestamp',
         'kibana_stats.process.memory.resident_set_size_in_bytes',

x-pack/plugins/monitoring/server/lib/logs/get_log_types.js

@@ -65,7 +65,7 @@ export async function getLogTypes(
     filterPath: ['aggregations.levels.buckets', 'aggregations.types.buckets'],
     ignoreUnavailable: true,
     body: {
-      sort: { '@timestamp': { order: 'desc' } },
+      sort: { '@timestamp': { order: 'desc', unmapped_type: 'long' } },
       query: {
         bool: {
           filter,

x-pack/plugins/monitoring/server/lib/logs/get_logs.js

@@ -82,7 +82,7 @@ export async function getLogs(
     ],
     ignoreUnavailable: true,
     body: {
-      sort: { '@timestamp': { order: 'desc' } },
+      sort: { '@timestamp': { order: 'desc', unmapped_type: 'long' } },
       query: {
         bool: {
           filter,