[8.6] [ML] Explain Log Rate Spikes: Fix field candidate selection. (e…

…lastic#147614) (elastic#147792)

# Backport

This will backport the following commits from `main` to `8.6`:
- [[ML] Explain Log Rate Spikes: Fix field candidate selection.
(elastic#147614)](elastic#147614)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Walter
Rafelsberger","email":"walter.rafelsberger@elastic.co"},"sourceCommit":{"committedDate":"2022-12-19T17:35:31Z","message":"[ML]
Explain Log Rate Spikes: Fix field candidate selection. (elastic#147614)\n\nThe
field candidate selection for Explain Log Rate Spikes was missing
a\r\ncheck if the supported field type is also aggregatable. For
example, a\r\n`keyword` type field could still be non-aggregatable if it
was both not\r\nindexed and `doc_values` set to `false`. Additionally,
if no groups were\r\ndetected, we showed a \"Try to continue analysis\"
button in the UI even\r\nif the analysis was able to finish. In this PR
the artificial logs dataset for functional tests was extended\r\nto
include a field like
that.","sha":"aecad27159764d8ea2d0aeddc94fd03954d480e5","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix",":ml","Feature:ML/AIOps","v8.6.0","v8.7.0"],"number":147614,"url":"elastic#147614
Explain Log Rate Spikes: Fix field candidate selection. (elastic#147614)\n\nThe
field candidate selection for Explain Log Rate Spikes was missing
a\r\ncheck if the supported field type is also aggregatable. For
example, a\r\n`keyword` type field could still be non-aggregatable if it
was both not\r\nindexed and `doc_values` set to `false`. Additionally,
if no groups were\r\ndetected, we showed a \"Try to continue analysis\"
button in the UI even\r\nif the analysis was able to finish. In this PR
the artificial logs dataset for functional tests was extended\r\nto
include a field like
that.","sha":"aecad27159764d8ea2d0aeddc94fd03954d480e5"}},"sourceBranch":"main","suggestedTargetBranches":["8.6"],"targetPullRequestStates":[{"branch":"8.6","label":"v8.6.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"elastic#147614
Explain Log Rate Spikes: Fix field candidate selection. (elastic#147614)\n\nThe
field candidate selection for Explain Log Rate Spikes was missing
a\r\ncheck if the supported field type is also aggregatable. For
example, a\r\n`keyword` type field could still be non-aggregatable if it
was both not\r\nindexed and `doc_values` set to `false`. Additionally,
if no groups were\r\ndetected, we showed a \"Try to continue analysis\"
button in the UI even\r\nif the analysis was able to finish. In this PR
the artificial logs dataset for functional tests was extended\r\nto
include a field like
that.","sha":"aecad27159764d8ea2d0aeddc94fd03954d480e5"}}]}] BACKPORT-->

Co-authored-by: Walter Rafelsberger <walter.rafelsberger@elastic.co>

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx

@@ -112,8 +112,9 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
       const { loaded, remainingFieldCandidates, groupsMissing } = data;
 
       if (
-        (Array.isArray(remainingFieldCandidates) && remainingFieldCandidates.length > 0) ||
-        groupsMissing
+        loaded < 1 &&
+        ((Array.isArray(remainingFieldCandidates) && remainingFieldCandidates.length > 0) ||
+          groupsMissing)
       ) {
         setOverrides({ loaded, remainingFieldCandidates, changePoints: data.changePoints });
       } else {

x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts

@@ -70,9 +70,16 @@ describe('fetch_index_info', () => {
     it('returns field candidates and total hits', async () => {
       const esClientFieldCapsMock = jest.fn(() => ({
         fields: {
-          myIpFieldName: { ip: {} },
-          myKeywordFieldName: { keyword: {} },
-          myUnpopulatedKeywordFieldName: { keyword: {} },
+          // Should end up as a field candidate
+          myIpFieldName: { ip: { aggregatable: true } },
+          // Should end up as a field candidate
+          myKeywordFieldName: { keyword: { aggregatable: true } },
+          // Should not end up as a field candidate, it's a keyword but non-aggregatable
+          myKeywordFieldNameToBeIgnored: { keyword: { aggregatable: false } },
+          // Should not end up as a field candidate, based on this field caps result it would be
+          // but it will not be part of the mocked search result so will count as unpopulated.
+          myUnpopulatedKeywordFieldName: { keyword: { aggregatable: true } },
+          // Should not end up as a field candidate since fields of type number will not be considered
           myNumericFieldName: { number: {} },
         },
       }));

x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts

@@ -68,9 +68,10 @@ export const fetchIndexInfo = async (
   Object.entries(respMapping.fields).forEach(([key, value]) => {
     const fieldTypes = Object.keys(value) as ES_FIELD_TYPES[];
     const isSupportedType = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES.includes(type));
+    const isAggregatable = fieldTypes.some((type) => value[type].aggregatable);
 
     // Check if fieldName is something we can aggregate on
-    if (isSupportedType) {
+    if (isSupportedType && isAggregatable) {
       acceptableFields.add(key);
     }
   });

x-pack/test/functional/services/aiops/explain_log_rate_spikes_data_generator.ts

@@ -15,6 +15,7 @@ export interface GeneratedDoc {
   url: string;
   version: string;
   '@timestamp': number;
+  should_ignore_this_field: string;
 }
 
 const REFERENCE_TS = 1669018354793;
@@ -50,6 +51,7 @@ function getArtificialLogsWithSpike(index: string) {
                 url,
                 version: 'v1.0.0',
                 '@timestamp': ts + tsOffset,
+                should_ignore_this_field: 'should_ignore_this_field',
               };
 
               bulkBody.push(action);
@@ -74,6 +76,7 @@ function getArtificialLogsWithSpike(index: string) {
           url,
           version: 'v1.0.0',
           '@timestamp': DEVIATION_TS + tsOffset,
+          should_ignore_this_field: 'should_ignore_this_field',
         });
       });
     });
@@ -91,6 +94,7 @@ function getArtificialLogsWithSpike(index: string) {
           url,
           version: 'v1.0.0',
           '@timestamp': DEVIATION_TS + tsOffset,
+          should_ignore_this_field: 'should_ignore_this_field',
         });
       });
     });
@@ -158,6 +162,7 @@ export function ExplainLogRateSpikesDataGeneratorProvider({ getService }: FtrPro
                 url: { type: 'keyword' },
                 version: { type: 'keyword' },
                 '@timestamp': { type: 'date' },
+                should_ignore_this_field: { type: 'keyword', doc_values: false, index: false },
               },
             },
           });