WIP [Lens] Expose Elasticsearch accuracy warnings to the user elastic…

…#94918

Closes: elastic#94918

src/plugins/data/common/search/aggs/agg_type.ts

@@ -53,6 +53,7 @@ export interface AggTypeConfig<
   json?: boolean;
   decorateAggConfig?: () => any;
   postFlightRequest?: PostFlightRequestFn<TAggConfig>;
+  hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
   getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat;
   getValue?: (agg: TAggConfig, bucket: any) => any;
   getKey?: (bucket: any, key: any, agg: TAggConfig) => any;
@@ -180,6 +181,9 @@ export class AggType<
    * is created, giving the agg type a chance to modify the agg config
    */
   decorateAggConfig: () => any;
+
+  hasPrecisionError?: (aggBucket: Record<string, unknown>) => boolean;
+
   /**
    * A function that needs to be called after the main request has been made
    * and should return an updated response
@@ -283,6 +287,7 @@ export class AggType<
     this.getResponseAggs = config.getResponseAggs || (() => {});
     this.decorateAggConfig = config.decorateAggConfig || (() => ({}));
     this.postFlightRequest = config.postFlightRequest || identity;
+    this.hasPrecisionError = config.hasPrecisionError;
 
     this.getSerializedFormat =
       config.getSerializedFormat ||

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -85,6 +85,7 @@ export const getTermsBucketAgg = () =>
       };
     },
     createFilter: createFilterTerms,
+    hasPrecisionError: (aggBucket) => Boolean(aggBucket?.doc_count_error_upper_bound),
     postFlightRequest: async (
       resp,
       aggConfigs,

src/plugins/data/common/search/tabify/response_writer.test.ts

@@ -164,6 +164,7 @@ describe('TabbedAggResponseWriter class', () => {
             },
           },
           field: 'geo.src',
+          hasPrecisionError: false,
           source: 'esaggs',
           sourceParams: {
             enabled: true,
@@ -188,6 +189,7 @@ describe('TabbedAggResponseWriter class', () => {
         expect(response.columns[1]).toHaveProperty('meta', {
           index: 'logstash-*',
           field: 'machine.os.raw',
+          hasPrecisionError: false,
           params: {
             id: 'number',
           },
@@ -225,6 +227,7 @@ describe('TabbedAggResponseWriter class', () => {
             },
           },
           field: 'geo.src',
+          hasPrecisionError: false,
           source: 'esaggs',
           sourceParams: {
             enabled: true,
@@ -249,6 +252,7 @@ describe('TabbedAggResponseWriter class', () => {
         expect(response.columns[1]).toHaveProperty('meta', {
           index: 'logstash-*',
           field: 'machine.os.raw',
+          hasPrecisionError: false,
           params: {
             id: 'number',
           },

src/plugins/data/common/search/tabify/response_writer.ts

@@ -10,7 +10,7 @@ import { isEmpty } from 'lodash';
 import { IAggConfigs } from '../aggs';
 import { tabifyGetColumns } from './get_columns';
 
-import { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
+import type { TabbedResponseWriterOptions, TabbedAggColumn, TabbedAggRow } from './types';
 import { Datatable, DatatableColumn } from '../../../../expressions/common/expression_types/specs';
 
 interface BufferColumn {
@@ -77,6 +77,7 @@ export class TabbedAggResponseWriter {
               column.aggConfig.type.valueType || column.aggConfig.params.field?.type || 'number',
             field: column.aggConfig.params.field?.name,
             index: column.aggConfig.getIndexPattern()?.title,
+            hasPrecisionError: Boolean(column.hasPrecisionError),
             params: column.aggConfig.toSerializedFieldFormat(),
             source: 'esaggs',
             sourceParams: {

src/plugins/data/common/search/tabify/tabify.ts

@@ -42,8 +42,14 @@ export function tabifyAggResponse(
 
       switch (agg.type.type) {
         case AggGroupNames.Buckets:
-          const aggBucket = get(bucket, agg.id);
+          const aggBucket = get(bucket, agg.id) as Record<string, unknown>;
           const tabifyBuckets = new TabifyBuckets(aggBucket, agg.params, respOpts?.timeRange);
+          const precisionError = agg.type.hasPrecisionError?.(aggBucket);
+
+          if (precisionError) {
+            // "сolumn" mutation, we have to do this here as this value is filled in based on aggBucket value
+            column.hasPrecisionError = true;
+          }
 
           if (tabifyBuckets.length) {
             tabifyBuckets.forEach((subBucket, tabifyBucketKey) => {

src/plugins/data/common/search/tabify/types.ts

@@ -41,6 +41,7 @@ export interface TabbedAggColumn {
   aggConfig: IAggConfig;
   id: string;
   name: string;
+  hasPrecisionError?: boolean;
 }
 
 /** @public **/

src/plugins/expressions/common/expression_types/specs/datatable.ts

@@ -80,6 +80,11 @@ export interface DatatableColumnMeta {
    * any extra parameters for the source that produced this column
    */
   sourceParams?: SerializableRecord;
+
+  /**
+   * has precision error
+   */
+  hasPrecisionError?: boolean;
 }
 
 /**

test/interpreter_functional/snapshots/baseline/combined_test2.json

@@ -1 +1 @@
-{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}
+{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","hasPrecisionError":false,"index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"hasPrecisionError":false,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}

test/interpreter_functional/snapshots/baseline/combined_test3.json

@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","hasPrecisionError":false,"index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"hasPrecisionError":false,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}

test/interpreter_functional/snapshots/baseline/final_output_test.json

@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","hasPrecisionError":false,"index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"hasPrecisionError":false,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}

test/interpreter_functional/snapshots/baseline/partial_test_1.json

@@ -1 +1 @@
-{"as":"tagcloud","type":"render","value":{"syncColors":false,"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","palette":{"name":"custom","params":{"colors":["#882E72","#B178A6","#D6C1DE","#1965B0","#5289C7","#7BAFDE","#4EB265","#90C987","#CAE0AB","#F7EE55","#F6C141","#F1932D","#E8601C","#DC050C"],"continuity":"above","gradient":false,"range":"percent","rangeMax":100,"rangeMin":0,"stops":[]},"type":"palette"},"scale":"linear","showLabel":true},"visType":"tagcloud"}}
+{"as":"tagcloud","type":"render","value":{"syncColors":false,"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","hasPrecisionError":false,"index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"hasPrecisionError":false,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","palette":{"name":"custom","params":{"colors":["#882E72","#B178A6","#D6C1DE","#1965B0","#5289C7","#7BAFDE","#4EB265","#90C987","#CAE0AB","#F7EE55","#F6C141","#F1932D","#E8601C","#DC050C"],"continuity":"above","gradient":false,"range":"percent","rangeMax":100,"rangeMin":0,"stops":[]},"type":"palette"},"scale":"linear","showLabel":true},"visType":"tagcloud"}}

test/interpreter_functional/snapshots/baseline/partial_test_2.json

@@ -1 +1 @@
-{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"metricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","hasPrecisionError":false,"index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"hasPrecisionError":false,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}