From c73a911bf1145bf82bed126e473a66f96285b369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 3 Oct 2019 14:52:00 +0200 Subject: [PATCH] [Logs UI] Support partitioned log rate results in API (#46751) * Add partitioning to log rate results API --- .../log_analysis/results/log_entry_rate.ts | 21 +- .../log_analysis/api/get_log_entry_rate.ts | 45 ++ .../log_entry_rate.tsx | 71 --- .../logs/log_analysis/log_entry_rate.tsx | 37 +- .../logs/analysis/chart_helpers/index.tsx | 20 - .../logs/analysis/page_results_content.tsx | 42 +- .../analysis/sections/log_rate/bar_chart.tsx | 167 ++++++ .../logs/analysis/sections/log_rate/chart.tsx | 196 ------- .../logs/analysis/sections/log_rate/index.tsx | 9 +- .../server/lib/log_analysis/log_analysis.ts | 253 ++++----- .../server/lib/log_analysis/queries/index.ts | 7 + .../log_analysis/queries/log_entry_rate.ts | 170 ++++++ .../translations/translations/ja-JP.json | 3 - .../translations/translations/zh-CN.json | 3 - .../apis/infra/log_analysis.ts | 13 +- .../data.json.gz | Bin 0 -> 6298 bytes .../mappings.json | 513 ++++++++++++++++++ 17 files changed, 1023 insertions(+), 547 deletions(-) create mode 100644 x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts delete mode 100644 x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx delete mode 100644 x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx create mode 100644 x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/bar_chart.tsx delete mode 100644 x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx create mode 100644 x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts create mode 100644 x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts create mode 100644 x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz create mode 100644 x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/mappings.json diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts index 2dcaf35cc41d9f..3af07980910b8a 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts @@ -6,13 +6,7 @@ import * as rt from 'io-ts'; -import { - badRequestErrorRT, - conflictErrorRT, - forbiddenErrorRT, - metricStatisticsRT, - timeRangeRT, -} from '../../shared'; +import { badRequestErrorRT, conflictErrorRT, forbiddenErrorRT, timeRangeRT } from '../../shared'; export const LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH = '/api/infra/log_analysis/results/log_entry_rate'; @@ -43,12 +37,15 @@ export const logEntryRateAnomaly = rt.type({ typicalLogEntryRate: rt.number, }); -export const logEntryRateHistogramBucket = rt.type({ +export const logEntryRateDataSetRT = rt.type({ + analysisBucketCount: rt.number, anomalies: rt.array(logEntryRateAnomaly), - duration: rt.number, - logEntryRateStats: metricStatisticsRT, - modelLowerBoundStats: metricStatisticsRT, - modelUpperBoundStats: metricStatisticsRT, + averageActualLogEntryRate: rt.number, + dataSetId: rt.string, +}); + +export const logEntryRateHistogramBucket = rt.type({ + dataSets: rt.array(logEntryRateDataSetRT), startTime: rt.number, }); diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts new file mode 100644 index 00000000000000..471a00d40984cf --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { identity } from 'fp-ts/lib/function'; +import { kfetch } from 'ui/kfetch'; + +import { + getLogEntryRateRequestPayloadRT, + getLogEntryRateSuccessReponsePayloadRT, + LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, +} from '../../../../../common/http_api/log_analysis'; +import { createPlainError, throwErrors } from '../../../../../common/runtime_types'; + +export const callGetLogEntryRateAPI = async ( + sourceId: string, + startTime: number, + endTime: number, + bucketDuration: number +) => { + const response = await kfetch({ + method: 'POST', + pathname: LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, + body: JSON.stringify( + getLogEntryRateRequestPayloadRT.encode({ + data: { + sourceId, + timeRange: { + startTime, + endTime, + }, + bucketDuration, + }, + }) + ), + }); + return pipe( + getLogEntryRateSuccessReponsePayloadRT.decode(response), + fold(throwErrors(createPlainError), identity) + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx deleted file mode 100644 index f54402a1a87073..00000000000000 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { useMemo } from 'react'; -import { GetLogEntryRateSuccessResponsePayload } from '../../../../../common/http_api/log_analysis'; - -interface LogRateAreaSeriesDataPoint { - x: number; - min: number | null; - max: number | null; -} -type LogRateAreaSeries = LogRateAreaSeriesDataPoint[]; -type LogRateLineSeriesDataPoint = [number, number | null]; -type LogRateLineSeries = LogRateLineSeriesDataPoint[]; -type LogRateAnomalySeriesDataPoint = [number, number]; -type LogRateAnomalySeries = LogRateAnomalySeriesDataPoint[]; - -export const useLogEntryRateGraphData = ({ - data, -}: { - data: GetLogEntryRateSuccessResponsePayload['data'] | null; -}) => { - const areaSeries: LogRateAreaSeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - acc.push({ - x: bucket.startTime, - min: bucket.modelLowerBoundStats.min, - max: bucket.modelUpperBoundStats.max, - }); - return acc; - }, []); - }, [data]); - - const lineSeries: LogRateLineSeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - acc.push([bucket.startTime, bucket.logEntryRateStats.avg]); - return acc; - }, []); - }, [data]); - - const anomalySeries: LogRateAnomalySeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - if (bucket.anomalies.length > 0) { - bucket.anomalies.forEach(anomaly => { - acc.push([anomaly.startTime, anomaly.actualLogEntryRate]); - }); - return acc; - } else { - return acc; - } - }, []); - }, [data]); - - return { - areaSeries, - lineSeries, - anomalySeries, - }; -}; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx index 4e7a6647a95793..8b21a7e8298944 100644 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx @@ -5,19 +5,10 @@ */ import { useMemo, useState } from 'react'; -import { kfetch } from 'ui/kfetch'; -import { fold } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { identity } from 'fp-ts/lib/function'; -import { - getLogEntryRateRequestPayloadRT, - getLogEntryRateSuccessReponsePayloadRT, - GetLogEntryRateSuccessResponsePayload, - LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, -} from '../../../../common/http_api/log_analysis'; -import { createPlainError, throwErrors } from '../../../../common/runtime_types'; +import { GetLogEntryRateSuccessResponsePayload } from '../../../../common/http_api/log_analysis'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { callGetLogEntryRateAPI } from './api/get_log_entry_rate'; type LogEntryRateResults = GetLogEntryRateSuccessResponsePayload['data']; @@ -38,30 +29,10 @@ export const useLogEntryRate = ({ { cancelPreviousOn: 'resolution', createPromise: async () => { - return await kfetch({ - method: 'POST', - pathname: LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, - body: JSON.stringify( - getLogEntryRateRequestPayloadRT.encode({ - data: { - sourceId, - timeRange: { - startTime, - endTime, - }, - bucketDuration, - }, - }) - ), - }); + return await callGetLogEntryRateAPI(sourceId, startTime, endTime, bucketDuration); }, onResolve: response => { - const { data } = pipe( - getLogEntryRateSuccessReponsePayloadRT.decode(response), - fold(throwErrors(createPlainError), identity) - ); - - setLogEntryRate(data); + setLogEntryRate(response.data); }, }, [sourceId, startTime, endTime, bucketDuration] diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx deleted file mode 100644 index df0eca449bb9f8..00000000000000 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import chrome from 'ui/chrome'; -import { SpecId, Theme, LIGHT_THEME, DARK_THEME } from '@elastic/charts'; - -export const getColorsMap = (color: string, specId: SpecId) => { - const map = new Map(); - map.set({ colorValues: [], specId }, color); - return map; -}; - -export const isDarkMode = () => chrome.getUiSettingsClient().get('theme:darkMode'); - -export const getChartTheme = (): Theme => { - return isDarkMode() ? DARK_THEME : LIGHT_THEME; -}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx index 3629413d6d30c5..aaf24c22594e52 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx @@ -6,7 +6,6 @@ import datemath from '@elastic/datemath'; import { - EuiBadge, EuiFlexGroup, EuiFlexItem, EuiPage, @@ -17,7 +16,6 @@ import { EuiSuperDatePicker, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n/react'; import moment from 'moment'; import React, { useCallback, useMemo, useState } from 'react'; @@ -122,21 +120,6 @@ export const AnalysisResultsContent = ({ [setAutoRefresh] ); - const anomaliesDetected = useMemo(() => { - if (!logEntryRate) { - return null; - } else { - if (logEntryRate.histogramBuckets && logEntryRate.histogramBuckets.length) { - return logEntryRate.histogramBuckets.reduce( - (acc, bucket) => acc + bucket.anomalies.length, - 0 - ); - } else { - return null; - } - } - }, [logEntryRate]); - return ( <> {isLoading && !logEntryRate ? ( @@ -150,29 +133,8 @@ export const AnalysisResultsContent = ({ - - - - {anomaliesDetected !== null ? ( - - - {anomaliesDetected} - - ), - number: anomaliesDetected, - }} - /> - - ) : null} - - - - + + void; + timeRange: TimeRange; +}> = ({ bucketDuration, histogramBuckets, setTimeRange, timeRange }) => { + const [dateFormat] = useKibanaUiSetting('dateFormat'); + const [isDarkMode] = useKibanaUiSetting('theme:darkMode'); + + const chartDateFormatter = useMemo( + () => niceTimeFormatter([timeRange.startTime, timeRange.endTime]), + [timeRange] + ); + + const logEntryRateSeries = useMemo( + () => + histogramBuckets + ? histogramBuckets.reduce>( + (buckets, bucket) => { + return [ + ...buckets, + ...bucket.dataSets.map(dataSet => ({ + group: dataSet.dataSetId === '' ? 'unknown' : dataSet.dataSetId, + time: bucket.startTime, + value: dataSet.averageActualLogEntryRate, + })), + ]; + }, + [] + ) + : [], + [histogramBuckets] + ); + + const logEntryRateAnomalyAnnotations = useMemo( + () => + histogramBuckets + ? histogramBuckets.reduce((annotatedBuckets, bucket) => { + const anomalies = bucket.dataSets.reduce( + (accumulatedAnomalies, dataSet) => [...accumulatedAnomalies, ...dataSet.anomalies], + [] + ); + if (anomalies.length <= 0) { + return annotatedBuckets; + } + return [ + ...annotatedBuckets, + { + coordinates: { + x0: bucket.startTime, + x1: bucket.startTime + bucketDuration, + }, + details: i18n.translate( + 'xpack.infra.logs.analysis.logRateSectionAnomalyCountTooltipLabel', + { + defaultMessage: `{anomalyCount, plural, one {# anomaly} other {# anomalies}}`, + values: { + anomalyCount: anomalies.length, + }, + } + ), + }, + ]; + }, []) + : [], + [histogramBuckets] + ); + + const logEntryRateSpecId = getSpecId('averageValues'); + const logEntryRateAnomalyAnnotationsId = getAnnotationId('anomalies'); + + const tooltipProps = useMemo( + () => ({ + headerFormatter: (tooltipData: TooltipValue) => + moment(tooltipData.value).format(dateFormat || 'Y-MM-DD HH:mm:ss.SSS'), + }), + [dateFormat] + ); + + const handleBrushEnd = useCallback( + (startTime: number, endTime: number) => { + setTimeRange({ + endTime, + startTime, + }); + }, + [setTimeRange] + ); + + return ( +
+ + + Number(value).toFixed(0)} + /> + + + + +
+ ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx deleted file mode 100644 index 0d703420e7412a..00000000000000 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { useMemo, useCallback, useState } from 'react'; -import { i18n } from '@kbn/i18n'; -import moment from 'moment'; -import { - Axis, - Chart, - getAxisId, - getSpecId, - AreaSeries, - LineSeries, - niceTimeFormatter, - Settings, - TooltipValue, -} from '@elastic/charts'; -import { EuiFlexGroup, EuiFlexItem, EuiCheckbox } from '@elastic/eui'; -import { getColorsMap, isDarkMode, getChartTheme } from '../../chart_helpers'; -import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; -import { useLogEntryRateGraphData } from '../../../../../containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate'; -import { useKibanaUiSetting } from '../../../../../utils/use_kibana_ui_setting'; -import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; - -const areaSeriesColour = 'rgb(224, 237, 255)'; -const lineSeriesColour = 'rgb(49, 133, 252)'; - -interface Props { - data: GetLogEntryRateSuccessResponsePayload['data'] | null; - setTimeRange: (timeRange: TimeRange) => void; - timeRange: TimeRange; -} - -export const ChartView = ({ data, setTimeRange, timeRange }: Props) => { - const { areaSeries, lineSeries, anomalySeries } = useLogEntryRateGraphData({ data }); - - const dateFormatter = useMemo( - () => - lineSeries.length > 0 - ? niceTimeFormatter([timeRange.startTime, timeRange.endTime]) - : (value: number) => `${value}`, - [lineSeries, timeRange] - ); - - const areaSpecId = getSpecId('modelBounds'); - const lineSpecId = getSpecId('averageValues'); - const anomalySpecId = getSpecId('anomalies'); - - const [dateFormat] = useKibanaUiSetting('dateFormat'); - - const tooltipProps = useMemo( - () => ({ - headerFormatter: (tooltipData: TooltipValue) => - moment(tooltipData.value).format(dateFormat || 'Y-MM-DD HH:mm:ss.SSS'), - }), - [dateFormat] - ); - - const [isShowingModelBounds, setIsShowingModelBounds] = useState(true); - - const handleBrushEnd = useCallback( - (startTime: number, endTime: number) => { - setTimeRange({ - endTime, - startTime, - }); - }, - [setTimeRange] - ); - - return ( - <> - - - { - setIsShowingModelBounds(e.target.checked); - }} - /> - - -
- - - Number(value).toFixed(0)} - /> - {isShowingModelBounds ? ( - - ) : null} - - - - -
- - ); -}; - -const showModelBoundsLabel = i18n.translate( - 'xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel', - { defaultMessage: 'Show model bounds' } -); diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx index c03a4817e4d4c1..1f01af33e33c4f 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx @@ -16,8 +16,8 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; -import { ChartView } from './chart'; import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { LogEntryRateBarChart } from './bar_chart'; export const LogRateResults = ({ isLoading, @@ -70,7 +70,12 @@ export const LogRateResults = ({ } /> ) : ( - + )} ); diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts index ac7f7b6df8d62f..31d9c5403e2d2b 100644 --- a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts @@ -4,8 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as rt from 'io-ts'; - import { pipe } from 'fp-ts/lib/pipeable'; import { map, fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; @@ -13,8 +11,14 @@ import { getJobId } from '../../../common/log_analysis'; import { throwErrors, createPlainError } from '../../../common/runtime_types'; import { InfraBackendFrameworkAdapter, InfraFrameworkRequest } from '../adapters/framework'; import { NoLogRateResultsIndexError } from './errors'; +import { + logRateModelPlotResponseRT, + createLogEntryRateQuery, + LogRateModelPlotBucket, + CompositeTimestampDataSetKey, +} from './queries'; -const ML_ANOMALY_INDEX_PREFIX = '.ml-anomalies-'; +const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; export class InfraLogAnalysis { constructor( @@ -38,168 +42,95 @@ export class InfraLogAnalysis { ) { const logRateJobId = this.getJobIds(request, sourceId).logEntryRate; - const mlModelPlotResponse = await this.libs.framework.callWithRequest(request, 'search', { - allowNoIndices: true, - body: { - query: { - bool: { - filter: [ - { - range: { - timestamp: { - gte: startTime, - lt: endTime, - }, - }, - }, - { - terms: { - result_type: ['model_plot', 'record'], - }, - }, - { - term: { - detector_index: { - value: 0, - }, - }, - }, - ], - }, - }, - aggs: { - timestamp_buckets: { - date_histogram: { - field: 'timestamp', - fixed_interval: `${bucketDuration}ms`, - }, - aggs: { - filter_model_plot: { - filter: { - term: { - result_type: 'model_plot', - }, - }, - aggs: { - stats_model_lower: { - stats: { - field: 'model_lower', - }, - }, - stats_model_upper: { - stats: { - field: 'model_upper', - }, - }, - stats_actual: { - stats: { - field: 'actual', - }, - }, - }, - }, - filter_records: { - filter: { - term: { - result_type: 'record', - }, - }, - aggs: { - top_hits_record: { - top_hits: { - _source: Object.keys(logRateMlRecordRT.props), - size: 100, - sort: [ - { - timestamp: 'asc', - }, - ], - }, - }, - }, - }, - }, - }, - }, - }, - ignoreUnavailable: true, - index: `${ML_ANOMALY_INDEX_PREFIX}${logRateJobId}`, - size: 0, - trackScores: false, - trackTotalHits: false, - }); + let mlModelPlotBuckets: LogRateModelPlotBucket[] = []; + let afterLatestBatchKey: CompositeTimestampDataSetKey | undefined; - if (mlModelPlotResponse._shards.total === 0) { - throw new NoLogRateResultsIndexError( - `Failed to find ml result index for job ${logRateJobId}.` + while (true) { + const mlModelPlotResponse = await this.libs.framework.callWithRequest( + request, + 'search', + createLogEntryRateQuery( + logRateJobId, + startTime, + endTime, + bucketDuration, + COMPOSITE_AGGREGATION_BATCH_SIZE, + afterLatestBatchKey + ) ); - } - const mlModelPlotBuckets = pipe( - logRateModelPlotResponseRT.decode(mlModelPlotResponse), - map(response => response.aggregations.timestamp_buckets.buckets), - fold(throwErrors(createPlainError), identity) - ); + if (mlModelPlotResponse._shards.total === 0) { + throw new NoLogRateResultsIndexError( + `Failed to find ml result index for job ${logRateJobId}.` + ); + } - return mlModelPlotBuckets.map(bucket => ({ - anomalies: bucket.filter_records.top_hits_record.hits.hits.map(({ _source: record }) => ({ - actualLogEntryRate: record.actual[0], - anomalyScore: record.record_score, - duration: record.bucket_span * 1000, - startTime: record.timestamp, - typicalLogEntryRate: record.typical[0], - })), - duration: bucketDuration, - logEntryRateStats: bucket.filter_model_plot.stats_actual, - modelLowerBoundStats: bucket.filter_model_plot.stats_model_lower, - modelUpperBoundStats: bucket.filter_model_plot.stats_model_upper, - startTime: bucket.key, - })); - } -} + const { after_key: afterKey, buckets: latestBatchBuckets } = pipe( + logRateModelPlotResponseRT.decode(mlModelPlotResponse), + map(response => response.aggregations.timestamp_data_set_buckets), + fold(throwErrors(createPlainError), identity) + ); -const logRateMlRecordRT = rt.type({ - actual: rt.array(rt.number), - bucket_span: rt.number, - record_score: rt.number, - timestamp: rt.number, - typical: rt.array(rt.number), -}); + mlModelPlotBuckets = [...mlModelPlotBuckets, ...latestBatchBuckets]; + afterLatestBatchKey = afterKey; -const logRateStatsAggregationRT = rt.type({ - avg: rt.union([rt.number, rt.null]), - count: rt.number, - max: rt.union([rt.number, rt.null]), - min: rt.union([rt.number, rt.null]), - sum: rt.number, -}); + if (latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) { + break; + } + } -const logRateModelPlotResponseRT = rt.type({ - aggregations: rt.type({ - timestamp_buckets: rt.type({ - buckets: rt.array( - rt.type({ - key: rt.number, - filter_records: rt.type({ - doc_count: rt.number, - top_hits_record: rt.type({ - hits: rt.type({ - hits: rt.array( - rt.type({ - _source: logRateMlRecordRT, - }) - ), - }), - }), - }), - filter_model_plot: rt.type({ - doc_count: rt.number, - stats_actual: logRateStatsAggregationRT, - stats_model_lower: logRateStatsAggregationRT, - stats_model_upper: logRateStatsAggregationRT, - }), - }) - ), - }), - }), -}); + return mlModelPlotBuckets.reduce< + Array<{ + dataSets: Array<{ + analysisBucketCount: number; + anomalies: Array<{ + actualLogEntryRate: number; + anomalyScore: number; + duration: number; + startTime: number; + typicalLogEntryRate: number; + }>; + averageActualLogEntryRate: number; + dataSetId: string; + }>; + startTime: number; + }> + >((histogramBuckets, timestampDataSetBucket) => { + const previousHistogramBucket = histogramBuckets[histogramBuckets.length - 1]; + const dataSet = { + analysisBucketCount: timestampDataSetBucket.filter_model_plot.doc_count, + anomalies: timestampDataSetBucket.filter_records.top_hits_record.hits.hits.map( + ({ _source: record }) => ({ + actualLogEntryRate: record.actual[0], + anomalyScore: record.record_score, + duration: record.bucket_span * 1000, + startTime: record.timestamp, + typicalLogEntryRate: record.typical[0], + }) + ), + averageActualLogEntryRate: timestampDataSetBucket.filter_model_plot.average_actual.value, + dataSetId: timestampDataSetBucket.key.data_set, + }; + if ( + previousHistogramBucket && + previousHistogramBucket.startTime === timestampDataSetBucket.key.timestamp + ) { + return [ + ...histogramBuckets.slice(0, -1), + { + ...previousHistogramBucket, + dataSets: [...previousHistogramBucket.dataSets, dataSet], + }, + ]; + } else { + return [ + ...histogramBuckets, + { + dataSets: [dataSet], + startTime: timestampDataSetBucket.key.timestamp, + }, + ]; + } + }, []); + } +} diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts new file mode 100644 index 00000000000000..17494212777198 --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './log_entry_rate'; diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts new file mode 100644 index 00000000000000..b10b1fe04db24f --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; + +const ML_ANOMALY_INDEX_PREFIX = '.ml-anomalies-'; + +export const createLogEntryRateQuery = ( + logRateJobId: string, + startTime: number, + endTime: number, + bucketDuration: number, + size: number, + afterKey?: CompositeTimestampDataSetKey +) => ({ + allowNoIndices: true, + body: { + query: { + bool: { + filter: [ + { + range: { + timestamp: { + gte: startTime, + lt: endTime, + }, + }, + }, + { + terms: { + result_type: ['model_plot', 'record'], + }, + }, + { + term: { + detector_index: { + value: 0, + }, + }, + }, + ], + }, + }, + aggs: { + timestamp_data_set_buckets: { + composite: { + after: afterKey, + size, + sources: [ + { + timestamp: { + date_histogram: { + field: 'timestamp', + fixed_interval: `${bucketDuration}ms`, + order: 'asc', + }, + }, + }, + { + data_set: { + terms: { + field: 'partition_field_value', + order: 'asc', + }, + }, + }, + ], + }, + aggs: { + filter_model_plot: { + filter: { + term: { + result_type: 'model_plot', + }, + }, + aggs: { + average_actual: { + avg: { + field: 'actual', + }, + }, + }, + }, + filter_records: { + filter: { + term: { + result_type: 'record', + }, + }, + aggs: { + top_hits_record: { + top_hits: { + _source: Object.keys(logRateMlRecordRT.props), + size: 100, + sort: [ + { + timestamp: 'asc', + }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + ignoreUnavailable: true, + index: `${ML_ANOMALY_INDEX_PREFIX}${logRateJobId}`, + size: 0, + trackScores: false, + trackTotalHits: false, +}); + +const logRateMlRecordRT = rt.type({ + actual: rt.array(rt.number), + bucket_span: rt.number, + record_score: rt.number, + timestamp: rt.number, + typical: rt.array(rt.number), +}); + +const metricAggregationRT = rt.type({ + value: rt.number, +}); + +const compositeTimestampDataSetKeyRT = rt.type({ + data_set: rt.string, + timestamp: rt.number, +}); + +export type CompositeTimestampDataSetKey = rt.TypeOf; + +export const logRateModelPlotBucketRT = rt.type({ + key: compositeTimestampDataSetKeyRT, + filter_records: rt.type({ + doc_count: rt.number, + top_hits_record: rt.type({ + hits: rt.type({ + hits: rt.array( + rt.type({ + _source: logRateMlRecordRT, + }) + ), + }), + }), + }), + filter_model_plot: rt.type({ + doc_count: rt.number, + average_actual: metricAggregationRT, + }), +}); + +export type LogRateModelPlotBucket = rt.TypeOf; + +export const logRateModelPlotResponseRT = rt.type({ + aggregations: rt.type({ + timestamp_data_set_buckets: rt.intersection([ + rt.type({ + buckets: rt.array(logRateModelPlotBucketRT), + }), + rt.partial({ + after_key: compositeTimestampDataSetKeyRT, + }), + ]), + }), +}); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 120afca8e8742d..c2af0a91db5f92 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -5319,11 +5319,8 @@ "xpack.infra.header.logsTitle": "ログ", "xpack.infra.homePage.settingsTabTitle": "設定", "xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage": "{metricId} のモデルには cloudId が必要ですが、{nodeId} に cloudId が指定されていません。", - "xpack.infra.logs.analysis.logRateSectionAnomalySeriesName": "異常", - "xpack.infra.logs.analysis.logRateSectionAreaSeriesName": "期待値", "xpack.infra.logs.analysis.logRateSectionLineSeriesName": "15 分ごとのログエントリー (平均)", "xpack.infra.logs.analysis.logRateSectionLoadingAriaLabel": "ログレートの結果を読み込み中", - "xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel": "モデルバウンドを表示", "xpack.infra.logs.analysis.logRateSectionNoDataBody": "時間範囲を調整する必要があるかもしれません。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "表示するデータがありません。", "xpack.infra.logs.analysis.logRateSectionTitle": "ログレート", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 2082d22ebb12b7..68ae356337ccca 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -5322,11 +5322,8 @@ "xpack.infra.header.logsTitle": "Logs", "xpack.infra.homePage.settingsTabTitle": "设置", "xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage": "{metricId} 的模型需要云 ID，但没有为 {nodeId} 提供。", - "xpack.infra.logs.analysis.logRateSectionAnomalySeriesName": "异常", - "xpack.infra.logs.analysis.logRateSectionAreaSeriesName": "预期", "xpack.infra.logs.analysis.logRateSectionLineSeriesName": "每 15 分钟日志条目数（平均值）", "xpack.infra.logs.analysis.logRateSectionLoadingAriaLabel": "正在加载日志速率结果", - "xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel": "显示模型边界", "xpack.infra.logs.analysis.logRateSectionNoDataBody": "您可能想调整时间范围。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "没有可显示的数据。", "xpack.infra.logs.analysis.logRateSectionTitle": "日志速率", diff --git a/x-pack/test/api_integration/apis/infra/log_analysis.ts b/x-pack/test/api_integration/apis/infra/log_analysis.ts index bd09cdf6ff56ee..fe7d55649d1d63 100644 --- a/x-pack/test/api_integration/apis/infra/log_analysis.ts +++ b/x-pack/test/api_integration/apis/infra/log_analysis.ts @@ -20,8 +20,8 @@ import { } from '../../../../legacy/plugins/infra/common/runtime_types'; import { FtrProviderContext } from '../../ftr_provider_context'; -const TIME_BEFORE_START = 1564315100000; -const TIME_AFTER_END = 1565040700000; +const TIME_BEFORE_START = 1569934800000; +const TIME_AFTER_END = 1570016700000; const COMMON_HEADERS = { 'kbn-xsrf': 'some-xsrf-token', }; @@ -32,8 +32,8 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); describe('log analysis apis', () => { - before(() => esArchiver.load('infra/8.0.0/ml_anomalies_log_rate')); - after(() => esArchiver.unload('infra/8.0.0/ml_anomalies_log_rate')); + before(() => esArchiver.load('infra/8.0.0/ml_anomalies_partitioned_log_rate')); + after(() => esArchiver.unload('infra/8.0.0/ml_anomalies_partitioned_log_rate')); describe('log rate results', () => { describe('with the default source', () => { @@ -62,11 +62,12 @@ export default ({ getService }: FtrProviderContext) => { getLogEntryRateSuccessReponsePayloadRT.decode(body), fold(throwErrors(createPlainError), identity) ); - expect(logEntryRateBuckets.data.bucketDuration).to.be(15 * 60 * 1000); expect(logEntryRateBuckets.data.histogramBuckets).to.not.be.empty(); expect( - logEntryRateBuckets.data.histogramBuckets.some(bucket => bucket.anomalies.length > 0) + logEntryRateBuckets.data.histogramBuckets.some(bucket => { + return bucket.dataSets.some(dataSet => dataSet.anomalies.length > 0); + }) ).to.be(true); }); diff --git a/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz b/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..8d15ff8ccb02265ae0e1f7268ff1fc90035d2258 GIT binary patch literal 6298 zcmYkAXE@t$)W?n1rmw;pcF+B)NE~`gqjsw z?e*|~o)^#k@^{X4uHSvu_jBDWu|zm`2akF?ThoK7sjlzjnuh=F$e&(*PyDNa{SyzJ~;F5HN0Z$e5X#fgbH- z4(1AjK9?2_<_-{AMkZez^jn^#U1CpLn6H8^U;9EXpJZH1-T2C0`_2bGTF|A7mpxo( z$(rlgT3UYd)saAQ9BVK7HA{;gpY?I9y5gBw?PUlMhew^5*cRuV->7@`9Cq!edwFG6 zf8WyfKl>*Pl85n!n8^Aye?3Q;w%oWJ7f^^Ov(Qdd@+I+Q!`a68)y9cEI2}Xodboa; z6t7y7r7UKmErJj`12{}vcCkALFh1F`(dG(xxBNZ^mZcbeM+o)l2JkN0*v|_{$>QRfpkcvR77xsikzm{B=N#_+ii9T1|sW zD?06ddrg*`N2ENZ+4TlC>NAVX<&lYonX|QSM36y~;F(Bze92%hMe83XOz_A_Fl1mj z`@xMz&_7EaM^CiH`(p1#ed+e$VTn&w}<7^eK19rQeebN#W9+Rmu%&Wv^D;6b==+boub~dGDhx9R99zc6gjT( zg7|47fi1;KCe+#5H{VFJ)HMRSA0(Abk%H6(SJGhB!GO?|mx~G2;6)wSeFD6ZYulFu zVi+pGt43en!HT-ye(q5eDV*XzxpQ! z2_fHv+C9d-7ajJ|2+GbVQ3!tuR>CAFukfd~3yp8;{4=-iO?b#X9k$+`@NKbrrw_`OAyh{7LXiY&XR1vQ zJ{oaO0IjD>RCJ}$WuuyNXpdjiQIEM`@R-qdZM!!xlxwi2NS!KWqF)3wN{fCjHg=nn zX@K6V&hxh!hLEjq-s$J0Q<7C2&>%MMPjLcTU_QbGoyZsBT#3AL>A!peBhF%O&G=9q zp#4t45ucm{%3oAVzf;^EBA75n>5!zyDVJIY;v?86L&kwV$yEuBJHfIivpjE$2a_+u zhqkq;6DPROQ5A7^Wdk%dlE3l?8xI$pt=NvC-NuiSzHb^#n~yf1L&cBJZt~Onv`pr{VZin_-=Kif9nO~6=~7-Vvksp?WjJLrqg?1RvkvnY- zX6kkEjNg3!3a43!%(brAkEX_ShdA3h1#eGHfQxQWRhG|_3@z2}Nz=#7x}s8Rge7nM zC3UU?(fot6Z>3SxN|{-Ix2(@TofI#DN;`H%H^;jR_@YFOp-(*D{JMPVl!;wC$N6D+ zf9h2R+_iUb)t!xxIBJ>5&nc982(BqM>qv;Y=dLacd;(o4g1GgLMK0!TgHc=l-eQiD-uyBp|UCqf>BcSfDSXCr} zPsbJ;_R*J^kKj3PmXdYMnt%OB9O#=I9BV=)46yNWR#$^Y7+J!K_1CDHCzzd1O8x@1 zdF8Yc4lLWr!SBAL6jhH6)E-hy$Fvr678$kmFX-tsn@8x|te_I(ZG(x}zL6}+fVE&C6%4suejMpRcNkXOwRBhxD%2c#rwd`;h-H~7!{*2g!@$eq6C zr+)X*?Y_cI38!*qpu|2#0YvO7h4>t@J_RvBSNKgjs>g0;&qodZTNx|XCsl!&15j)A zbIw_@m(6#dKmTJYWbX`z3RGY?x@Qa;f=!FPjBdVp16yOK9Lh(}+Zebh`4!#KR4}4S5706lvr!=88js%2?C5sS`Z# zA$_&0A=?x^6zm@Ty(y26|Amjtx+r7nyHaBOi-mHJLQ+XUJqcJC;6Yy{ERUVAb=4dK zvc=e4Xh0o`oFjp5wp<9_47-~7{{P!QQj&gC&38q651eqjS^zAUr;13&Qq_BGPcs|yo0^lf8d1Z$=7m^AC z6l)fWsA2i7jzQ609#_|jP!cubl>5}~J_-)pwRKk%X`V^#O^Uxax=2ybc{2x9PzPZ7(XftY7>Q7)zvKOD{Qmx-NS4DOUw)+O|WGnFS?e6)cYPSU??x>o{tSxtR>`0#yoML-1domFMeCV-Z zeezn`o`0E_9sPuTSqq+l9%;o%KacmL&Ym~loQYqG=fLS7D8G8e7T`|wI)l#Q{X1uu zy_NIXu+Gf@p3uc|RQJ4Z!(rE!*q&l*`Ooh7)aNG*b(0DLIku)W&9b`f)+-Hezb3bS z1`0qHevE@^DYa4rh!uv6E!ldfjy19*5Bx0s%`4xmSF=vq8Z8j9f0?e%@bxHb0G2Jnp$GH2p%!OA%QWcwqg)~ z`R!e1cXIqq5>+mpTI)g)MHW*%FXZQlKBI= zQ||#h5?K>c%y}km!6^2STOvOvt)Ko~lai(QgamC6r9E%4&r1#j$KQWtoa9{8rY>TC zAS5d0)xTUT8*0j*2zQI0+87O&P6u2+F|~XVZv5-VlA4q>Fo&vOPDWZYi4ggm58`k~zd)KPBdKG)+|iAp-cXq6F)x8czD0;0Tr=Fei|VG2 zvb8>(B-{zDoTD9dd#k(N&*Pr(fX3XB9;y~Jy6YN)D5fGwnC}M`&(YWr^V3OCaUMwX zHjYNn!>r8727Jnz7>$XTd_2ikiV#Gty^aS z>pS`4k{@NwWQ#~_5d!}yiqS$N?)`g+ZMpjiN<-LFBBM&L zwKdv5n2px3ao4aP&xuw8AfF+8oh&FPown=x`$wxa`*flOg70xF9CeMGHU20+1+G}tjaMJ*&~*TDvbt#1Im5Vi|$ z2~HTeYFz_qR;To*wy&kqxxd z*mb_p%4}hv%b-eIsqI&Q*XV7&wO#sP{T6Wxf>mT5cf5@|i4(ivovxt%D@P+Q-TNYm z(mg`4aU$IA=lD_CwF;dDkU)P~An-KlR%!c0su+&W=d1_V6?$?fJ|L-~=?_dIAF344 znC)64|0}R|`FamP(P!zL&gLNiON=aXADY~t0v~L;+Ac*{gQsx8 zJY=!Aj45P&E48A6b5`JGfn@l;QJ#RUHv<8g)rS#sN0{``e)EQQN`2T-1Ws(@f3kCOwdlzzkrCibTeIT7tf{p(0g}Et8yDOPBFsB6&Aso_?|WG=;}8yG69dY zZ_BU5s#6m6pBUcB*2C%b#RpKss9W_W>v^Kg9@CZQeh)f;w&&2Z0pHU5n|4-vFRh_y zuHM^-lF^#{sCE|UsIPi)Iy+AtN2*bY8TtcZRaNaj>005Mf}{T$Kp2HGZ@9J>axlM~ z@1`1*V4&f8odhp)iR6<{pur>2^K2vid6(9bm(#(JeeOxihW1Ivs|7Mr6|=f8NTqr23+QJ!$G5ur0KeV&g9Y!A22$)cHx=RCFBx z=f7R}C~sSn?xUX$u(P{q%!|aHL;-DK^7rtHFYB40M%+B=l&nf>HEaa^w0G8kT2vsf zP4VK{sq8Q-W)|8$MwB9GliwsWH4OQrRZ_ULDa7=VCxiJA-zK|_i6^A}4MKa{<J)6vY?sBNh7yhI0!`nq6uhxV^sw(GTRN$) zi{3bTnOYB3@KvzgQ`z5EPdbAd4WBLcBy)Q7iA*tu&WAh^Oej;v)t44^0O2AytqW}5 zw419Nz5>`(Q_kqH_HI%@*(U*5S>{wCsBO2XboqQL&@N3cL+Hj$MwstUjP8Dv*yZ`diGWVes}NfX#${*VO0i$`KUpf z)@S_U-R`f&mW+7&<<^hTDC5BW)8CK?otFzsyMkH%7wq%#Ujv8wLPF*P_Q?09<2sh4 zXANwr>8$!6%Z0Oa-DCy2efz83C+TC%fA)qYWF|K2B3HLtce&qp|GB3ZEglyEn>v4| zktxS3p6O`{GD8dm_nt1-qn+Nud3yW!FqT>Ephv+CA6kAURMVO4-+P{Lak#>yqC-k( zK72jYRJR=WY#T2=d{m$=l-E81+K9BPl^K2QI6$NSw1v1(g`S@_%@Z+}$#}50i&# zRE0(ssLfeQt-#1A9Ix)RHHn9mcw2F+IJsEA)F!+Nlw+Ht|!RtDTl>C(c%(%448yyaXYwXKaIq zAIV`n{-drai9o?%T5o{K_|uE>vz!>|u?4r~R>Wy>26;#jYR}qvoxfbA(+X^*O1(YYs2GGsy9^-xjV~`5f@OF zCTUy9Zk@^s?+sAgFk^mWn5#7%+g{(|q2a!chqtDq1%WW@6p(}kOhfVf>-1}zVXt$% z`uGpXq8y&$@ALKXJyaiT(V<#&LBayFcT`ckCUt}d_H2^BkDA8hvA10m9dc<&1kBj^ z70o6Rkw7V<4ec3wjt+L>1iy(0wzrY=wBE!sv+q0xw}VDjjnYY}@$%(J9#u`*73BvHA`_c&89| z@l4joEblH{I#OjjeqT3mH6NJZ@KI4EKOYhrB6`X7itEHANq~EOP;CdQ>H(T0_*dW* zyRFyfS{~Qpf-0TGv2kH8qDi*HyyR7X#@WPaXPPsDRT|?gHeEy8~Y$?n1U(QS_gJr%X`|bz_Zk#2ZPpglNALr~7s>{f z7dwwUJ*_6r_V)(b&`j1(d`X=hCdL zzv8qza{DrPZ_HF z$q*;%lQCnZY0zu=h61CgdiTicjP7dG6j;b-(Mf${&1GL?<|#)5?mM3j-%pbGMD~!r z#`Ofbck8F;h7|Naj-=W_2IJo}TDq-kV(i*nwI8|L8r04Ak5^aB=n{ikq~Ev5s