diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts index 2dcaf35cc41d9f..3af07980910b8a 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts @@ -6,13 +6,7 @@ import * as rt from 'io-ts'; -import { - badRequestErrorRT, - conflictErrorRT, - forbiddenErrorRT, - metricStatisticsRT, - timeRangeRT, -} from '../../shared'; +import { badRequestErrorRT, conflictErrorRT, forbiddenErrorRT, timeRangeRT } from '../../shared'; export const LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH = '/api/infra/log_analysis/results/log_entry_rate'; @@ -43,12 +37,15 @@ export const logEntryRateAnomaly = rt.type({ typicalLogEntryRate: rt.number, }); -export const logEntryRateHistogramBucket = rt.type({ +export const logEntryRateDataSetRT = rt.type({ + analysisBucketCount: rt.number, anomalies: rt.array(logEntryRateAnomaly), - duration: rt.number, - logEntryRateStats: metricStatisticsRT, - modelLowerBoundStats: metricStatisticsRT, - modelUpperBoundStats: metricStatisticsRT, + averageActualLogEntryRate: rt.number, + dataSetId: rt.string, +}); + +export const logEntryRateHistogramBucket = rt.type({ + dataSets: rt.array(logEntryRateDataSetRT), startTime: rt.number, }); diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts new file mode 100644 index 00000000000000..471a00d40984cf --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/get_log_entry_rate.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { identity } from 'fp-ts/lib/function'; +import { kfetch } from 'ui/kfetch'; + +import { + getLogEntryRateRequestPayloadRT, + getLogEntryRateSuccessReponsePayloadRT, + LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, +} from '../../../../../common/http_api/log_analysis'; +import { createPlainError, throwErrors } from '../../../../../common/runtime_types'; + +export const callGetLogEntryRateAPI = async ( + sourceId: string, + startTime: number, + endTime: number, + bucketDuration: number +) => { + const response = await kfetch({ + method: 'POST', + pathname: LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, + body: JSON.stringify( + getLogEntryRateRequestPayloadRT.encode({ + data: { + sourceId, + timeRange: { + startTime, + endTime, + }, + bucketDuration, + }, + }) + ), + }); + return pipe( + getLogEntryRateSuccessReponsePayloadRT.decode(response), + fold(throwErrors(createPlainError), identity) + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx deleted file mode 100644 index f54402a1a87073..00000000000000 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate.tsx +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { useMemo } from 'react'; -import { GetLogEntryRateSuccessResponsePayload } from '../../../../../common/http_api/log_analysis'; - -interface LogRateAreaSeriesDataPoint { - x: number; - min: number | null; - max: number | null; -} -type LogRateAreaSeries = LogRateAreaSeriesDataPoint[]; -type LogRateLineSeriesDataPoint = [number, number | null]; -type LogRateLineSeries = LogRateLineSeriesDataPoint[]; -type LogRateAnomalySeriesDataPoint = [number, number]; -type LogRateAnomalySeries = LogRateAnomalySeriesDataPoint[]; - -export const useLogEntryRateGraphData = ({ - data, -}: { - data: GetLogEntryRateSuccessResponsePayload['data'] | null; -}) => { - const areaSeries: LogRateAreaSeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - acc.push({ - x: bucket.startTime, - min: bucket.modelLowerBoundStats.min, - max: bucket.modelUpperBoundStats.max, - }); - return acc; - }, []); - }, [data]); - - const lineSeries: LogRateLineSeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - acc.push([bucket.startTime, bucket.logEntryRateStats.avg]); - return acc; - }, []); - }, [data]); - - const anomalySeries: LogRateAnomalySeries = useMemo(() => { - if (!data || (data && data.histogramBuckets && !data.histogramBuckets.length)) { - return []; - } - return data.histogramBuckets.reduce((acc: any, bucket) => { - if (bucket.anomalies.length > 0) { - bucket.anomalies.forEach(anomaly => { - acc.push([anomaly.startTime, anomaly.actualLogEntryRate]); - }); - return acc; - } else { - return acc; - } - }, []); - }, [data]); - - return { - areaSeries, - lineSeries, - anomalySeries, - }; -}; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx index 4e7a6647a95793..8b21a7e8298944 100644 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_entry_rate.tsx @@ -5,19 +5,10 @@ */ import { useMemo, useState } from 'react'; -import { kfetch } from 'ui/kfetch'; -import { fold } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { identity } from 'fp-ts/lib/function'; -import { - getLogEntryRateRequestPayloadRT, - getLogEntryRateSuccessReponsePayloadRT, - GetLogEntryRateSuccessResponsePayload, - LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, -} from '../../../../common/http_api/log_analysis'; -import { createPlainError, throwErrors } from '../../../../common/runtime_types'; +import { GetLogEntryRateSuccessResponsePayload } from '../../../../common/http_api/log_analysis'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { callGetLogEntryRateAPI } from './api/get_log_entry_rate'; type LogEntryRateResults = GetLogEntryRateSuccessResponsePayload['data']; @@ -38,30 +29,10 @@ export const useLogEntryRate = ({ { cancelPreviousOn: 'resolution', createPromise: async () => { - return await kfetch({ - method: 'POST', - pathname: LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, - body: JSON.stringify( - getLogEntryRateRequestPayloadRT.encode({ - data: { - sourceId, - timeRange: { - startTime, - endTime, - }, - bucketDuration, - }, - }) - ), - }); + return await callGetLogEntryRateAPI(sourceId, startTime, endTime, bucketDuration); }, onResolve: response => { - const { data } = pipe( - getLogEntryRateSuccessReponsePayloadRT.decode(response), - fold(throwErrors(createPlainError), identity) - ); - - setLogEntryRate(data); + setLogEntryRate(response.data); }, }, [sourceId, startTime, endTime, bucketDuration] diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx deleted file mode 100644 index df0eca449bb9f8..00000000000000 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/chart_helpers/index.tsx +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import chrome from 'ui/chrome'; -import { SpecId, Theme, LIGHT_THEME, DARK_THEME } from '@elastic/charts'; - -export const getColorsMap = (color: string, specId: SpecId) => { - const map = new Map(); - map.set({ colorValues: [], specId }, color); - return map; -}; - -export const isDarkMode = () => chrome.getUiSettingsClient().get('theme:darkMode'); - -export const getChartTheme = (): Theme => { - return isDarkMode() ? DARK_THEME : LIGHT_THEME; -}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx index 3629413d6d30c5..aaf24c22594e52 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx @@ -6,7 +6,6 @@ import datemath from '@elastic/datemath'; import { - EuiBadge, EuiFlexGroup, EuiFlexItem, EuiPage, @@ -17,7 +16,6 @@ import { EuiSuperDatePicker, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n/react'; import moment from 'moment'; import React, { useCallback, useMemo, useState } from 'react'; @@ -122,21 +120,6 @@ export const AnalysisResultsContent = ({ [setAutoRefresh] ); - const anomaliesDetected = useMemo(() => { - if (!logEntryRate) { - return null; - } else { - if (logEntryRate.histogramBuckets && logEntryRate.histogramBuckets.length) { - return logEntryRate.histogramBuckets.reduce( - (acc, bucket) => acc + bucket.anomalies.length, - 0 - ); - } else { - return null; - } - } - }, [logEntryRate]); - return ( <> {isLoading && !logEntryRate ? ( @@ -150,29 +133,8 @@ export const AnalysisResultsContent = ({ - - - - {anomaliesDetected !== null ? ( - - - {anomaliesDetected} - - ), - number: anomaliesDetected, - }} - /> - - ) : null} - - - - + + void; + timeRange: TimeRange; +}> = ({ bucketDuration, histogramBuckets, setTimeRange, timeRange }) => { + const [dateFormat] = useKibanaUiSetting('dateFormat'); + const [isDarkMode] = useKibanaUiSetting('theme:darkMode'); + + const chartDateFormatter = useMemo( + () => niceTimeFormatter([timeRange.startTime, timeRange.endTime]), + [timeRange] + ); + + const logEntryRateSeries = useMemo( + () => + histogramBuckets + ? histogramBuckets.reduce>( + (buckets, bucket) => { + return [ + ...buckets, + ...bucket.dataSets.map(dataSet => ({ + group: dataSet.dataSetId === '' ? 'unknown' : dataSet.dataSetId, + time: bucket.startTime, + value: dataSet.averageActualLogEntryRate, + })), + ]; + }, + [] + ) + : [], + [histogramBuckets] + ); + + const logEntryRateAnomalyAnnotations = useMemo( + () => + histogramBuckets + ? histogramBuckets.reduce((annotatedBuckets, bucket) => { + const anomalies = bucket.dataSets.reduce( + (accumulatedAnomalies, dataSet) => [...accumulatedAnomalies, ...dataSet.anomalies], + [] + ); + if (anomalies.length <= 0) { + return annotatedBuckets; + } + return [ + ...annotatedBuckets, + { + coordinates: { + x0: bucket.startTime, + x1: bucket.startTime + bucketDuration, + }, + details: i18n.translate( + 'xpack.infra.logs.analysis.logRateSectionAnomalyCountTooltipLabel', + { + defaultMessage: `{anomalyCount, plural, one {# anomaly} other {# anomalies}}`, + values: { + anomalyCount: anomalies.length, + }, + } + ), + }, + ]; + }, []) + : [], + [histogramBuckets] + ); + + const logEntryRateSpecId = getSpecId('averageValues'); + const logEntryRateAnomalyAnnotationsId = getAnnotationId('anomalies'); + + const tooltipProps = useMemo( + () => ({ + headerFormatter: (tooltipData: TooltipValue) => + moment(tooltipData.value).format(dateFormat || 'Y-MM-DD HH:mm:ss.SSS'), + }), + [dateFormat] + ); + + const handleBrushEnd = useCallback( + (startTime: number, endTime: number) => { + setTimeRange({ + endTime, + startTime, + }); + }, + [setTimeRange] + ); + + return ( +
+ + + Number(value).toFixed(0)} + /> + + + + +
+ ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx deleted file mode 100644 index 0d703420e7412a..00000000000000 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/chart.tsx +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { useMemo, useCallback, useState } from 'react'; -import { i18n } from '@kbn/i18n'; -import moment from 'moment'; -import { - Axis, - Chart, - getAxisId, - getSpecId, - AreaSeries, - LineSeries, - niceTimeFormatter, - Settings, - TooltipValue, -} from '@elastic/charts'; -import { EuiFlexGroup, EuiFlexItem, EuiCheckbox } from '@elastic/eui'; -import { getColorsMap, isDarkMode, getChartTheme } from '../../chart_helpers'; -import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; -import { useLogEntryRateGraphData } from '../../../../../containers/logs/log_analysis/log_analysis_graph_data/log_entry_rate'; -import { useKibanaUiSetting } from '../../../../../utils/use_kibana_ui_setting'; -import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; - -const areaSeriesColour = 'rgb(224, 237, 255)'; -const lineSeriesColour = 'rgb(49, 133, 252)'; - -interface Props { - data: GetLogEntryRateSuccessResponsePayload['data'] | null; - setTimeRange: (timeRange: TimeRange) => void; - timeRange: TimeRange; -} - -export const ChartView = ({ data, setTimeRange, timeRange }: Props) => { - const { areaSeries, lineSeries, anomalySeries } = useLogEntryRateGraphData({ data }); - - const dateFormatter = useMemo( - () => - lineSeries.length > 0 - ? niceTimeFormatter([timeRange.startTime, timeRange.endTime]) - : (value: number) => `${value}`, - [lineSeries, timeRange] - ); - - const areaSpecId = getSpecId('modelBounds'); - const lineSpecId = getSpecId('averageValues'); - const anomalySpecId = getSpecId('anomalies'); - - const [dateFormat] = useKibanaUiSetting('dateFormat'); - - const tooltipProps = useMemo( - () => ({ - headerFormatter: (tooltipData: TooltipValue) => - moment(tooltipData.value).format(dateFormat || 'Y-MM-DD HH:mm:ss.SSS'), - }), - [dateFormat] - ); - - const [isShowingModelBounds, setIsShowingModelBounds] = useState(true); - - const handleBrushEnd = useCallback( - (startTime: number, endTime: number) => { - setTimeRange({ - endTime, - startTime, - }); - }, - [setTimeRange] - ); - - return ( - <> - - - { - setIsShowingModelBounds(e.target.checked); - }} - /> - - -
- - - Number(value).toFixed(0)} - /> - {isShowingModelBounds ? ( - - ) : null} - - - - -
- - ); -}; - -const showModelBoundsLabel = i18n.translate( - 'xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel', - { defaultMessage: 'Show model bounds' } -); diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx index c03a4817e4d4c1..1f01af33e33c4f 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx @@ -16,8 +16,8 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; -import { ChartView } from './chart'; import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { LogEntryRateBarChart } from './bar_chart'; export const LogRateResults = ({ isLoading, @@ -70,7 +70,12 @@ export const LogRateResults = ({ } /> ) : ( - + )} ); diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts index ac7f7b6df8d62f..31d9c5403e2d2b 100644 --- a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts @@ -4,8 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as rt from 'io-ts'; - import { pipe } from 'fp-ts/lib/pipeable'; import { map, fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; @@ -13,8 +11,14 @@ import { getJobId } from '../../../common/log_analysis'; import { throwErrors, createPlainError } from '../../../common/runtime_types'; import { InfraBackendFrameworkAdapter, InfraFrameworkRequest } from '../adapters/framework'; import { NoLogRateResultsIndexError } from './errors'; +import { + logRateModelPlotResponseRT, + createLogEntryRateQuery, + LogRateModelPlotBucket, + CompositeTimestampDataSetKey, +} from './queries'; -const ML_ANOMALY_INDEX_PREFIX = '.ml-anomalies-'; +const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; export class InfraLogAnalysis { constructor( @@ -38,168 +42,95 @@ export class InfraLogAnalysis { ) { const logRateJobId = this.getJobIds(request, sourceId).logEntryRate; - const mlModelPlotResponse = await this.libs.framework.callWithRequest(request, 'search', { - allowNoIndices: true, - body: { - query: { - bool: { - filter: [ - { - range: { - timestamp: { - gte: startTime, - lt: endTime, - }, - }, - }, - { - terms: { - result_type: ['model_plot', 'record'], - }, - }, - { - term: { - detector_index: { - value: 0, - }, - }, - }, - ], - }, - }, - aggs: { - timestamp_buckets: { - date_histogram: { - field: 'timestamp', - fixed_interval: `${bucketDuration}ms`, - }, - aggs: { - filter_model_plot: { - filter: { - term: { - result_type: 'model_plot', - }, - }, - aggs: { - stats_model_lower: { - stats: { - field: 'model_lower', - }, - }, - stats_model_upper: { - stats: { - field: 'model_upper', - }, - }, - stats_actual: { - stats: { - field: 'actual', - }, - }, - }, - }, - filter_records: { - filter: { - term: { - result_type: 'record', - }, - }, - aggs: { - top_hits_record: { - top_hits: { - _source: Object.keys(logRateMlRecordRT.props), - size: 100, - sort: [ - { - timestamp: 'asc', - }, - ], - }, - }, - }, - }, - }, - }, - }, - }, - ignoreUnavailable: true, - index: `${ML_ANOMALY_INDEX_PREFIX}${logRateJobId}`, - size: 0, - trackScores: false, - trackTotalHits: false, - }); + let mlModelPlotBuckets: LogRateModelPlotBucket[] = []; + let afterLatestBatchKey: CompositeTimestampDataSetKey | undefined; - if (mlModelPlotResponse._shards.total === 0) { - throw new NoLogRateResultsIndexError( - `Failed to find ml result index for job ${logRateJobId}.` + while (true) { + const mlModelPlotResponse = await this.libs.framework.callWithRequest( + request, + 'search', + createLogEntryRateQuery( + logRateJobId, + startTime, + endTime, + bucketDuration, + COMPOSITE_AGGREGATION_BATCH_SIZE, + afterLatestBatchKey + ) ); - } - const mlModelPlotBuckets = pipe( - logRateModelPlotResponseRT.decode(mlModelPlotResponse), - map(response => response.aggregations.timestamp_buckets.buckets), - fold(throwErrors(createPlainError), identity) - ); + if (mlModelPlotResponse._shards.total === 0) { + throw new NoLogRateResultsIndexError( + `Failed to find ml result index for job ${logRateJobId}.` + ); + } - return mlModelPlotBuckets.map(bucket => ({ - anomalies: bucket.filter_records.top_hits_record.hits.hits.map(({ _source: record }) => ({ - actualLogEntryRate: record.actual[0], - anomalyScore: record.record_score, - duration: record.bucket_span * 1000, - startTime: record.timestamp, - typicalLogEntryRate: record.typical[0], - })), - duration: bucketDuration, - logEntryRateStats: bucket.filter_model_plot.stats_actual, - modelLowerBoundStats: bucket.filter_model_plot.stats_model_lower, - modelUpperBoundStats: bucket.filter_model_plot.stats_model_upper, - startTime: bucket.key, - })); - } -} + const { after_key: afterKey, buckets: latestBatchBuckets } = pipe( + logRateModelPlotResponseRT.decode(mlModelPlotResponse), + map(response => response.aggregations.timestamp_data_set_buckets), + fold(throwErrors(createPlainError), identity) + ); -const logRateMlRecordRT = rt.type({ - actual: rt.array(rt.number), - bucket_span: rt.number, - record_score: rt.number, - timestamp: rt.number, - typical: rt.array(rt.number), -}); + mlModelPlotBuckets = [...mlModelPlotBuckets, ...latestBatchBuckets]; + afterLatestBatchKey = afterKey; -const logRateStatsAggregationRT = rt.type({ - avg: rt.union([rt.number, rt.null]), - count: rt.number, - max: rt.union([rt.number, rt.null]), - min: rt.union([rt.number, rt.null]), - sum: rt.number, -}); + if (latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) { + break; + } + } -const logRateModelPlotResponseRT = rt.type({ - aggregations: rt.type({ - timestamp_buckets: rt.type({ - buckets: rt.array( - rt.type({ - key: rt.number, - filter_records: rt.type({ - doc_count: rt.number, - top_hits_record: rt.type({ - hits: rt.type({ - hits: rt.array( - rt.type({ - _source: logRateMlRecordRT, - }) - ), - }), - }), - }), - filter_model_plot: rt.type({ - doc_count: rt.number, - stats_actual: logRateStatsAggregationRT, - stats_model_lower: logRateStatsAggregationRT, - stats_model_upper: logRateStatsAggregationRT, - }), - }) - ), - }), - }), -}); + return mlModelPlotBuckets.reduce< + Array<{ + dataSets: Array<{ + analysisBucketCount: number; + anomalies: Array<{ + actualLogEntryRate: number; + anomalyScore: number; + duration: number; + startTime: number; + typicalLogEntryRate: number; + }>; + averageActualLogEntryRate: number; + dataSetId: string; + }>; + startTime: number; + }> + >((histogramBuckets, timestampDataSetBucket) => { + const previousHistogramBucket = histogramBuckets[histogramBuckets.length - 1]; + const dataSet = { + analysisBucketCount: timestampDataSetBucket.filter_model_plot.doc_count, + anomalies: timestampDataSetBucket.filter_records.top_hits_record.hits.hits.map( + ({ _source: record }) => ({ + actualLogEntryRate: record.actual[0], + anomalyScore: record.record_score, + duration: record.bucket_span * 1000, + startTime: record.timestamp, + typicalLogEntryRate: record.typical[0], + }) + ), + averageActualLogEntryRate: timestampDataSetBucket.filter_model_plot.average_actual.value, + dataSetId: timestampDataSetBucket.key.data_set, + }; + if ( + previousHistogramBucket && + previousHistogramBucket.startTime === timestampDataSetBucket.key.timestamp + ) { + return [ + ...histogramBuckets.slice(0, -1), + { + ...previousHistogramBucket, + dataSets: [...previousHistogramBucket.dataSets, dataSet], + }, + ]; + } else { + return [ + ...histogramBuckets, + { + dataSets: [dataSet], + startTime: timestampDataSetBucket.key.timestamp, + }, + ]; + } + }, []); + } +} diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts new file mode 100644 index 00000000000000..17494212777198 --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './log_entry_rate'; diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts new file mode 100644 index 00000000000000..b10b1fe04db24f --- /dev/null +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; + +const ML_ANOMALY_INDEX_PREFIX = '.ml-anomalies-'; + +export const createLogEntryRateQuery = ( + logRateJobId: string, + startTime: number, + endTime: number, + bucketDuration: number, + size: number, + afterKey?: CompositeTimestampDataSetKey +) => ({ + allowNoIndices: true, + body: { + query: { + bool: { + filter: [ + { + range: { + timestamp: { + gte: startTime, + lt: endTime, + }, + }, + }, + { + terms: { + result_type: ['model_plot', 'record'], + }, + }, + { + term: { + detector_index: { + value: 0, + }, + }, + }, + ], + }, + }, + aggs: { + timestamp_data_set_buckets: { + composite: { + after: afterKey, + size, + sources: [ + { + timestamp: { + date_histogram: { + field: 'timestamp', + fixed_interval: `${bucketDuration}ms`, + order: 'asc', + }, + }, + }, + { + data_set: { + terms: { + field: 'partition_field_value', + order: 'asc', + }, + }, + }, + ], + }, + aggs: { + filter_model_plot: { + filter: { + term: { + result_type: 'model_plot', + }, + }, + aggs: { + average_actual: { + avg: { + field: 'actual', + }, + }, + }, + }, + filter_records: { + filter: { + term: { + result_type: 'record', + }, + }, + aggs: { + top_hits_record: { + top_hits: { + _source: Object.keys(logRateMlRecordRT.props), + size: 100, + sort: [ + { + timestamp: 'asc', + }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + ignoreUnavailable: true, + index: `${ML_ANOMALY_INDEX_PREFIX}${logRateJobId}`, + size: 0, + trackScores: false, + trackTotalHits: false, +}); + +const logRateMlRecordRT = rt.type({ + actual: rt.array(rt.number), + bucket_span: rt.number, + record_score: rt.number, + timestamp: rt.number, + typical: rt.array(rt.number), +}); + +const metricAggregationRT = rt.type({ + value: rt.number, +}); + +const compositeTimestampDataSetKeyRT = rt.type({ + data_set: rt.string, + timestamp: rt.number, +}); + +export type CompositeTimestampDataSetKey = rt.TypeOf; + +export const logRateModelPlotBucketRT = rt.type({ + key: compositeTimestampDataSetKeyRT, + filter_records: rt.type({ + doc_count: rt.number, + top_hits_record: rt.type({ + hits: rt.type({ + hits: rt.array( + rt.type({ + _source: logRateMlRecordRT, + }) + ), + }), + }), + }), + filter_model_plot: rt.type({ + doc_count: rt.number, + average_actual: metricAggregationRT, + }), +}); + +export type LogRateModelPlotBucket = rt.TypeOf; + +export const logRateModelPlotResponseRT = rt.type({ + aggregations: rt.type({ + timestamp_data_set_buckets: rt.intersection([ + rt.type({ + buckets: rt.array(logRateModelPlotBucketRT), + }), + rt.partial({ + after_key: compositeTimestampDataSetKeyRT, + }), + ]), + }), +}); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 120afca8e8742d..c2af0a91db5f92 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -5319,11 +5319,8 @@ "xpack.infra.header.logsTitle": "ログ", "xpack.infra.homePage.settingsTabTitle": "設定", "xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage": "{metricId} のモデルには cloudId が必要ですが、{nodeId} に cloudId が指定されていません。", - "xpack.infra.logs.analysis.logRateSectionAnomalySeriesName": "異常", - "xpack.infra.logs.analysis.logRateSectionAreaSeriesName": "期待値", "xpack.infra.logs.analysis.logRateSectionLineSeriesName": "15 分ごとのログエントリー (平均)", "xpack.infra.logs.analysis.logRateSectionLoadingAriaLabel": "ログレートの結果を読み込み中", - "xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel": "モデルバウンドを表示", "xpack.infra.logs.analysis.logRateSectionNoDataBody": "時間範囲を調整する必要があるかもしれません。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "表示するデータがありません。", "xpack.infra.logs.analysis.logRateSectionTitle": "ログレート", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 2082d22ebb12b7..68ae356337ccca 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -5322,11 +5322,8 @@ "xpack.infra.header.logsTitle": "Logs", "xpack.infra.homePage.settingsTabTitle": "设置", "xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage": "{metricId} 的模型需要云 ID，但没有为 {nodeId} 提供。", - "xpack.infra.logs.analysis.logRateSectionAnomalySeriesName": "异常", - "xpack.infra.logs.analysis.logRateSectionAreaSeriesName": "预期", "xpack.infra.logs.analysis.logRateSectionLineSeriesName": "每 15 分钟日志条目数（平均值）", "xpack.infra.logs.analysis.logRateSectionLoadingAriaLabel": "正在加载日志速率结果", - "xpack.infra.logs.analysis.logRateSectionModelBoundsCheckboxLabel": "显示模型边界", "xpack.infra.logs.analysis.logRateSectionNoDataBody": "您可能想调整时间范围。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "没有可显示的数据。", "xpack.infra.logs.analysis.logRateSectionTitle": "日志速率", diff --git a/x-pack/test/api_integration/apis/infra/log_analysis.ts b/x-pack/test/api_integration/apis/infra/log_analysis.ts index bd09cdf6ff56ee..fe7d55649d1d63 100644 --- a/x-pack/test/api_integration/apis/infra/log_analysis.ts +++ b/x-pack/test/api_integration/apis/infra/log_analysis.ts @@ -20,8 +20,8 @@ import { } from '../../../../legacy/plugins/infra/common/runtime_types'; import { FtrProviderContext } from '../../ftr_provider_context'; -const TIME_BEFORE_START = 1564315100000; -const TIME_AFTER_END = 1565040700000; +const TIME_BEFORE_START = 1569934800000; +const TIME_AFTER_END = 1570016700000; const COMMON_HEADERS = { 'kbn-xsrf': 'some-xsrf-token', }; @@ -32,8 +32,8 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); describe('log analysis apis', () => { - before(() => esArchiver.load('infra/8.0.0/ml_anomalies_log_rate')); - after(() => esArchiver.unload('infra/8.0.0/ml_anomalies_log_rate')); + before(() => esArchiver.load('infra/8.0.0/ml_anomalies_partitioned_log_rate')); + after(() => esArchiver.unload('infra/8.0.0/ml_anomalies_partitioned_log_rate')); describe('log rate results', () => { describe('with the default source', () => { @@ -62,11 +62,12 @@ export default ({ getService }: FtrProviderContext) => { getLogEntryRateSuccessReponsePayloadRT.decode(body), fold(throwErrors(createPlainError), identity) ); - expect(logEntryRateBuckets.data.bucketDuration).to.be(15 * 60 * 1000); expect(logEntryRateBuckets.data.histogramBuckets).to.not.be.empty(); expect( - logEntryRateBuckets.data.histogramBuckets.some(bucket => bucket.anomalies.length > 0) + logEntryRateBuckets.data.histogramBuckets.some(bucket => { + return bucket.dataSets.some(dataSet => dataSet.anomalies.length > 0); + }) ).to.be(true); }); diff --git a/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz b/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz new file mode 100644 index 00000000000000..8d15ff8ccb0226 Binary files /dev/null and b/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/mappings.json b/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/mappings.json new file mode 100644 index 00000000000000..69ffc922ede7d3 --- /dev/null +++ b/x-pack/test/functional/es_archives/infra/8.0.0/ml_anomalies_partitioned_log_rate/mappings.json @@ -0,0 +1,513 @@ +{ + "type": "index", + "value": { + "aliases": { + ".ml-anomalies-.write-kibana-logs-ui-default-default-log-entry-rate": { + }, + ".ml-anomalies-kibana-logs-ui-default-default-log-entry-rate": { + "filter": { + "term": { + "job_id": { + "boost": 1, + "value": "kibana-logs-ui-default-default-log-entry-rate" + } + } + } + } + }, + "index": ".ml-anomalies-shared", + "mappings": { + "_meta": { + "version": "8.0.0" + }, + "dynamic_templates": [ + { + "strings_as_keywords": { + "mapping": { + "type": "keyword" + }, + "match": "*" + } + } + ], + "properties": { + "actual": { + "type": "double" + }, + "all_field_values": { + "analyzer": "whitespace", + "type": "text" + }, + "anomaly_score": { + "type": "double" + }, + "average_bucket_processing_time_ms": { + "type": "double" + }, + "bucket_allocation_failures_count": { + "type": "long" + }, + "bucket_count": { + "type": "long" + }, + "bucket_influencers": { + "properties": { + "anomaly_score": { + "type": "double" + }, + "bucket_span": { + "type": "long" + }, + "influencer_field_name": { + "type": "keyword" + }, + "initial_anomaly_score": { + "type": "double" + }, + "is_interim": { + "type": "boolean" + }, + "job_id": { + "type": "keyword" + }, + "probability": { + "type": "double" + }, + "raw_anomaly_score": { + "type": "double" + }, + "result_type": { + "type": "keyword" + }, + "timestamp": { + "type": "date" + } + }, + "type": "nested" + }, + "bucket_span": { + "type": "long" + }, + "by_field_name": { + "type": "keyword" + }, + "by_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "category_id": { + "type": "long" + }, + "causes": { + "properties": { + "actual": { + "type": "double" + }, + "by_field_name": { + "type": "keyword" + }, + "by_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "correlated_by_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "field_name": { + "type": "keyword" + }, + "function": { + "type": "keyword" + }, + "function_description": { + "type": "keyword" + }, + "over_field_name": { + "type": "keyword" + }, + "over_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "partition_field_name": { + "type": "keyword" + }, + "partition_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "probability": { + "type": "double" + }, + "typical": { + "type": "double" + } + }, + "type": "nested" + }, + "description": { + "type": "text" + }, + "detector_index": { + "type": "integer" + }, + "earliest_record_timestamp": { + "type": "date" + }, + "empty_bucket_count": { + "type": "long" + }, + "event": { + "properties": { + "dataset": { + "type": "keyword" + } + } + }, + "event_count": { + "type": "long" + }, + "examples": { + "type": "text" + }, + "exponential_average_bucket_processing_time_ms": { + "type": "double" + }, + "exponential_average_calculation_context": { + "properties": { + "incremental_metric_value_ms": { + "type": "double" + }, + "latest_timestamp": { + "type": "date" + }, + "previous_exponential_average_ms": { + "type": "double" + } + } + }, + "field_name": { + "type": "keyword" + }, + "forecast_create_timestamp": { + "type": "date" + }, + "forecast_end_timestamp": { + "type": "date" + }, + "forecast_expiry_timestamp": { + "type": "date" + }, + "forecast_id": { + "type": "keyword" + }, + "forecast_lower": { + "type": "double" + }, + "forecast_memory_bytes": { + "type": "long" + }, + "forecast_messages": { + "type": "keyword" + }, + "forecast_prediction": { + "type": "double" + }, + "forecast_progress": { + "type": "double" + }, + "forecast_start_timestamp": { + "type": "date" + }, + "forecast_status": { + "type": "keyword" + }, + "forecast_upper": { + "type": "double" + }, + "function": { + "type": "keyword" + }, + "function_description": { + "type": "keyword" + }, + "influencer_field_name": { + "type": "keyword" + }, + "influencer_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "influencer_score": { + "type": "double" + }, + "influencers": { + "properties": { + "influencer_field_name": { + "type": "keyword" + }, + "influencer_field_values": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + } + }, + "type": "nested" + }, + "initial_anomaly_score": { + "type": "double" + }, + "initial_influencer_score": { + "type": "double" + }, + "initial_record_score": { + "type": "double" + }, + "input_bytes": { + "type": "long" + }, + "input_field_count": { + "type": "long" + }, + "input_record_count": { + "type": "long" + }, + "invalid_date_count": { + "type": "long" + }, + "is_interim": { + "type": "boolean" + }, + "job_id": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "last_data_time": { + "type": "date" + }, + "latest_empty_bucket_timestamp": { + "type": "date" + }, + "latest_record_time_stamp": { + "type": "date" + }, + "latest_record_timestamp": { + "type": "date" + }, + "latest_result_time_stamp": { + "type": "date" + }, + "latest_sparse_bucket_timestamp": { + "type": "date" + }, + "log_time": { + "type": "date" + }, + "max_matching_length": { + "type": "long" + }, + "maximum_bucket_processing_time_ms": { + "type": "double" + }, + "memory_status": { + "type": "keyword" + }, + "min_version": { + "type": "keyword" + }, + "minimum_bucket_processing_time_ms": { + "type": "double" + }, + "missing_field_count": { + "type": "long" + }, + "model_bytes": { + "type": "long" + }, + "model_bytes_exceeded": { + "type": "keyword" + }, + "model_bytes_memory_limit": { + "type": "keyword" + }, + "model_feature": { + "type": "keyword" + }, + "model_lower": { + "type": "double" + }, + "model_median": { + "type": "double" + }, + "model_size_stats": { + "properties": { + "bucket_allocation_failures_count": { + "type": "long" + }, + "job_id": { + "type": "keyword" + }, + "log_time": { + "type": "date" + }, + "memory_status": { + "type": "keyword" + }, + "model_bytes": { + "type": "long" + }, + "model_bytes_exceeded": { + "type": "keyword" + }, + "model_bytes_memory_limit": { + "type": "keyword" + }, + "result_type": { + "type": "keyword" + }, + "timestamp": { + "type": "date" + }, + "total_by_field_count": { + "type": "long" + }, + "total_over_field_count": { + "type": "long" + }, + "total_partition_field_count": { + "type": "long" + } + } + }, + "model_upper": { + "type": "double" + }, + "multi_bucket_impact": { + "type": "double" + }, + "out_of_order_timestamp_count": { + "type": "long" + }, + "over_field_name": { + "type": "keyword" + }, + "over_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "partition_field_name": { + "type": "keyword" + }, + "partition_field_value": { + "copy_to": [ + "all_field_values" + ], + "type": "keyword" + }, + "probability": { + "type": "double" + }, + "processed_field_count": { + "type": "long" + }, + "processed_record_count": { + "type": "long" + }, + "processing_time_ms": { + "type": "long" + }, + "quantiles": { + "enabled": false, + "type": "object" + }, + "raw_anomaly_score": { + "type": "double" + }, + "record_score": { + "type": "double" + }, + "regex": { + "type": "keyword" + }, + "result_type": { + "type": "keyword" + }, + "retain": { + "type": "boolean" + }, + "scheduled_events": { + "type": "keyword" + }, + "search_count": { + "type": "long" + }, + "snapshot_doc_count": { + "type": "integer" + }, + "snapshot_id": { + "type": "keyword" + }, + "sparse_bucket_count": { + "type": "long" + }, + "terms": { + "type": "text" + }, + "timestamp": { + "type": "date" + }, + "total_by_field_count": { + "type": "long" + }, + "total_over_field_count": { + "type": "long" + }, + "total_partition_field_count": { + "type": "long" + }, + "total_search_time_ms": { + "type": "double" + }, + "typical": { + "type": "double" + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1", + "query": { + "default_field": "all_field_values" + }, + "translog": { + "durability": "async" + }, + "unassigned": { + "node_left": { + "delayed_timeout": "1m" + } + } + } + } + } +} \ No newline at end of file