diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts index 3af07980910b8a..5a1412fd8f3d4e 100644 --- a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts +++ b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts @@ -37,15 +37,17 @@ export const logEntryRateAnomaly = rt.type({ typicalLogEntryRate: rt.number, }); -export const logEntryRateDataSetRT = rt.type({ +export const logEntryRatePartitionRT = rt.type({ analysisBucketCount: rt.number, anomalies: rt.array(logEntryRateAnomaly), averageActualLogEntryRate: rt.number, - dataSetId: rt.string, + maximumAnomalyScore: rt.number, + numberOfLogEntries: rt.number, + partitionId: rt.string, }); export const logEntryRateHistogramBucket = rt.type({ - dataSets: rt.array(logEntryRateDataSetRT), + partitions: rt.array(logEntryRatePartitionRT), startTime: rt.number, }); @@ -53,6 +55,7 @@ export const getLogEntryRateSuccessReponsePayloadRT = rt.type({ data: rt.type({ bucketDuration: rt.number, histogramBuckets: rt.array(logEntryRateHistogramBucket), + totalNumberOfLogEntries: rt.number, }), }); diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx index aaf24c22594e52..f4d78457535288 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx @@ -9,19 +9,19 @@ import { EuiFlexGroup, EuiFlexItem, EuiPage, - EuiPageBody, - EuiPageContent, - EuiPageContentBody, EuiPanel, EuiSuperDatePicker, + EuiBadge, + EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import numeral from '@elastic/numeral'; +import { FormattedMessage } from '@kbn/i18n/react'; import moment from 'moment'; import React, { useCallback, useMemo, useState } from 'react'; - -import euiStyled from '../../../../../../common/eui_styled_components'; import { TimeRange } from '../../../../common/http_api/shared/time_range'; import { bucketSpan } from '../../../../common/log_analysis'; +import euiStyled from '../../../../../../common/eui_styled_components'; import { LoadingPage } from '../../../components/loading_page'; import { StringTimeRange, @@ -31,6 +31,8 @@ import { import { useTrackPageview } from '../../../hooks/use_track_metric'; import { FirstUseCallout } from './first_use'; import { LogRateResults } from './sections/log_rate'; +import { AnomaliesResults } from './sections/anomalies'; +import { useKibanaUiSetting } from '../../../utils/use_kibana_ui_setting'; export const AnalysisResultsContent = ({ sourceId, @@ -42,6 +44,8 @@ export const AnalysisResultsContent = ({ useTrackPageview({ app: 'infra_logs', path: 'analysis_results' }); useTrackPageview({ app: 'infra_logs', path: 'analysis_results', delay: 15000 }); + const [dateFormat] = useKibanaUiSetting('dateFormat', 'MMMM D, YYYY h:mm A'); + const { timeRange: selectedTimeRange, setTimeRange: setSelectedTimeRange, @@ -56,13 +60,13 @@ export const AnalysisResultsContent = ({ const bucketDuration = useMemo(() => { // This function takes the current time range in ms, // works out the bucket interval we'd need to always - // display 200 data points, and then takes that new + // display 100 data points, and then takes that new // value and works out the nearest multiple of // 900000 (15 minutes) to it, so that we don't end up with // jaggy bucket boundaries between the ML buckets and our // aggregation buckets. const msRange = moment(queryTimeRange.endTime).diff(moment(queryTimeRange.startTime)); - const bucketIntervalInMs = msRange / 200; + const bucketIntervalInMs = msRange / 100; const result = bucketSpan * Math.round(bucketIntervalInMs / bucketSpan); const roundedResult = parseInt(Number(result).toFixed(0), 10); return roundedResult < bucketSpan ? bucketSpan : roundedResult; @@ -130,28 +134,50 @@ export const AnalysisResultsContent = ({ /> ) : ( <> - - - - - - - - - - - - - - + + + + + + + {!isLoading && logEntryRate ? ( + + + + {numeral(logEntryRate.totalNumberOfLogEntries).format('0.00a')} + + + ), + startTime: ( + {moment(queryTimeRange.startTime).format(dateFormat)} + ), + endTime: {moment(queryTimeRange.endTime).format(dateFormat)}, + }} + /> + + ) : null} + + + + + + + + + {isFirstUse && !hasResults ? : null} - - - - + + + + + + + + + )} @@ -183,6 +219,10 @@ const stringToNumericTimeRange = (timeRange: StringTimeRange): TimeRange => ({ ).valueOf(), }); -const ExpandingPage = euiStyled(EuiPage)` - flex: 1 0 0%; +// This is needed due to the flex-basis: 100% !important; rule that +// kicks in on small screens via media queries breaking when using direction="column" +export const ResultsContentPage = euiStyled(EuiPage)` + .euiFlexGroup--responsive > .euiFlexItem { + flex-basis: auto !important; + } `; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/chart.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/chart.tsx new file mode 100644 index 00000000000000..73adcd13e24412 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/chart.tsx @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RectAnnotationDatum, AnnotationId } from '@elastic/charts'; +import { + Axis, + BarSeries, + Chart, + getAxisId, + getSpecId, + niceTimeFormatter, + Settings, + TooltipValue, + LIGHT_THEME, + DARK_THEME, + getAnnotationId, + RectAnnotation, +} from '@elastic/charts'; +import numeral from '@elastic/numeral'; +import { i18n } from '@kbn/i18n'; +import moment from 'moment'; +import React, { useCallback, useMemo } from 'react'; + +import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { useKibanaUiSetting } from '../../../../../utils/use_kibana_ui_setting'; +import { MLSeverityScoreCategories } from '../helpers/data_formatters'; + +export const AnomaliesChart: React.FunctionComponent<{ + chartId: string; + setTimeRange: (timeRange: TimeRange) => void; + timeRange: TimeRange; + series: Array<{ time: number; value: number }>; + annotations: Record; + renderAnnotationTooltip?: (details?: string) => JSX.Element; +}> = ({ chartId, series, annotations, setTimeRange, timeRange, renderAnnotationTooltip }) => { + const [dateFormat] = useKibanaUiSetting('dateFormat', 'Y-MM-DD HH:mm:ss.SSS'); + const [isDarkMode] = useKibanaUiSetting('theme:darkMode'); + + const chartDateFormatter = useMemo( + () => niceTimeFormatter([timeRange.startTime, timeRange.endTime]), + [timeRange] + ); + + const logEntryRateSpecId = getSpecId('averageValues'); + + const tooltipProps = useMemo( + () => ({ + headerFormatter: (tooltipData: TooltipValue) => moment(tooltipData.value).format(dateFormat), + }), + [dateFormat] + ); + + const handleBrushEnd = useCallback( + (startTime: number, endTime: number) => { + setTimeRange({ + endTime, + startTime, + }); + }, + [setTimeRange] + ); + + return ( +
+ + + numeral(value.toPrecision(3)).format('0[.][00]a')} // https://github.com/adamwdraper/Numeral-js/issues/194 + /> + + {renderAnnotations(annotations, chartId, renderAnnotationTooltip)} + + +
+ ); +}; + +interface SeverityConfig { + annotationId: AnnotationId; + style: { + fill: string; + opacity: number; + }; +} + +const severityConfigs: Record = { + warning: { + annotationId: getAnnotationId(`anomalies-warning`), + style: { fill: 'rgb(125, 180, 226)', opacity: 0.7 }, + }, + minor: { + annotationId: getAnnotationId(`anomalies-minor`), + style: { fill: 'rgb(255, 221, 0)', opacity: 0.7 }, + }, + major: { + annotationId: getAnnotationId(`anomalies-major`), + style: { fill: 'rgb(229, 113, 0)', opacity: 0.7 }, + }, + critical: { + annotationId: getAnnotationId(`anomalies-critical`), + style: { fill: 'rgb(228, 72, 72)', opacity: 0.7 }, + }, +}; + +const renderAnnotations = ( + annotations: Record, + chartId: string, + renderAnnotationTooltip?: (details?: string) => JSX.Element +) => { + return Object.entries(annotations).map((entry, index) => { + return ( + + ); + }); +}; + +const barSeriesStyle = { rect: { fill: '#D3DAE6', opacity: 0.6 } }; // TODO: Acquire this from "theme" as euiColorLightShade diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/expanded_row.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/expanded_row.tsx new file mode 100644 index 00000000000000..a6f2ca71068c27 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/expanded_row.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import numeral from '@elastic/numeral'; +import { EuiFlexGroup, EuiFlexItem, EuiStat } from '@elastic/eui'; +import { AnomaliesChart } from './chart'; +import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; +import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { + getLogEntryRateSeriesForPartition, + getAnnotationsForPartition, + formatAnomalyScore, + getTotalNumberOfLogEntriesForPartition, +} from '../helpers/data_formatters'; + +export const AnomaliesTableExpandedRow: React.FunctionComponent<{ + partitionId: string; + topAnomalyScore: number; + results: GetLogEntryRateSuccessResponsePayload['data']; + setTimeRange: (timeRange: TimeRange) => void; + timeRange: TimeRange; +}> = ({ results, timeRange, setTimeRange, topAnomalyScore, partitionId }) => { + const logEntryRateSeries = useMemo( + () => + results && results.histogramBuckets + ? getLogEntryRateSeriesForPartition(results, partitionId) + : [], + [results, partitionId] + ); + const anomalyAnnotations = useMemo( + () => + results && results.histogramBuckets + ? getAnnotationsForPartition(results, partitionId) + : { + warning: [], + minor: [], + major: [], + critical: [], + }, + [results, partitionId] + ); + const totalNumberOfLogEntries = useMemo( + () => + results && results.histogramBuckets + ? getTotalNumberOfLogEntriesForPartition(results, partitionId) + : undefined, + [results, partitionId] + ); + return ( + + + + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/index.tsx new file mode 100644 index 00000000000000..c4024e67dc1a03 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/index.tsx @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + EuiEmptyPrompt, + EuiFlexGroup, + EuiFlexItem, + EuiLoadingChart, + EuiSpacer, + EuiTitle, + EuiStat, +} from '@elastic/eui'; +import numeral from '@elastic/numeral'; +import { i18n } from '@kbn/i18n'; +import React, { useMemo } from 'react'; + +import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; +import euiStyled from '../../../../../../../../common/eui_styled_components'; +import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { AnomaliesChart } from './chart'; +import { AnomaliesTable } from './table'; +import { + getLogEntryRateCombinedSeries, + getAnnotationsForAll, + getTopAnomalyScoreAcrossAllPartitions, + formatAnomalyScore, +} from '../helpers/data_formatters'; + +export const AnomaliesResults = ({ + isLoading, + results, + setTimeRange, + timeRange, +}: { + isLoading: boolean; + results: GetLogEntryRateSuccessResponsePayload['data'] | null; + setTimeRange: (timeRange: TimeRange) => void; + timeRange: TimeRange; +}) => { + const title = i18n.translate('xpack.infra.logs.analysis.anomaliesSectionTitle', { + defaultMessage: 'Anomalies', + }); + + const loadingAriaLabel = i18n.translate( + 'xpack.infra.logs.analysis.anomaliesSectionLoadingAriaLabel', + { defaultMessage: 'Loading anomalies' } + ); + + const hasAnomalies = useMemo(() => { + return results && results.histogramBuckets + ? results.histogramBuckets.some(bucket => { + return bucket.partitions.some(partition => { + return partition.anomalies.length > 0; + }); + }) + : false; + }, [results]); + + const logEntryRateSeries = useMemo( + () => (results && results.histogramBuckets ? getLogEntryRateCombinedSeries(results) : []), + [results] + ); + const anomalyAnnotations = useMemo( + () => + results && results.histogramBuckets + ? getAnnotationsForAll(results) + : { + warning: [], + minor: [], + major: [], + critical: [], + }, + [results] + ); + + const topAnomalyScore = useMemo( + () => + results && results.histogramBuckets + ? getTopAnomalyScoreAcrossAllPartitions(results) + : undefined, + [results] + ); + + return ( + <> + +

{title}

+ + + {isLoading ? ( + + + + + + ) : !results || (results && results.histogramBuckets && !results.histogramBuckets.length) ? ( + + {i18n.translate('xpack.infra.logs.analysis.anomalySectionNoDataTitle', { + defaultMessage: 'There is no data to display.', + })} + + } + titleSize="m" + body={ +

+ {i18n.translate('xpack.infra.logs.analysis.anomalySectionNoDataBody', { + defaultMessage: 'You may want to adjust your time range.', + })} +

+ } + /> + ) : !hasAnomalies ? ( + + {i18n.translate('xpack.infra.logs.analysis.anomalySectionNoAnomaliesTitle', { + defaultMessage: 'No anomalies were detected.', + })} + + } + titleSize="m" + /> + ) : ( + <> + + + + + + + + + + + + + )} + + ); +}; + +interface ParsedAnnotationDetails { + anomalyScoresByPartition: Array<{ partitionId: string; maximumAnomalyScore: number }>; +} + +const overallAnomalyScoreLabel = i18n.translate( + 'xpack.infra.logs.analysis.overallAnomalyChartMaxScoresLabel', + { + defaultMessage: 'Max anomaly scores:', + } +); +const AnnotationTooltip: React.FunctionComponent<{ details: string }> = ({ details }) => { + const parsedDetails: ParsedAnnotationDetails = JSON.parse(details); + return ( + + + {overallAnomalyScoreLabel} + +
    + {parsedDetails.anomalyScoresByPartition.map( + ({ partitionId, maximumAnomalyScore }, index) => { + return ( +
  • + + {`${partitionId}: `} + {maximumAnomalyScore} + +
    • + ); + } + )} +
+ + ); +}; + +const renderAnnotationTooltip = (details?: string) => { + // Note: Seems to be necessary to get things typed correctly all the way through to elastic-charts components + if (!details) { + return
; + } + return ; +}; + +const TooltipWrapper = euiStyled('div')` + white-space: nowrap; +`; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/table.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/table.tsx new file mode 100644 index 00000000000000..2a8ac44d09083b --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/anomalies/table.tsx @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useMemo, useState, useCallback } from 'react'; +import { EuiBasicTable, EuiButtonIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { RIGHT_ALIGNMENT } from '@elastic/eui/lib/services'; +import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; +import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; +import { AnomaliesTableExpandedRow } from './expanded_row'; +import { getTopAnomalyScoresByPartition, formatAnomalyScore } from '../helpers/data_formatters'; + +interface TableItem { + id: string; + partition: string; + topAnomalyScore: number; +} + +interface SortingOptions { + sort: { + field: string; + direction: string; + }; +} + +const collapseAriaLabel = i18n.translate('xpack.infra.logs.analysis.anomaliesTableCollapseLabel', { + defaultMessage: 'Collapse', +}); + +const expandAriaLabel = i18n.translate('xpack.infra.logs.analysis.anomaliesTableExpandLabel', { + defaultMessage: 'Expand', +}); + +const partitionColumnName = i18n.translate( + 'xpack.infra.logs.analysis.anomaliesTablePartitionColumnName', + { + defaultMessage: 'Partition', + } +); + +const maxAnomalyScoreColumnName = i18n.translate( + 'xpack.infra.logs.analysis.anomaliesTableMaxAnomalyScoreColumnName', + { + defaultMessage: 'Max anomaly score', + } +); + +export const AnomaliesTable: React.FunctionComponent<{ + results: GetLogEntryRateSuccessResponsePayload['data']; + setTimeRange: (timeRange: TimeRange) => void; + timeRange: TimeRange; +}> = ({ results, timeRange, setTimeRange }) => { + const tableItems: TableItem[] = useMemo(() => { + return Object.entries(getTopAnomalyScoresByPartition(results)).map(([key, value]) => { + return { + id: key || 'unknown', // Note: EUI's table expanded rows won't work with a key of '' in itemIdToExpandedRowMap + partition: key || 'unknown', + topAnomalyScore: formatAnomalyScore(value), + }; + }); + }, [results]); + + const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState< + Record + >({}); + + const [sorting, setSorting] = useState({ + sort: { + field: 'topAnomalyScore', + direction: 'desc', + }, + }); + + const handleTableChange = useCallback( + ({ sort = {} }) => { + const { field, direction } = sort; + setSorting({ + sort: { + field, + direction, + }, + }); + }, + [setSorting] + ); + + const sortedTableItems = useMemo(() => { + let sortedItems: TableItem[] = []; + if (sorting.sort.field === 'partition') { + sortedItems = tableItems.sort((a, b) => (a.partition > b.partition ? 1 : -1)); + } else if (sorting.sort.field === 'topAnomalyScore') { + sortedItems = tableItems.sort((a, b) => a.topAnomalyScore - b.topAnomalyScore); + } + return sorting.sort.direction === 'asc' ? sortedItems : sortedItems.reverse(); + }, [tableItems, sorting]); + + const toggleExpandedItems = useCallback( + item => { + if (itemIdToExpandedRowMap[item.id]) { + const { [item.id]: toggledItem, ...remainingExpandedRowMap } = itemIdToExpandedRowMap; + setItemIdToExpandedRowMap(remainingExpandedRowMap); + } else { + const newItemIdToExpandedRowMap = { + ...itemIdToExpandedRowMap, + [item.id]: ( + + ), + }; + setItemIdToExpandedRowMap(newItemIdToExpandedRowMap); + } + }, + [results, setTimeRange, timeRange, itemIdToExpandedRowMap, setItemIdToExpandedRowMap] + ); + + const columns = [ + { + field: 'partition', + name: partitionColumnName, + sortable: true, + truncateText: true, + }, + { + field: 'topAnomalyScore', + name: maxAnomalyScoreColumnName, + sortable: true, + truncateText: true, + }, + { + align: RIGHT_ALIGNMENT, + width: '40px', + isExpander: true, + render: (item: TableItem) => ( + toggleExpandedItems(item)} + aria-label={itemIdToExpandedRowMap[item.id] ? collapseAriaLabel : expandAriaLabel} + iconType={itemIdToExpandedRowMap[item.id] ? 'arrowUp' : 'arrowDown'} + /> + ), + }, + ]; + + return ( + + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/helpers/data_formatters.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/helpers/data_formatters.tsx new file mode 100644 index 00000000000000..105604507b4250 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/helpers/data_formatters.tsx @@ -0,0 +1,259 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RectAnnotationDatum } from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; + +export type MLSeverityScoreCategories = 'warning' | 'minor' | 'major' | 'critical'; +type MLSeverityScores = Record; +const ML_SEVERITY_SCORES: MLSeverityScores = { + warning: 3, + minor: 25, + major: 50, + critical: 75, +}; + +export const getLogEntryRatePartitionedSeries = ( + results: GetLogEntryRateSuccessResponsePayload['data'] +) => { + return results.histogramBuckets.reduce>( + (buckets, bucket) => { + return [ + ...buckets, + ...bucket.partitions.map(partition => ({ + group: partition.partitionId === '' ? 'unknown' : partition.partitionId, + time: bucket.startTime, + value: partition.averageActualLogEntryRate, + })), + ]; + }, + [] + ); +}; + +export const getLogEntryRateCombinedSeries = ( + results: GetLogEntryRateSuccessResponsePayload['data'] +) => { + return results.histogramBuckets.reduce>( + (buckets, bucket) => { + return [ + ...buckets, + { + time: bucket.startTime, + value: bucket.partitions.reduce((accumulatedValue, partition) => { + return accumulatedValue + partition.averageActualLogEntryRate; + }, 0), + }, + ]; + }, + [] + ); +}; + +export const getLogEntryRateSeriesForPartition = ( + results: GetLogEntryRateSuccessResponsePayload['data'], + partitionId: string +) => { + return results.histogramBuckets.reduce>( + (buckets, bucket) => { + const partitionResults = bucket.partitions.find(partition => { + return ( + partition.partitionId === partitionId || + (partition.partitionId === '' && partitionId === 'unknown') + ); + }); + if (!partitionResults) { + return buckets; + } + return [ + ...buckets, + { + time: bucket.startTime, + value: partitionResults.averageActualLogEntryRate, + }, + ]; + }, + [] + ); +}; + +export const getTopAnomalyScoresByPartition = ( + results: GetLogEntryRateSuccessResponsePayload['data'] +) => { + return results.histogramBuckets.reduce>((topScores, bucket) => { + bucket.partitions.forEach(partition => { + if (partition.maximumAnomalyScore > 0) { + topScores = { + ...topScores, + [partition.partitionId]: + !topScores[partition.partitionId] || + partition.maximumAnomalyScore > topScores[partition.partitionId] + ? partition.maximumAnomalyScore + : topScores[partition.partitionId], + }; + } + }); + return topScores; + }, {}); +}; + +export const getAnnotationsForPartition = ( + results: GetLogEntryRateSuccessResponsePayload['data'], + partitionId: string +) => { + return results.histogramBuckets.reduce>( + (annotatedBucketsBySeverity, bucket) => { + const partitionResults = bucket.partitions.find(partition => { + return ( + partition.partitionId === partitionId || + (partition.partitionId === '' && partitionId === 'unknown') + ); + }); + const severityCategory = partitionResults + ? getSeverityCategoryForScore(partitionResults.maximumAnomalyScore) + : null; + if (!partitionResults || !partitionResults.maximumAnomalyScore || !severityCategory) { + return annotatedBucketsBySeverity; + } + + return { + ...annotatedBucketsBySeverity, + [severityCategory]: [ + ...annotatedBucketsBySeverity[severityCategory], + { + coordinates: { + x0: bucket.startTime, + x1: bucket.startTime + results.bucketDuration, + }, + details: i18n.translate( + 'xpack.infra.logs.analysis.partitionMaxAnomalyScoreAnnotationLabel', + { + defaultMessage: 'Max anomaly score: {maxAnomalyScore}', + values: { + maxAnomalyScore: formatAnomalyScore(partitionResults.maximumAnomalyScore), + }, + } + ), + }, + ], + }; + }, + { + warning: [], + minor: [], + major: [], + critical: [], + } + ); +}; + +export const getTotalNumberOfLogEntriesForPartition = ( + results: GetLogEntryRateSuccessResponsePayload['data'], + partitionId: string +) => { + return results.histogramBuckets.reduce((sumPartitionNumberOfLogEntries, bucket) => { + const partitionResults = bucket.partitions.find(partition => { + return ( + partition.partitionId === partitionId || + (partition.partitionId === '' && partitionId === 'unknown') + ); + }); + if (!partitionResults || !partitionResults.numberOfLogEntries) { + return sumPartitionNumberOfLogEntries; + } else { + return (sumPartitionNumberOfLogEntries += partitionResults.numberOfLogEntries); + } + }, 0); +}; + +export const getAnnotationsForAll = (results: GetLogEntryRateSuccessResponsePayload['data']) => { + return results.histogramBuckets.reduce>( + (annotatedBucketsBySeverity, bucket) => { + const maxAnomalyScoresByPartition = bucket.partitions.reduce< + Array<{ partitionId: string; maximumAnomalyScore: number }> + >((bucketMaxAnomalyScoresByPartition, partition) => { + if (!getSeverityCategoryForScore(partition.maximumAnomalyScore)) { + return bucketMaxAnomalyScoresByPartition; + } + return [ + ...bucketMaxAnomalyScoresByPartition, + { + partitionId: partition.partitionId ? partition.partitionId : 'unknown', + maximumAnomalyScore: formatAnomalyScore(partition.maximumAnomalyScore), + }, + ]; + }, []); + + if (maxAnomalyScoresByPartition.length === 0) { + return annotatedBucketsBySeverity; + } + const severityCategory = getSeverityCategoryForScore( + Math.max( + ...maxAnomalyScoresByPartition.map(partitionScore => partitionScore.maximumAnomalyScore) + ) + ); + if (!severityCategory) { + return annotatedBucketsBySeverity; + } + const sortedMaxAnomalyScoresByPartition = maxAnomalyScoresByPartition.sort( + (a, b) => b.maximumAnomalyScore - a.maximumAnomalyScore + ); + return { + ...annotatedBucketsBySeverity, + [severityCategory]: [ + ...annotatedBucketsBySeverity[severityCategory], + { + coordinates: { + x0: bucket.startTime, + x1: bucket.startTime + results.bucketDuration, + }, + details: JSON.stringify({ + anomalyScoresByPartition: sortedMaxAnomalyScoresByPartition, + }), + }, + ], + }; + }, + { + warning: [], + minor: [], + major: [], + critical: [], + } + ); +}; + +export const getTopAnomalyScoreAcrossAllPartitions = ( + results: GetLogEntryRateSuccessResponsePayload['data'] +) => { + const allMaxScores = results.histogramBuckets.reduce((scores, bucket) => { + const bucketMaxScores = bucket.partitions.reduce((bucketScores, partition) => { + return [...bucketScores, partition.maximumAnomalyScore]; + }, []); + return [...scores, ...bucketMaxScores]; + }, []); + return Math.max(...allMaxScores); +}; + +const getSeverityCategoryForScore = (score: number): MLSeverityScoreCategories | undefined => { + if (score >= ML_SEVERITY_SCORES.critical) { + return 'critical'; + } else if (score >= ML_SEVERITY_SCORES.major) { + return 'major'; + } else if (score >= ML_SEVERITY_SCORES.minor) { + return 'minor'; + } else if (score >= ML_SEVERITY_SCORES.warning) { + return 'warning'; + } else { + // Category is too low to include + return undefined; + } +}; + +export const formatAnomalyScore = (score: number) => { + return Math.round(score); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/bar_chart.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/bar_chart.tsx index 0719bf956b680c..de856bee905137 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/bar_chart.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/bar_chart.tsx @@ -8,33 +8,27 @@ import { Axis, BarSeries, Chart, - getAnnotationId, getAxisId, getSpecId, niceTimeFormatter, - RectAnnotation, - RectAnnotationDatum, Settings, TooltipValue, LIGHT_THEME, DARK_THEME, } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; +import numeral from '@elastic/numeral'; import moment from 'moment'; import React, { useCallback, useMemo } from 'react'; -import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; import { useKibanaUiSetting } from '../../../../../utils/use_kibana_ui_setting'; -type LogEntryRateHistogramBuckets = GetLogEntryRateSuccessResponsePayload['data']['histogramBuckets']; - export const LogEntryRateBarChart: React.FunctionComponent<{ - bucketDuration: number; - histogramBuckets: LogEntryRateHistogramBuckets | null; setTimeRange: (timeRange: TimeRange) => void; timeRange: TimeRange; -}> = ({ bucketDuration, histogramBuckets, setTimeRange, timeRange }) => { + series: Array<{ group: string; time: number; value: number }>; +}> = ({ series, setTimeRange, timeRange }) => { const [dateFormat] = useKibanaUiSetting('dateFormat'); const [isDarkMode] = useKibanaUiSetting('theme:darkMode'); @@ -43,62 +37,7 @@ export const LogEntryRateBarChart: React.FunctionComponent<{ [timeRange] ); - const logEntryRateSeries = useMemo( - () => - histogramBuckets - ? histogramBuckets.reduce>( - (buckets, bucket) => { - return [ - ...buckets, - ...bucket.dataSets.map(dataSet => ({ - group: dataSet.dataSetId === '' ? 'unknown' : dataSet.dataSetId, - time: bucket.startTime, - value: dataSet.averageActualLogEntryRate, - })), - ]; - }, - [] - ) - : [], - [histogramBuckets] - ); - - const logEntryRateAnomalyAnnotations = useMemo( - () => - histogramBuckets - ? histogramBuckets.reduce((annotatedBuckets, bucket) => { - const anomalies = bucket.dataSets.reduce( - (accumulatedAnomalies, dataSet) => [...accumulatedAnomalies, ...dataSet.anomalies], - [] - ); - if (anomalies.length <= 0) { - return annotatedBuckets; - } - return [ - ...annotatedBuckets, - { - coordinates: { - x0: bucket.startTime, - x1: bucket.startTime + bucketDuration, - }, - details: i18n.translate( - 'xpack.infra.logs.analysis.logRateSectionAnomalyCountTooltipLabel', - { - defaultMessage: `{anomalyCount, plural, one {# anomaly} other {# anomalies}}`, - values: { - anomalyCount: anomalies.length, - }, - } - ), - }, - ]; - }, []) - : [], - [histogramBuckets] - ); - const logEntryRateSpecId = getSpecId('averageValues'); - const logEntryRateAnomalyAnnotationsId = getAnnotationId('anomalies'); const tooltipProps = useMemo( () => ({ @@ -119,24 +58,18 @@ export const LogEntryRateBarChart: React.FunctionComponent<{ ); return ( -
+
Number(value).toFixed(0)} + tickFormat={value => numeral(value.toPrecision(3)).format('0[.][00]a')} // https://github.com/adamwdraper/Numeral-js/issues/194 /> -
diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx index 1f01af33e33c4f..d4ddd14bfaa289 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/log_rate/index.tsx @@ -11,13 +11,15 @@ import { EuiLoadingChart, EuiSpacer, EuiTitle, + EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useMemo } from 'react'; import { GetLogEntryRateSuccessResponsePayload } from '../../../../../../common/http_api/log_analysis/results/log_entry_rate'; import { TimeRange } from '../../../../../../common/http_api/shared/time_range'; import { LogEntryRateBarChart } from './bar_chart'; +import { getLogEntryRatePartitionedSeries } from '../helpers/data_formatters'; export const LogRateResults = ({ isLoading, @@ -31,7 +33,7 @@ export const LogRateResults = ({ timeRange: TimeRange; }) => { const title = i18n.translate('xpack.infra.logs.analysis.logRateSectionTitle', { - defaultMessage: 'Log rate', + defaultMessage: 'Log entries', }); const loadingAriaLabel = i18n.translate( @@ -39,43 +41,66 @@ export const LogRateResults = ({ { defaultMessage: 'Loading log rate results' } ); + const logEntryRateSeries = useMemo( + () => (results && results.histogramBuckets ? getLogEntryRatePartitionedSeries(results) : []), + [results] + ); + return ( <>

{title}

 - {isLoading ? ( - - - - - + <> + + + + + + + ) : !results || (results && results.histogramBuckets && !results.histogramBuckets.length) ? ( - - {i18n.translate('xpack.infra.logs.analysis.logRateSectionNoDataTitle', { - defaultMessage: 'There is no data to display.', - })} - - } - titleSize="m" - body={ + <> + + + {i18n.translate('xpack.infra.logs.analysis.logRateSectionNoDataTitle', { + defaultMessage: 'There is no data to display.', + })} + + } + titleSize="m" + body={ +

+ {i18n.translate('xpack.infra.logs.analysis.logRateSectionNoDataBody', { + defaultMessage: 'You may want to adjust your time range.', + })} +

+ } + /> + + ) : ( + <> +

- {i18n.translate('xpack.infra.logs.analysis.logRateSectionNoDataBody', { - defaultMessage: 'You may want to adjust your time range.', + + {i18n.translate('xpack.infra.logs.analysis.logRateSectionBucketSpanLabel', { + defaultMessage: 'Bucket span: ', + })} + + {i18n.translate('xpack.infra.logs.analysis.logRateSectionBucketSpanValue', { + defaultMessage: '15 minutes', })}

- } - /> - ) : ( - + + + )} ); diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts index 31d9c5403e2d2b..d970a142c5c235 100644 --- a/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/log_analysis.ts @@ -15,7 +15,7 @@ import { logRateModelPlotResponseRT, createLogEntryRateQuery, LogRateModelPlotBucket, - CompositeTimestampDataSetKey, + CompositeTimestampPartitionKey, } from './queries'; const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; @@ -43,7 +43,7 @@ export class InfraLogAnalysis { const logRateJobId = this.getJobIds(request, sourceId).logEntryRate; let mlModelPlotBuckets: LogRateModelPlotBucket[] = []; - let afterLatestBatchKey: CompositeTimestampDataSetKey | undefined; + let afterLatestBatchKey: CompositeTimestampPartitionKey | undefined; while (true) { const mlModelPlotResponse = await this.libs.framework.callWithRequest( @@ -67,7 +67,7 @@ export class InfraLogAnalysis { const { after_key: afterKey, buckets: latestBatchBuckets } = pipe( logRateModelPlotResponseRT.decode(mlModelPlotResponse), - map(response => response.aggregations.timestamp_data_set_buckets), + map(response => response.aggregations.timestamp_partition_buckets), fold(throwErrors(createPlainError), identity) ); @@ -81,7 +81,7 @@ export class InfraLogAnalysis { return mlModelPlotBuckets.reduce< Array<{ - dataSets: Array<{ + partitions: Array<{ analysisBucketCount: number; anomalies: Array<{ actualLogEntryRate: number; @@ -91,15 +91,17 @@ export class InfraLogAnalysis { typicalLogEntryRate: number; }>; averageActualLogEntryRate: number; - dataSetId: string; + maximumAnomalyScore: number; + numberOfLogEntries: number; + partitionId: string; }>; startTime: number; }> - >((histogramBuckets, timestampDataSetBucket) => { + >((histogramBuckets, timestampPartitionBucket) => { const previousHistogramBucket = histogramBuckets[histogramBuckets.length - 1]; - const dataSet = { - analysisBucketCount: timestampDataSetBucket.filter_model_plot.doc_count, - anomalies: timestampDataSetBucket.filter_records.top_hits_record.hits.hits.map( + const partition = { + analysisBucketCount: timestampPartitionBucket.filter_model_plot.doc_count, + anomalies: timestampPartitionBucket.filter_records.top_hits_record.hits.hits.map( ({ _source: record }) => ({ actualLogEntryRate: record.actual[0], anomalyScore: record.record_score, @@ -108,26 +110,30 @@ export class InfraLogAnalysis { typicalLogEntryRate: record.typical[0], }) ), - averageActualLogEntryRate: timestampDataSetBucket.filter_model_plot.average_actual.value, - dataSetId: timestampDataSetBucket.key.data_set, + averageActualLogEntryRate: + timestampPartitionBucket.filter_model_plot.average_actual.value || 0, + maximumAnomalyScore: + timestampPartitionBucket.filter_records.maximum_record_score.value || 0, + numberOfLogEntries: timestampPartitionBucket.filter_model_plot.sum_actual.value || 0, + partitionId: timestampPartitionBucket.key.partition, }; if ( previousHistogramBucket && - previousHistogramBucket.startTime === timestampDataSetBucket.key.timestamp + previousHistogramBucket.startTime === timestampPartitionBucket.key.timestamp ) { return [ ...histogramBuckets.slice(0, -1), { ...previousHistogramBucket, - dataSets: [...previousHistogramBucket.dataSets, dataSet], + partitions: [...previousHistogramBucket.partitions, partition], }, ]; } else { return [ ...histogramBuckets, { - dataSets: [dataSet], - startTime: timestampDataSetBucket.key.timestamp, + partitions: [partition], + startTime: timestampPartitionBucket.key.timestamp, }, ]; } diff --git a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts index b10b1fe04db24f..2dd0880cbf8cb7 100644 --- a/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts +++ b/x-pack/legacy/plugins/infra/server/lib/log_analysis/queries/log_entry_rate.ts @@ -14,7 +14,7 @@ export const createLogEntryRateQuery = ( endTime: number, bucketDuration: number, size: number, - afterKey?: CompositeTimestampDataSetKey + afterKey?: CompositeTimestampPartitionKey ) => ({ allowNoIndices: true, body: { @@ -45,7 +45,7 @@ export const createLogEntryRateQuery = ( }, }, aggs: { - timestamp_data_set_buckets: { + timestamp_partition_buckets: { composite: { after: afterKey, size, @@ -60,7 +60,7 @@ export const createLogEntryRateQuery = ( }, }, { - data_set: { + partition: { terms: { field: 'partition_field_value', order: 'asc', @@ -82,6 +82,11 @@ export const createLogEntryRateQuery = ( field: 'actual', }, }, + sum_actual: { + sum: { + field: 'actual', + }, + }, }, }, filter_records: { @@ -91,6 +96,11 @@ export const createLogEntryRateQuery = ( }, }, aggs: { + maximum_record_score: { + max: { + field: 'record_score', + }, + }, top_hits_record: { top_hits: { _source: Object.keys(logRateMlRecordRT.props), @@ -124,20 +134,21 @@ const logRateMlRecordRT = rt.type({ }); const metricAggregationRT = rt.type({ - value: rt.number, + value: rt.union([rt.number, rt.null]), }); -const compositeTimestampDataSetKeyRT = rt.type({ - data_set: rt.string, +const compositeTimestampPartitionKeyRT = rt.type({ + partition: rt.string, timestamp: rt.number, }); -export type CompositeTimestampDataSetKey = rt.TypeOf; +export type CompositeTimestampPartitionKey = rt.TypeOf; export const logRateModelPlotBucketRT = rt.type({ - key: compositeTimestampDataSetKeyRT, + key: compositeTimestampPartitionKeyRT, filter_records: rt.type({ doc_count: rt.number, + maximum_record_score: metricAggregationRT, top_hits_record: rt.type({ hits: rt.type({ hits: rt.array( @@ -151,6 +162,7 @@ export const logRateModelPlotBucketRT = rt.type({ filter_model_plot: rt.type({ doc_count: rt.number, average_actual: metricAggregationRT, + sum_actual: metricAggregationRT, }), }); @@ -158,12 +170,12 @@ export type LogRateModelPlotBucket = rt.TypeOf; export const logRateModelPlotResponseRT = rt.type({ aggregations: rt.type({ - timestamp_data_set_buckets: rt.intersection([ + timestamp_partition_buckets: rt.intersection([ rt.type({ buckets: rt.array(logRateModelPlotBucketRT), }), rt.partial({ - after_key: compositeTimestampDataSetKeyRT, + after_key: compositeTimestampPartitionKeyRT, }), ]), }), diff --git a/x-pack/legacy/plugins/infra/server/routes/log_analysis/results/log_entry_rate.ts b/x-pack/legacy/plugins/infra/server/routes/log_analysis/results/log_entry_rate.ts index 3212ecaf65e888..fc06ea48f4353b 100644 --- a/x-pack/legacy/plugins/infra/server/routes/log_analysis/results/log_entry_rate.ts +++ b/x-pack/legacy/plugins/infra/server/routes/log_analysis/results/log_entry_rate.ts @@ -14,6 +14,7 @@ import { LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH, getLogEntryRateRequestPayloadRT, getLogEntryRateSuccessReponsePayloadRT, + GetLogEntryRateSuccessResponsePayload, } from '../../../../common/http_api/log_analysis'; import { throwErrors } from '../../../../common/runtime_types'; import { NoLogRateResultsIndexError } from '../../../lib/log_analysis'; @@ -52,9 +53,21 @@ export const initLogAnalysisGetLogEntryRateRoute = ({ data: { bucketDuration: payload.data.bucketDuration, histogramBuckets: logEntryRateBuckets, + totalNumberOfLogEntries: getTotalNumberOfLogEntries(logEntryRateBuckets), }, }) ); }, }); }; + +const getTotalNumberOfLogEntries = ( + logEntryRateBuckets: GetLogEntryRateSuccessResponsePayload['data']['histogramBuckets'] +) => { + return logEntryRateBuckets.reduce((sumNumberOfLogEntries, bucket) => { + const sumPartitions = bucket.partitions.reduce((partitionsTotal, partition) => { + return (partitionsTotal += partition.numberOfLogEntries); + }, 0); + return (sumNumberOfLogEntries += sumPartitions); + }, 0); +}; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index d8b8aa2bd5d4c0..d64197e409e4f4 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -5241,8 +5241,6 @@ "xpack.infra.logs.analysis.logRateSectionNoDataBody": "時間範囲を調整する必要があるかもしれません。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "表示するデータがありません。", "xpack.infra.logs.analysis.logRateSectionTitle": "ログレート", - "xpack.infra.logs.analysis.logRateSectionXaxisTitle": "時間", - "xpack.infra.logs.analysis.logRateSectionYaxisTitle": "15 分ごとのログエントリー", "xpack.infra.logs.analysisPage.loadingMessage": "分析ジョブのステータスを確認中…", "xpack.infra.logs.analysisPage.unavailable.mlAppButton": "機械学習を開く", "xpack.infra.logs.analysisPage.unavailable.mlAppLink": "機械学習アプリ", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 5949c56b81f4a6..76956401122bcb 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -5243,8 +5243,6 @@ "xpack.infra.logs.analysis.logRateSectionNoDataBody": "您可能想调整时间范围。", "xpack.infra.logs.analysis.logRateSectionNoDataTitle": "没有可显示的数据。", "xpack.infra.logs.analysis.logRateSectionTitle": "日志速率", - "xpack.infra.logs.analysis.logRateSectionXaxisTitle": "时间", - "xpack.infra.logs.analysis.logRateSectionYaxisTitle": "每 15 分钟日志条目数", "xpack.infra.logs.analysisPage.loadingMessage": "正在检查分析作业的状态......", "xpack.infra.logs.analysisPage.unavailable.mlAppButton": "打开 Machine Learning", "xpack.infra.logs.analysisPage.unavailable.mlAppLink": "Machine Learning 应用", diff --git a/x-pack/test/api_integration/apis/infra/log_analysis.ts b/x-pack/test/api_integration/apis/infra/log_analysis.ts index fe7d55649d1d63..2e57678d1d4c25 100644 --- a/x-pack/test/api_integration/apis/infra/log_analysis.ts +++ b/x-pack/test/api_integration/apis/infra/log_analysis.ts @@ -66,7 +66,7 @@ export default ({ getService }: FtrProviderContext) => { expect(logEntryRateBuckets.data.histogramBuckets).to.not.be.empty(); expect( logEntryRateBuckets.data.histogramBuckets.some(bucket => { - return bucket.dataSets.some(dataSet => dataSet.anomalies.length > 0); + return bucket.partitions.some(partition => partition.anomalies.length > 0); }) ).to.be(true); });