From 22ff125b6ec9c3a66baae64ec67a088762707c14 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 22 Jan 2024 20:13:20 +0100 Subject: [PATCH] [Dataset quality] Introduce management fly out (#173554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit related to https://github.com/elastic/kibana/issues/170441 ## 📝 Summary This PR introduces the dataset quality flyout and shows the first 2 sections in the flyout, dataset and integration details. The new Actions column is also added as part of this PR. --------- Co-authored-by: mohamedhamed-ahmed Co-authored-by: Abdul Zahid --- .../dataset_quality/common/api_types.ts | 8 + .../dataset_quality/common/constants.ts | 1 + .../data_streams_stats/data_stream_details.ts | 10 + .../data_streams_stats/data_stream_stat.ts | 11 +- .../common/data_streams_stats/types.ts | 10 +- .../dataset_quality/common/translations.ts | 53 ++++ .../common/types/dataset_types.ts | 19 ++ .../dataset_quality/common/types/index.ts | 8 + .../common/utils/dataset_name.test.ts | 55 +++++ .../common/utils/dataset_name.ts | 40 ++++ .../dataset_quality/common/utils/index.ts | 8 + .../public/components/common/index.ts | 8 + .../components/common/integration_icon.tsx | 30 +++ .../components/dataset_quality/columns.tsx | 72 ++++-- .../components/dataset_quality/table.tsx | 16 +- .../components/flyout/dataset_summary.tsx | 57 +++++ .../public/components/flyout/fields_list.tsx | 82 +++++++ .../public/components/flyout/flyout.tsx | 78 ++++++ .../public/components/flyout/header.tsx | 66 +++++ .../public/components/flyout/index.tsx | 8 + .../components/flyout/integration_summary.tsx | 49 ++++ .../public/components/log_explorer_link.tsx | 54 ----- .../percentage_indicator.tsx | 2 +- .../dataset_quality/public/hooks/index.ts | 2 + .../hooks/use_dataset_quality_flyout.tsx | 64 +++++ .../hooks/use_dataset_quality_table.tsx | 17 +- .../public/hooks/use_link_to_log_explorer.ts | 51 ++++ .../data_streams_stats_client.ts | 27 ++- .../services/data_streams_stats/types.ts | 3 + .../get_data_stream_details.test.ts | 226 ++++++++++++++++++ .../get_data_stream_details/index.ts | 36 +++ .../data_streams/get_data_streams/index.ts | 4 +- .../get_data_streams_stats/index.ts | 4 +- .../routes/data_streams/get_degraded_docs.ts | 15 +- .../server/routes/data_streams/routes.ts | 44 +++- .../server/routes/register_routes.ts | 1 + .../server/services/data_stream.ts | 27 ++- .../server/types/default_api_types.ts | 14 +- .../data_streams/data_stream_details.spec.ts | 90 +++++++ .../utils/data_stream.ts | 13 + .../utils/index.ts | 1 + 41 files changed, 1274 insertions(+), 110 deletions(-) create mode 100644 x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_details.ts create mode 100644 x-pack/plugins/dataset_quality/common/types/dataset_types.ts create mode 100644 x-pack/plugins/dataset_quality/common/types/index.ts create mode 100644 x-pack/plugins/dataset_quality/common/utils/dataset_name.test.ts create mode 100644 x-pack/plugins/dataset_quality/common/utils/dataset_name.ts create mode 100644 x-pack/plugins/dataset_quality/common/utils/index.ts create mode 100644 x-pack/plugins/dataset_quality/public/components/common/index.ts create mode 100644 x-pack/plugins/dataset_quality/public/components/common/integration_icon.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/dataset_summary.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/fields_list.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/flyout.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/header.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/index.tsx create mode 100644 x-pack/plugins/dataset_quality/public/components/flyout/integration_summary.tsx delete mode 100644 x-pack/plugins/dataset_quality/public/components/log_explorer_link.tsx create mode 100644 x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx create mode 100644 x-pack/plugins/dataset_quality/public/hooks/use_link_to_log_explorer.ts create mode 100644 x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts create mode 100644 x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts create mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts create mode 100644 x-pack/test/dataset_quality_api_integration/utils/data_stream.ts diff --git a/x-pack/plugins/dataset_quality/common/api_types.ts b/x-pack/plugins/dataset_quality/common/api_types.ts index 70a5c3e5971481..7be4481b40878f 100644 --- a/x-pack/plugins/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/dataset_quality/common/api_types.ts @@ -54,6 +54,12 @@ export const degradedDocsRt = rt.type({ export type DegradedDocs = rt.TypeOf; +export const dataStreamDetailsRt = rt.type({ + createdOn: rt.number, +}); + +export type DataStreamDetails = rt.TypeOf; + export const getDataStreamsStatsResponseRt = rt.exact( rt.intersection([ rt.type({ @@ -70,3 +76,5 @@ export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact( degradedDocs: rt.array(degradedDocsRt), }) ); + +export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt); diff --git a/x-pack/plugins/dataset_quality/common/constants.ts b/x-pack/plugins/dataset_quality/common/constants.ts index cd5030a4e9a8d7..4808ac4325753e 100644 --- a/x-pack/plugins/dataset_quality/common/constants.ts +++ b/x-pack/plugins/dataset_quality/common/constants.ts @@ -6,6 +6,7 @@ */ export const DATASET_QUALITY_APP_ID = 'dataset_quality'; +export const DEFAULT_DATASET_TYPE = 'logs'; export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3; export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0; diff --git a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_details.ts b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_details.ts new file mode 100644 index 00000000000000..3d4dc00905ec1f --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_details.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface DataStreamDetails { + createdOn?: number; +} diff --git a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 6f806be70c25ba..97fdb0610933de 100644 --- a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -5,22 +5,26 @@ * 2.0. */ +import { DataStreamType } from '../types'; +import { indexNameToDataStreamParts } from '../utils'; import { Integration } from './integration'; -import { DataStreamStatType, IntegrationType } from './types'; +import { DataStreamStatType } from './types'; export class DataStreamStat { rawName: string; + type: DataStreamType; name: DataStreamStatType['name']; namespace: string; title: string; size?: DataStreamStatType['size']; sizeBytes?: DataStreamStatType['sizeBytes']; lastActivity?: DataStreamStatType['lastActivity']; - integration?: IntegrationType; + integration?: Integration; degradedDocs?: number; private constructor(dataStreamStat: DataStreamStat) { this.rawName = dataStreamStat.rawName; + this.type = dataStreamStat.type; this.name = dataStreamStat.name; this.title = dataStreamStat.title ?? dataStreamStat.name; this.namespace = dataStreamStat.namespace; @@ -32,10 +36,11 @@ export class DataStreamStat { } public static create(dataStreamStat: DataStreamStatType) { - const [_type, dataset, namespace] = dataStreamStat.name.split('-'); + const { type, dataset, namespace } = indexNameToDataStreamParts(dataStreamStat.name); const dataStreamStatProps = { rawName: dataStreamStat.name, + type, name: dataset, title: dataStreamStat.integration?.datasets?.[dataset] ?? dataset, namespace, diff --git a/x-pack/plugins/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/dataset_quality/common/data_streams_stats/types.ts index 1db25258a04f8c..c7549d592a0008 100644 --- a/x-pack/plugins/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/dataset_quality/common/data_streams_stats/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { APIClientRequestParamsOf, APIReturnType } from '../rest/create_call_dataset_quality_api'; +import { APIClientRequestParamsOf, APIReturnType } from '../rest'; import { DataStreamStat } from './data_stream_stat'; export type GetDataStreamsStatsParams = @@ -26,3 +26,11 @@ export type GetDataStreamsDegradedDocsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/degraded_docs`>; export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0]; + +export type GetDataStreamDetailsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>['params']['path']; +export type GetDataStreamDetailsResponse = + APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>; + +export type { DataStreamStat } from './data_stream_stat'; +export type { DataStreamDetails } from './data_stream_details'; diff --git a/x-pack/plugins/dataset_quality/common/translations.ts b/x-pack/plugins/dataset_quality/common/translations.ts index 596d1f38ab70cf..1353491f146f7b 100644 --- a/x-pack/plugins/dataset_quality/common/translations.ts +++ b/x-pack/plugins/dataset_quality/common/translations.ts @@ -30,3 +30,56 @@ export const tableSummaryAllText = i18n.translate('xpack.datasetQuality.tableSum export const tableSummaryOfText = i18n.translate('xpack.datasetQuality.tableSummaryOfText', { defaultMessage: 'of', }); + +export const flyoutCancelText = i18n.translate('xpack.datasetQuality.flyoutCancelText', { + defaultMessage: 'Cancel', +}); + +export const flyoutOpenInLogExplorerText = i18n.translate( + 'xpack.datasetQuality.flyoutOpenInLogExplorerText', + { + defaultMessage: 'Open in Logs Explorer', + } +); + +export const flyoutDatasetDetailsText = i18n.translate( + 'xpack.datasetQuality.flyoutDatasetDetailsText', + { + defaultMessage: 'Dataset details', + } +); + +export const flyoutDatasetLastActivityText = i18n.translate( + 'xpack.datasetQuality.flyoutDatasetLastActivityText', + { + defaultMessage: 'Last Activity', + } +); + +export const flyoutDatasetCreatedOnText = i18n.translate( + 'xpack.datasetQuality.flyoutDatasetCreatedOnText', + { + defaultMessage: 'Created on', + } +); + +export const flyoutIntegrationDetailsText = i18n.translate( + 'xpack.datasetQuality.flyoutIntegrationDetailsText', + { + defaultMessage: 'Integration details', + } +); + +export const flyoutIntegrationVersionText = i18n.translate( + 'xpack.datasetQuality.flyoutIntegrationVersionText', + { + defaultMessage: 'Version', + } +); + +export const flyoutIntegrationNameText = i18n.translate( + 'xpack.datasetQuality.flyoutIntegrationNameText', + { + defaultMessage: 'Name', + } +); diff --git a/x-pack/plugins/dataset_quality/common/types/dataset_types.ts b/x-pack/plugins/dataset_quality/common/types/dataset_types.ts new file mode 100644 index 00000000000000..75fb85c1d35421 --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/types/dataset_types.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals +import * as t from 'io-ts'; + +export const dataStreamTypesRt = t.keyof({ + logs: null, + metrics: null, + traces: null, + synthetics: null, + profiling: null, +}); + +export type DataStreamType = t.TypeOf; diff --git a/x-pack/plugins/dataset_quality/common/types/index.ts b/x-pack/plugins/dataset_quality/common/types/index.ts new file mode 100644 index 00000000000000..12303f8b81bb54 --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './dataset_types'; diff --git a/x-pack/plugins/dataset_quality/common/utils/dataset_name.test.ts b/x-pack/plugins/dataset_quality/common/utils/dataset_name.test.ts new file mode 100644 index 00000000000000..fe5b47dc66bb70 --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/utils/dataset_name.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + dataStreamPartsToIndexName, + streamPartsToIndexPattern, + indexNameToDataStreamParts, +} from './dataset_name'; + +describe('dataset_name', () => { + describe('streamPartsToIndexPattern', () => { + it('returns the correct index pattern', () => { + expect( + streamPartsToIndexPattern({ + typePattern: 'logs', + datasetPattern: '*nginx.access*', + }) + ).toEqual('logs-*nginx.access*'); + }); + }); + + describe('dataStreamPartsToIndexName', () => { + it('returns the correct index name', () => { + expect( + dataStreamPartsToIndexName({ + type: 'logs', + dataset: 'nginx.access', + namespace: 'default', + }) + ).toEqual('logs-nginx.access-default'); + }); + }); + + describe('indexNameToDataStreamParts', () => { + it('returns the correct data stream name', () => { + expect(indexNameToDataStreamParts('logs-nginx.access-default')).toEqual({ + type: 'logs', + dataset: 'nginx.access', + namespace: 'default', + }); + }); + + it('handles the case where the dataset name contains a hyphen', () => { + expect(indexNameToDataStreamParts('logs-heartbeat-8-default')).toEqual({ + type: 'logs', + dataset: 'heartbeat-8', + namespace: 'default', + }); + }); + }); +}); diff --git a/x-pack/plugins/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/dataset_quality/common/utils/dataset_name.ts new file mode 100644 index 00000000000000..4bb20e92657b7a --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/utils/dataset_name.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStreamType } from '../types'; + +export interface DataStreamNameParts { + type: DataStreamType; + dataset: string; + namespace: string; +} + +export const streamPartsToIndexPattern = ({ + typePattern, + datasetPattern, +}: { + datasetPattern: string; + typePattern: string; +}) => { + return `${typePattern}-${datasetPattern}`; +}; + +export const dataStreamPartsToIndexName = ({ type, dataset, namespace }: DataStreamNameParts) => { + return `${type}-${dataset}-${namespace}`; +}; + +export const indexNameToDataStreamParts = (dataStreamName: string) => { + const [type, ...dataStreamParts] = dataStreamName.split('-'); + const namespace = dataStreamParts[dataStreamParts.length - 1]; + const dataset = dataStreamParts.slice(0, dataStreamParts.length - 1).join('-'); + + return { + type: type as DataStreamType, + dataset, + namespace, + }; +}; diff --git a/x-pack/plugins/dataset_quality/common/utils/index.ts b/x-pack/plugins/dataset_quality/common/utils/index.ts new file mode 100644 index 00000000000000..780c040fd9b1c2 --- /dev/null +++ b/x-pack/plugins/dataset_quality/common/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './dataset_name'; diff --git a/x-pack/plugins/dataset_quality/public/components/common/index.ts b/x-pack/plugins/dataset_quality/public/components/common/index.ts new file mode 100644 index 00000000000000..36c9d7f744a581 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './integration_icon'; diff --git a/x-pack/plugins/dataset_quality/public/components/common/integration_icon.tsx b/x-pack/plugins/dataset_quality/public/components/common/integration_icon.tsx new file mode 100644 index 00000000000000..9c53dc45a289fe --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/common/integration_icon.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiIcon } from '@elastic/eui'; +import { PackageIcon } from '@kbn/fleet-plugin/public'; +import React from 'react'; +import { Integration } from '../../../common/data_streams_stats/integration'; +import loggingIcon from '../../icons/logging.svg'; + +interface IntegrationIconProps { + integration?: Integration; +} + +export const IntegrationIcon = ({ integration }: IntegrationIconProps) => { + return integration ? ( + + ) : ( + + ); +}; diff --git a/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx b/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx index cdda4dfa004b1e..b4e9035fef215c 100644 --- a/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx +++ b/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx @@ -12,24 +12,33 @@ import { EuiFlexGroup, EuiFlexItem, EuiIcon, + EuiLink, EuiSkeletonRectangle, EuiToolTip, + EuiButtonIcon, + EuiText, } from '@elastic/eui'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; -import { PackageIcon } from '@kbn/fleet-plugin/public'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; -import React from 'react'; +import React, { Dispatch, SetStateAction } from 'react'; +import { css } from '@emotion/react'; import { DEGRADED_QUALITY_MINIMUM_PERCENTAGE, POOR_QUALITY_MINIMUM_PERCENTAGE, } from '../../../common/constants'; import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; -import loggingIcon from '../../icons/logging.svg'; -import { LogExplorerLink } from '../log_explorer_link'; import { QualityIndicator, QualityPercentageIndicator } from '../quality_indicator'; +import { IntegrationIcon } from '../common'; +import { useLinkToLogExplorer } from '../../hooks'; +const expandDatasetAriaLabel = i18n.translate('xpack.datasetQuality.expandLabel', { + defaultMessage: 'Expand', +}); +const collapseDatasetAriaLabel = i18n.translate('xpack.datasetQuality.collapseLabel', { + defaultMessage: 'Collapse', +}); const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', { defaultMessage: 'Dataset Name', }); @@ -96,14 +105,41 @@ const degradedDocsColumnTooltip = ( /> ); -export const getDatasetQualitTableColumns = ({ +export const getDatasetQualityTableColumns = ({ fieldFormats, + selectedDataset, + setSelectedDataset, loadingDegradedStats, }: { fieldFormats: FieldFormatsStart; + selectedDataset?: DataStreamStat; loadingDegradedStats?: boolean; + setSelectedDataset: Dispatch>; }): Array> => { return [ + { + name: '', + render: (dataStreamStat: DataStreamStat) => { + const isExpanded = dataStreamStat === selectedDataset; + + return ( + setSelectedDataset(isExpanded ? undefined : dataStreamStat)} + iconType={isExpanded ? 'minimize' : 'expand'} + title={!isExpanded ? expandDatasetAriaLabel : collapseDatasetAriaLabel} + aria-label={!isExpanded ? expandDatasetAriaLabel : collapseDatasetAriaLabel} + /> + ); + }, + width: '40px', + css: css` + &.euiTableCellContent { + padding: 0; + } + `, + }, { name: nameColumnName, field: 'title', @@ -114,19 +150,9 @@ export const getDatasetQualitTableColumns = ({ return ( - {integration ? ( - - ) : ( - - )} + - {title} + {title} ); }, @@ -187,3 +213,15 @@ export const getDatasetQualitTableColumns = ({ }, ]; }; + +const LogExplorerLink = ({ + dataStreamStat, + title, +}: { + dataStreamStat: DataStreamStat; + title: string; +}) => { + const logExplorerLinkProps = useLinkToLogExplorer({ dataStreamStat }); + + return {title}; +}; diff --git a/x-pack/plugins/dataset_quality/public/components/dataset_quality/table.tsx b/x-pack/plugins/dataset_quality/public/components/dataset_quality/table.tsx index f45df671706368..c78fa66e3f6fc0 100644 --- a/x-pack/plugins/dataset_quality/public/components/dataset_quality/table.tsx +++ b/x-pack/plugins/dataset_quality/public/components/dataset_quality/table.tsx @@ -10,10 +10,20 @@ import { EuiBasicTable, EuiHorizontalRule, EuiSpacer, EuiText, EuiEmptyPrompt } import { FormattedMessage } from '@kbn/i18n-react'; import { loadingDatasetsText, noDatasetsTitle } from '../../../common/translations'; import { useDatasetQualityTable } from '../../hooks'; +import { Flyout } from '../flyout'; export const Table = () => { - const { sort, onTableChange, pagination, renderedItems, columns, loading, resultsCount } = - useDatasetQualityTable(); + const { + sort, + onTableChange, + pagination, + renderedItems, + columns, + loading, + resultsCount, + selectedDataset, + closeFlyout, + } = useDatasetQualityTable(); return ( <> @@ -29,6 +39,7 @@ export const Table = () => { { ) } /> + {selectedDataset && } ); }; diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/dataset_summary.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/dataset_summary.tsx new file mode 100644 index 00000000000000..17691f57dbddf8 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/dataset_summary.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; +import { + flyoutDatasetCreatedOnText, + flyoutDatasetDetailsText, + flyoutDatasetLastActivityText, +} from '../../../common/translations'; +import { DataStreamStat, DataStreamDetails } from '../../../common/data_streams_stats'; +import { FieldsList, FieldsListLoading } from './fields_list'; + +interface DatasetSummaryProps { + fieldFormats: FieldFormatsStart; + dataStreamDetails?: DataStreamDetails; + dataStreamStat: DataStreamStat; +} + +export function DatasetSummary({ + dataStreamStat, + dataStreamDetails, + fieldFormats, +}: DatasetSummaryProps) { + const dataFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ + ES_FIELD_TYPES.DATE, + ]); + const formattedLastActivity = dataFormatter.convert(dataStreamStat.lastActivity); + const formattedCreatedOn = dataStreamDetails?.createdOn + ? dataFormatter.convert(dataStreamDetails.createdOn) + : '-'; + + return ( + + ); +} + +export function DatasetSummaryLoading() { + return ; +} diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/fields_list.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/fields_list.tsx new file mode 100644 index 00000000000000..319b76c1dd05c3 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/fields_list.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { ReactNode, Fragment } from 'react'; +import { + EuiFlexGroup, + EuiPanel, + EuiFlexItem, + EuiSpacer, + EuiTitle, + EuiHorizontalRule, + EuiSkeletonTitle, + EuiSkeletonText, +} from '@elastic/eui'; + +export function FieldsList({ + title, + fields, +}: { + title: string; + fields: Array<{ fieldTitle: string; fieldValue: ReactNode }>; +}) { + return ( + + + {title} + + + + {fields.map(({ fieldTitle, fieldValue }, index) => ( + + + + + {fieldTitle} + + + {fieldValue} + + + {index < fields.length - 1 ? : null} + + ))} + + + ); +} + +export function FieldsListLoading() { + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/flyout.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/flyout.tsx new file mode 100644 index 00000000000000..eaccf05a02ae41 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/flyout.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiSpacer, +} from '@elastic/eui'; +import React, { Fragment } from 'react'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; +import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; +import { flyoutCancelText } from '../../../common/translations'; +import { useDatasetQualityFlyout } from '../../hooks'; +import { DatasetSummary, DatasetSummaryLoading } from './dataset_summary'; +import { Header } from './header'; +import { IntegrationSummary } from './integration_summary'; + +interface FlyoutProps { + dataset: DataStreamStat; + closeFlyout: () => void; +} + +export function Flyout({ dataset, closeFlyout }: FlyoutProps) { + const { + dataStreamStat, + dataStreamDetails, + dataStreamStatLoading, + dataStreamDetailsLoading, + fieldFormats, + } = useDatasetQualityFlyout({ + type: DEFAULT_DATASET_TYPE, + dataset: dataset.name, + namespace: dataset.namespace, + }); + + return ( + + <> +
+ + {dataStreamStatLoading || dataStreamDetailsLoading ? ( + + ) : dataStreamStat ? ( + + + + {dataStreamStat.integration && ( + + )} + + ) : null} + + + + + + + {flyoutCancelText} + + + + + + + ); +} diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/header.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/header.tsx new file mode 100644 index 00000000000000..7e87b2a5a62c94 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/header.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiFlyoutHeader, + EuiTitle, + useEuiShadow, + useEuiTheme, +} from '@elastic/eui'; +import { css } from '@emotion/react'; +import React from 'react'; +import { flyoutOpenInLogExplorerText } from '../../../common/translations'; +import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; +import { useLinkToLogExplorer } from '../../hooks'; +import { IntegrationIcon } from '../common'; + +export function Header({ dataStreamStat }: { dataStreamStat: DataStreamStat }) { + const { integration, title } = dataStreamStat; + const euiShadow = useEuiShadow('s'); + const { euiTheme } = useEuiTheme(); + const logExplorerLinkProps = useLinkToLogExplorer({ dataStreamStat }); + + return ( + + + + + +

{title}

+ +
+ +
+ + + + + + {flyoutOpenInLogExplorerText} + + + + + + ); +} diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/index.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/index.tsx new file mode 100644 index 00000000000000..6a2c75f0054a74 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './flyout'; diff --git a/x-pack/plugins/dataset_quality/public/components/flyout/integration_summary.tsx b/x-pack/plugins/dataset_quality/public/components/flyout/integration_summary.tsx new file mode 100644 index 00000000000000..19b147bcea1fd6 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/components/flyout/integration_summary.tsx @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiBadge, EuiText } from '@elastic/eui'; +import React from 'react'; +import { css } from '@emotion/react'; +import { + flyoutIntegrationDetailsText, + flyoutIntegrationNameText, + flyoutIntegrationVersionText, +} from '../../../common/translations'; +import { Integration } from '../../../common/data_streams_stats/integration'; +import { IntegrationIcon } from '../common'; +import { FieldsList } from './fields_list'; + +export function IntegrationSummary({ integration }: { integration: Integration }) { + const { name, version } = integration; + return ( + + + + {name} + + + ), + }, + { + fieldTitle: flyoutIntegrationVersionText, + fieldValue: version, + }, + ]} + /> + ); +} diff --git a/x-pack/plugins/dataset_quality/public/components/log_explorer_link.tsx b/x-pack/plugins/dataset_quality/public/components/log_explorer_link.tsx deleted file mode 100644 index 11227d0a6bd3a5..00000000000000 --- a/x-pack/plugins/dataset_quality/public/components/log_explorer_link.tsx +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiLink } from '@elastic/eui'; -import React from 'react'; -import { getRouterLinkProps } from '@kbn/router-utils'; -import { - SingleDatasetLocatorParams, - SINGLE_DATASET_LOCATOR_ID, -} from '@kbn/deeplinks-observability'; -import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; -import { useKibanaContextForPlugin } from '../utils'; - -export const LogExplorerLink = React.memo( - ({ dataStreamStat, title }: { dataStreamStat: DataStreamStat; title: string }) => { - const { - services: { share }, - } = useKibanaContextForPlugin(); - const params: SingleDatasetLocatorParams = { - dataset: dataStreamStat.name, - timeRange: { - from: 'now-1d', - to: 'now', - }, - integration: dataStreamStat.integration?.name, - filterControls: { - namespace: { - mode: 'include', - values: [dataStreamStat.namespace], - }, - }, - }; - - const singleDatasetLocator = - share.url.locators.get(SINGLE_DATASET_LOCATOR_ID); - - const urlToLogExplorer = singleDatasetLocator?.getRedirectUrl(params); - - const navigateToLogExplorer = () => { - singleDatasetLocator?.navigate(params) as Promise; - }; - - const logExplorerLinkProps = getRouterLinkProps({ - href: urlToLogExplorer, - onClick: navigateToLogExplorer, - }); - - return {title}; - } -); diff --git a/x-pack/plugins/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx b/x-pack/plugins/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx index f5034bddab8c25..ceaf1f43a93381 100644 --- a/x-pack/plugins/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx +++ b/x-pack/plugins/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx @@ -23,7 +23,7 @@ export function QualityPercentageIndicator({ percentage = 0 }: { percentage?: nu : 'good'; const description = ( - + % ); diff --git a/x-pack/plugins/dataset_quality/public/hooks/index.ts b/x-pack/plugins/dataset_quality/public/hooks/index.ts index 36b6f1540c8288..29425c9b326d7e 100644 --- a/x-pack/plugins/dataset_quality/public/hooks/index.ts +++ b/x-pack/plugins/dataset_quality/public/hooks/index.ts @@ -6,3 +6,5 @@ */ export * from './use_dataset_quality_table'; +export * from './use_dataset_quality_flyout'; +export * from './use_link_to_log_explorer'; diff --git a/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx b/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx new file mode 100644 index 00000000000000..6f1cff66c6df71 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { useFetcher } from '@kbn/observability-shared-plugin/public'; +import { DataStreamStatServiceResponse } from '../../common/data_streams_stats'; +import { DataStreamNameParts, dataStreamPartsToIndexName } from '../../common/utils'; +import { useDatasetQualityContext } from '../components/dataset_quality/context'; +import { useKibanaContextForPlugin } from '../utils'; + +export const useDatasetQualityFlyout = ({ type, dataset, namespace }: DataStreamNameParts) => { + const { + services: { fieldFormats }, + } = useKibanaContextForPlugin(); + const { dataStreamsStatsServiceClient: client } = useDatasetQualityContext(); + const { data: dataStreamStatList = [], loading: dataStreamStatLoading } = useFetcher( + async () => client.getDataStreamsStats({ datasetQuery: `${dataset}-${namespace}`, type }), + [dataset, namespace, type] + ); + + const { data: dataStreamDetails = {}, loading: dataStreamDetailsLoading } = useFetcher( + async () => + client.getDataStreamDetails({ + dataStream: dataStreamPartsToIndexName({ type, dataset, namespace }), + }), + [dataset, namespace, type] + ); + + return useMemo(() => { + const isDataStreamStatStale = isStaleData({ type, dataset, namespace }, dataStreamStatList[0]); + + return { + dataStreamStat: isDataStreamStatStale ? undefined : dataStreamStatList[0], + dataStreamDetails: isDataStreamStatStale ? undefined : dataStreamDetails, + dataStreamStatLoading, + dataStreamDetailsLoading, + fieldFormats, + }; + }, [ + type, + dataset, + namespace, + dataStreamStatList, + dataStreamStatLoading, + dataStreamDetails, + dataStreamDetailsLoading, + fieldFormats, + ]); +}; + +function isStaleData(args: DataStreamNameParts, dataStreamStat?: DataStreamStatServiceResponse[0]) { + return ( + dataStreamStat && + dataStreamPartsToIndexName({ + type: dataStreamStat.type, + dataset: dataStreamStat.name, + namespace: dataStreamStat.namespace, + }) !== dataStreamPartsToIndexName(args) + ); +} diff --git a/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_table.tsx index c68fb3a47bd1d6..5230a3e79cdf94 100644 --- a/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -10,7 +10,7 @@ import { find, orderBy } from 'lodash'; import React, { useCallback, useMemo, useState } from 'react'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { tableSummaryAllText, tableSummaryOfText } from '../../common/translations'; -import { getDatasetQualitTableColumns } from '../components/dataset_quality/columns'; +import { getDatasetQualityTableColumns } from '../components/dataset_quality/columns'; import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { getDefaultTimeRange, useKibanaContextForPlugin } from '../utils'; @@ -28,6 +28,7 @@ export const useDatasetQualityTable = () => { const { services: { fieldFormats }, } = useKibanaContextForPlugin(); + const [selectedDataset, setSelectedDataset] = useState(); const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); @@ -47,8 +48,14 @@ export const useDatasetQualityTable = () => { ); const columns = useMemo( - () => getDatasetQualitTableColumns({ fieldFormats, loadingDegradedStats }), - [fieldFormats, loadingDegradedStats] + () => + getDatasetQualityTableColumns({ + fieldFormats, + selectedDataset, + setSelectedDataset, + loadingDegradedStats, + }), + [fieldFormats, loadingDegradedStats, selectedDataset, setSelectedDataset] ); const pagination = { @@ -107,6 +114,8 @@ export const useDatasetQualityTable = () => { ); }, [data.length, pageIndex, pageSize, renderedItems.length]); + const closeFlyout = useCallback(() => setSelectedDataset(undefined), []); + return { sort, onTableChange, @@ -115,5 +124,7 @@ export const useDatasetQualityTable = () => { columns, loading, resultsCount, + closeFlyout, + selectedDataset, }; }; diff --git a/x-pack/plugins/dataset_quality/public/hooks/use_link_to_log_explorer.ts b/x-pack/plugins/dataset_quality/public/hooks/use_link_to_log_explorer.ts new file mode 100644 index 00000000000000..ee019cb75c7ba8 --- /dev/null +++ b/x-pack/plugins/dataset_quality/public/hooks/use_link_to_log_explorer.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SINGLE_DATASET_LOCATOR_ID, + SingleDatasetLocatorParams, +} from '@kbn/deeplinks-observability'; +import { getRouterLinkProps } from '@kbn/router-utils'; +import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; +import { useKibanaContextForPlugin } from '../utils'; + +export const useLinkToLogExplorer = ({ dataStreamStat }: { dataStreamStat: DataStreamStat }) => { + const { + services: { share }, + } = useKibanaContextForPlugin(); + + const params: SingleDatasetLocatorParams = { + dataset: dataStreamStat.name, + timeRange: { + from: 'now-1d', + to: 'now', + }, + integration: dataStreamStat.integration?.name, + filterControls: { + namespace: { + mode: 'include', + values: [dataStreamStat.namespace], + }, + }, + }; + + const singleDatasetLocator = + share.url.locators.get(SINGLE_DATASET_LOCATOR_ID); + + const urlToLogExplorer = singleDatasetLocator?.getRedirectUrl(params); + + const navigateToLogExplorer = () => { + singleDatasetLocator?.navigate(params) as Promise; + }; + + const logExplorerLinkProps = getRouterLinkProps({ + href: urlToLogExplorer, + onClick: navigateToLogExplorer, + }); + + return logExplorerLinkProps; +}; diff --git a/x-pack/plugins/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 5fe6f9ab282316..2a4041934c7bfc 100644 --- a/x-pack/plugins/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -11,7 +11,9 @@ import { find, merge } from 'lodash'; import { getDataStreamsDegradedDocsStatsResponseRt, getDataStreamsStatsResponseRt, + getDataStreamsDetailsResponseRt, } from '../../../common/api_types'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, @@ -19,7 +21,10 @@ import { GetDataStreamsStatsError, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, + GetDataStreamDetailsParams, + GetDataStreamDetailsResponse, } from '../../../common/data_streams_stats'; +import { DataStreamDetails } from '../../../common/data_streams_stats'; import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat'; import { IDataStreamsStatsClient } from './types'; @@ -27,7 +32,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { constructor(private readonly http: HttpStart) {} public async getDataStreamsStats( - params: GetDataStreamsStatsQuery = { type: 'logs' } + params: GetDataStreamsStatsQuery = { type: DEFAULT_DATASET_TYPE } ): Promise { const response = await this.http .get('/internal/dataset_quality/data_streams/stats', { @@ -59,7 +64,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { { query: { ...params, - type: 'logs', + type: DEFAULT_DATASET_TYPE, }, } ) @@ -79,4 +84,22 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return degradedDocs; } + + public async getDataStreamDetails({ dataStream }: GetDataStreamDetailsParams) { + const response = await this.http + .get( + `/internal/dataset_quality/data_streams/${dataStream}/details` + ) + .catch((error) => { + throw new GetDataStreamsStatsError(`Failed to fetch data stream details": ${error}`); + }); + + const dataStreamDetails = decodeOrThrow( + getDataStreamsDetailsResponseRt, + (message: string) => + new GetDataStreamsStatsError(`Failed to decode data stream details response: ${message}"`) + )(response); + + return dataStreamDetails as DataStreamDetails; + } } diff --git a/x-pack/plugins/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/dataset_quality/public/services/data_streams_stats/types.ts index 46c39082ce1d40..26490f0a0bf43a 100644 --- a/x-pack/plugins/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/dataset_quality/public/services/data_streams_stats/types.ts @@ -11,7 +11,9 @@ import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, GetDataStreamsStatsQuery, + GetDataStreamDetailsParams, } from '../../../common/data_streams_stats'; +import { DataStreamDetails } from '../../../common/data_streams_stats/data_stream_details'; export type DataStreamsStatsServiceSetup = void; @@ -28,4 +30,5 @@ export interface IDataStreamsStatsClient { getDataStreamsDegradedStats( params?: GetDataStreamsDegradedDocsStatsQuery ): Promise; + getDataStreamDetails(params: GetDataStreamDetailsParams): Promise; } diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts new file mode 100644 index 00000000000000..faaf3d1c44b6f8 --- /dev/null +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + +import { getDataStreamDetails } from '.'; +const accessLogsDataStream = 'logs-nginx.access-default'; +const errorLogsDataStream = 'logs-nginx.error-default'; +const dateStr1 = '1702998651925'; // .ds-logs-nginx.access-default-2023.12.19-000001 +const dateStr2 = '1703110671019'; // .ds-logs-nginx.access-default-2023.12.20-000002 +const dateStr3 = '1702998866744'; // .ds-logs-nginx.error-default-2023.12.19-000001 + +describe('getDataStreamDetails', () => { + afterAll(() => { + jest.clearAllMocks(); + }); + + it('throws error if index is not found', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockRejectedValue(MOCK_INDEX_ERROR); + + try { + await getDataStreamDetails({ + esClient: esClientMock, + dataStream: 'non-existent', + }); + } catch (e) { + expect(e).toBe(MOCK_INDEX_ERROR); + } + }); + + it('returns creation date of a data stream', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockReturnValue( + Promise.resolve(MOCK_NGINX_ERROR_INDEX_SETTINGS) + ); + + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: errorLogsDataStream, + }); + expect(dataStreamDetails).toEqual({ createdOn: Number(dateStr3) }); + }); + + it('returns the earliest creation date of a data stream with multiple backing indices', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockReturnValue( + Promise.resolve(MOCK_NGINX_ACCESS_INDEX_SETTINGS) + ); + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: accessLogsDataStream, + }); + expect(dataStreamDetails).toEqual({ createdOn: Number(dateStr1) }); + }); +}); + +const MOCK_NGINX_ACCESS_INDEX_SETTINGS = { + [`.ds-${accessLogsDataStream}-2023.12.19-000001`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.access-default-2023.12.19-000001', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'cloud.account.id', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.machine.type', + 'cloud.provider', + 'cloud.region', + ], + }, + creation_date: dateStr1, + number_of_replicas: '1', + uuid: 'uml9fMQqQUibZi2pKkc5sQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + indexing_complete: true, + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.access-1.17.0', + }, + }, + }, + [`.ds-${accessLogsDataStream}-2023.12.20-000002`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.access-default-2023.12.20-000002', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'user.name', + 'user_agent.device.name', + 'user_agent.name', + 'user_agent.original', + 'user_agent.os.full', + 'user_agent.os.name', + 'user_agent.os.version', + 'user_agent.version', + 'nginx.access.remote_ip_list', + ], + }, + creation_date: dateStr2, + number_of_replicas: '1', + uuid: 'il9vJlOXRdiv44wU6WNtUQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.access-1.17.0', + }, + }, + }, +}; + +const MOCK_NGINX_ERROR_INDEX_SETTINGS = { + [`.ds-${errorLogsDataStream}-2023.12.19-000001`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.error-default-2023.12.19-000001', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'host.type', + 'input.type', + 'log.file.path', + 'log.level', + 'ecs.version', + 'message', + 'tags', + ], + }, + creation_date: dateStr3, + number_of_replicas: '1', + uuid: 'fGPYUppSRU62MZ3toF0MkQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.error-1.17.0', + }, + }, + }, +}; + +const MOCK_INDEX_ERROR = { + error: { + root_cause: [ + { + type: 'index_not_found_exception', + reason: 'no such index [logs-nginx.error-default-01]', + 'resource.type': 'index_or_alias', + 'resource.id': 'logs-nginx.error-default-01', + index_uuid: '_na_', + index: 'logs-nginx.error-default-01', + }, + ], + type: 'index_not_found_exception', + reason: 'no such index [logs-nginx.error-default-01]', + 'resource.type': 'index_or_alias', + 'resource.id': 'logs-nginx.error-default-01', + index_uuid: '_na_', + index: 'logs-nginx.error-default-01', + }, + status: 404, +}; diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts new file mode 100644 index 00000000000000..5d122df09398ff --- /dev/null +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { DataStreamDetails } from '../../../../common/api_types'; +import { dataStreamService } from '../../../services'; + +export async function getDataStreamDetails(args: { + esClient: ElasticsearchClient; + dataStream: string; +}): Promise { + const { esClient, dataStream } = args; + + if (!dataStream?.trim()) { + throw new Error(`Data Stream name cannot be empty. Received value "${dataStream}"`); + } + + const indexSettings = await dataStreamService.getDataSteamIndexSettings(esClient, dataStream); + + const indexesList = Object.values(indexSettings); + if (indexesList.length < 1) { + throw new Error('index_not_found_exception'); + } + + const indexCreationDate = indexesList + .map((index) => Number(index.settings?.index?.creation_date)) + .sort((a, b) => a - b)[0]; + + return { + createdOn: indexCreationDate, + }; +} diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index b79b4eeec01162..d101985db46616 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -6,12 +6,12 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { DataStreamTypes } from '../../../types/default_api_types'; +import { DataStreamType } from '../../../../common/types'; import { dataStreamService } from '../../../services'; export async function getDataStreams(options: { esClient: ElasticsearchClient; - type?: DataStreamTypes; + type?: DataStreamType; datasetQuery?: string; uncategorisedOnly: boolean; }) { diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts index a78f2fec53e29f..508f109f0c5288 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts @@ -6,12 +6,12 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { DataStreamTypes } from '../../../types/default_api_types'; +import { DataStreamType } from '../../../../common/types'; import { dataStreamService } from '../../../services'; export async function getDataStreamsStats(options: { esClient: ElasticsearchClient; - type?: DataStreamTypes; + type?: DataStreamType; datasetQuery?: string; }) { const { esClient, type, datasetQuery } = options; diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts index 1af5a603f36384..558a4a4ab4f7ca 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts @@ -7,6 +7,8 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; +import { DataStreamType } from '../../../common/types'; import { DegradedDocs } from '../../../common/api_types'; import { DATA_STREAM_DATASET, @@ -14,12 +16,11 @@ import { DATA_STREAM_TYPE, _IGNORED, } from '../../../common/es_fields'; -import { DataStreamTypes } from '../../types/default_api_types'; import { createDatasetQualityESClient, wildcardQuery } from '../../utils'; export async function getDegradedDocsPaginated(options: { esClient: ElasticsearchClient; - type?: DataStreamTypes; + type?: DataStreamType; start: number; end: number; datasetQuery?: string; @@ -29,7 +30,15 @@ export async function getDegradedDocsPaginated(options: { }; prevResults?: DegradedDocs[]; }): Promise { - const { esClient, type = 'logs', datasetQuery, start, end, after, prevResults = [] } = options; + const { + esClient, + type = DEFAULT_DATASET_TYPE, + datasetQuery, + start, + end, + after, + prevResults = [], + } = options; const datasetQualityESClient = createDatasetQualityESClient(esClient); diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts index 4f95331d97651e..340c957f6e41ae 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts @@ -7,12 +7,19 @@ import * as t from 'io-ts'; import { keyBy, merge, values } from 'lodash'; -import { typeRt, rangeRt } from '../../types/default_api_types'; +import Boom from '@hapi/boom'; +import { + DataStreamDetails, + DataStreamStat, + DegradedDocs, + Integration, +} from '../../../common/api_types'; +import { rangeRt, typeRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; +import { getDataStreamDetails } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; -import { DegradedDocs, DataStreamStat, Integration } from '../../../common/api_types'; import { getIntegrations } from './get_integrations'; const statsRoute = createDatasetQualityServerRoute({ @@ -92,7 +99,40 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ }, }); +const dataStreamDetailsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const { dataStream } = params.path; + const coreContext = await context.core; + + // Query datastreams as the current user as the Kibana internal user may not have all the required permissions + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + try { + return await getDataStreamDetails({ esClient, dataStream }); + } catch (e) { + if (e) { + if (e?.message?.indexOf('index_not_found_exception') > -1) { + throw Boom.notFound(`Data stream "${dataStream}" not found.`); + } + } + + throw e; + } + }, +}); + export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, + ...dataStreamDetailsRoute, }; diff --git a/x-pack/plugins/dataset_quality/server/routes/register_routes.ts b/x-pack/plugins/dataset_quality/server/routes/register_routes.ts index 7ebecaa1346f3a..9a4b25356a3770 100644 --- a/x-pack/plugins/dataset_quality/server/routes/register_routes.ts +++ b/x-pack/plugins/dataset_quality/server/routes/register_routes.ts @@ -73,6 +73,7 @@ export function registerRoutes({ repository, core, logger, plugins }: RegisterRo } logger.error(error); + const opts = { statusCode: 500, body: { diff --git a/x-pack/plugins/dataset_quality/server/services/data_stream.ts b/x-pack/plugins/dataset_quality/server/services/data_stream.ts index e0fc6e67488298..c8e675548354ed 100644 --- a/x-pack/plugins/dataset_quality/server/services/data_stream.ts +++ b/x-pack/plugins/dataset_quality/server/services/data_stream.ts @@ -11,11 +11,9 @@ import type { } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; -class DataStreamService { - public streamPartsToIndexPattern({ type, dataset }: { dataset: string; type: string }) { - return `${type}-${dataset}`; - } +import { streamPartsToIndexPattern } from '../../common/utils'; +class DataStreamService { public async getMatchingDataStreams( esClient: ElasticsearchClient, dataStreamParts: { @@ -25,7 +23,10 @@ class DataStreamService { ): Promise { try { const { data_streams: dataStreamsInfo } = await esClient.indices.getDataStream({ - name: this.streamPartsToIndexPattern(dataStreamParts), + name: streamPartsToIndexPattern({ + typePattern: dataStreamParts.type, + datasetPattern: dataStreamParts.dataset, + }), }); return dataStreamsInfo; @@ -46,7 +47,10 @@ class DataStreamService { ): Promise { try { const { data_streams: dataStreamsStats } = await esClient.indices.dataStreamsStats({ - name: this.streamPartsToIndexPattern(dataStreamParts), + name: streamPartsToIndexPattern({ + typePattern: dataStreamParts.type, + datasetPattern: dataStreamParts.dataset, + }), human: true, }); @@ -58,6 +62,17 @@ class DataStreamService { throw e; } } + + public async getDataSteamIndexSettings( + esClient: ElasticsearchClient, + dataStream: string + ): Promise>> { + const settings = await esClient.indices.getSettings({ + index: dataStream, + }); + + return settings; + } } export const dataStreamService = new DataStreamService(); diff --git a/x-pack/plugins/dataset_quality/server/types/default_api_types.ts b/x-pack/plugins/dataset_quality/server/types/default_api_types.ts index 6148832dad140b..91eb8698dfcacf 100644 --- a/x-pack/plugins/dataset_quality/server/types/default_api_types.ts +++ b/x-pack/plugins/dataset_quality/server/types/default_api_types.ts @@ -5,24 +5,14 @@ * 2.0. */ -import * as t from 'io-ts'; import { isoToEpochRt } from '@kbn/io-ts-utils'; - -// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals -export const dataStreamTypesRt = t.keyof({ - logs: null, - metrics: null, - traces: null, - synthetics: null, - profiling: null, -}); +import * as t from 'io-ts'; +import { dataStreamTypesRt } from '../../common/types'; export const typeRt = t.partial({ type: dataStreamTypesRt, }); -export type DataStreamTypes = t.TypeOf; - export const rangeRt = t.type({ start: isoToEpochRt, end: isoToEpochRt, diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts new file mode 100644 index 00000000000000..8c8643dc6ee04f --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { expectToReject, getDataStreamSettingsOfFirstIndex } from '../../utils'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const synthtrace = getService('logSynthtraceEsClient'); + const esClient = getService('es'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + const type = 'logs'; + const dataset = 'nginx.access'; + const namespace = 'default'; + + async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details', + params: { + path: { + dataStream, + }, + }, + }); + } + + registry.when('DataStream Details', { config: 'basic' }, () => { + describe('gets the data stream details', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + }) + ), + ]); + }); + + it('returns error when dataStream param is not provided', async () => { + expect( + (await expectToReject(() => callApiAs('datasetQualityLogsUser', encodeURIComponent(' ')))) + .message + ).to.contain('Data Stream name cannot be empty'); + }); + + it('returns 404 if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const expectedMessage = `"${nonExistentDataStream}" not found`; + const err = await expectToReject(() => + callApiAs('datasetQualityLogsUser', nonExistentDataStream) + ); + expect(err.res.status).to.be(404); + expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); + }); + + it('returns data stream details correctly', async () => { + const dataStreamSettings = await getDataStreamSettingsOfFirstIndex( + esClient, + `logs-${dataset}-${namespace}` + ); + const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + }); +} diff --git a/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts b/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts new file mode 100644 index 00000000000000..ff3a3cc95d1ac2 --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; + +export async function getDataStreamSettingsOfFirstIndex(es: Client, name: string) { + const matchingIndexesObj = await es.indices.getSettings({ index: name }); + return Object.values(matchingIndexesObj ?? {})[0]?.settings; +} diff --git a/x-pack/test/dataset_quality_api_integration/utils/index.ts b/x-pack/test/dataset_quality_api_integration/utils/index.ts index ae42a0d359d0b0..31fae4fe71199c 100644 --- a/x-pack/test/dataset_quality_api_integration/utils/index.ts +++ b/x-pack/test/dataset_quality_api_integration/utils/index.ts @@ -8,3 +8,4 @@ export { joinByKey } from './join_by_key'; export { maybe } from './maybe'; export { expectToReject } from './expect_to_reject'; +export * from './data_stream';