[Security Solution] Data Quality dashboard

.eslintrc.js

@@ -946,6 +946,8 @@ module.exports = {
     {
       // front end and common typescript and javascript files only
       files: [
+        'x-pack/plugins/ecs_data_quality_dashboard/common/**/*.{js,mjs,ts,tsx}',
+        'x-pack/packages/kbn-ecs-data-quality-dashboard/common/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/public/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/common/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/public/**/*.{js,mjs,ts,tsx}',
@@ -971,11 +973,15 @@ module.exports = {
       // We use this section to add rules in which we do not want to apply to test files.
       // This should be a very small set as most linter rules are useful for tests as well.
       files: [
+        'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
+        'x-pack/packages/kbn-ecs-data-quality-dashboard/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{ts,tsx}',
       ],
       excludedFiles: [
+        'x-pack/plugins/ecs_data_quality_dashboard/**/*.{test,mock,test_helper}.{ts,tsx}',
+        'x-pack/packages/kbn-ecs-data-quality-dashboard/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{test,mock,test_helper}.{ts,tsx}',
@@ -987,6 +993,8 @@ module.exports = {
     {
       // typescript only for front and back end
       files: [
+        'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
+        'x-pack/packages/kbn-ecs-data-quality-dashboard/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{ts,tsx}',
@@ -1017,6 +1025,8 @@ module.exports = {
     {
       // typescript and javascript for front and back end
       files: [
+        'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
+        'x-pack/packages/kbn-ecs-data-quality-dashboard/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',

.github/CODEOWNERS

@@ -442,6 +442,8 @@
 # Security Solution sub teams
 
 ## Security Solution sub teams - Threat Hunting Investigations
+/x-pack/packages/kbn-ecs-data-quality-dashboard @elastic/security-threat-hunting-investigations
+/x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
 /x-pack/plugins/timelines @elastic/security-threat-hunting-investigations
 
 /x-pack/plugins/security_solution/common/search_strategy/timeline @elastic/security-threat-hunting-investigations
@@ -1083,6 +1085,7 @@ packages/shared-ux/router/types @elastic/appex-sharedux
 packages/shared-ux/storybook/config @elastic/appex-sharedux
 packages/shared-ux/storybook/mock @elastic/appex-sharedux
 src/plugins/chart_expressions/common @elastic/kibana-visualizations
+x-pack/packages/kbn-ecs-data-quality-dashboard @elastic/security-threat-hunting-investigations
 x-pack/packages/ml/agg_utils @elastic/ml-ui
 x-pack/packages/ml/aiops_components @elastic/ml-ui
 x-pack/packages/ml/aiops_utils @elastic/ml-ui

.i18nrc.json

@@ -15,6 +15,7 @@
     "dashboard": "src/plugins/dashboard",
     "controls": "src/plugins/controls",
     "data": "src/plugins/data",
+    "ecsDataQualityDashboard": "x-pack/packages/kbn-ecs-data-quality-dashboard",
     "dataViews": "src/plugins/data_views",
     "devTools": "src/plugins/dev_tools",
     "discover": "src/plugins/discover",

docs/developer/plugin-list.asciidoc

@@ -502,6 +502,10 @@ This plugin is currently under active development; do not use.
 |Contains the enhancements to the OSS discover app.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ecs_data_quality_dashboard/README.md[ecsDataQualityDashboard]
+|This plugin implements (server) APIs used to render the content of the Data Quality dashboard.
+
+
 |<<enhanced-embeddables-plugin>>
 |Enhances Embeddables by registering a custom factory provider. The enhanced factory provider
 adds dynamic actions to every embeddables state, in order to support drilldowns.

package.json

@@ -286,6 +286,7 @@
     "@kbn/doc-links": "link:packages/kbn-doc-links",
     "@kbn/ebt-tools": "link:packages/kbn-ebt-tools",
     "@kbn/ecs": "link:packages/kbn-ecs",
+    "@kbn/ecs-data-quality-dashboard": "link:x-pack/packages/kbn-ecs-data-quality-dashboard",
     "@kbn/es-errors": "link:packages/kbn-es-errors",
     "@kbn/es-query": "link:packages/kbn-es-query",
     "@kbn/es-types": "link:packages/kbn-es-types",

packages/kbn-babel-preset/styled_components_files.js

@@ -12,7 +12,7 @@ module.exports = {
    * Used by `kbn-babel-preset` and `kbn-eslint-config`.
    */
   USES_STYLED_COMPONENTS: [
-    /packages[\/\\]kbn-ui-shared-deps-(npm|src)[\/\\]/,
+    /packages[\/\\](kbn-ui-shared-deps-(npm|src)|kbn-ecs-data-quality-dashboard)[\/\\]/,
     /src[\/\\]plugins[\/\\](kibana_react)[\/\\]/,
     /x-pack[\/\\]plugins[\/\\](apm|beats_management|cases|fleet|infra|lists|observability|osquery|security_solution|timelines|synthetics|ux)[\/\\]/,
     /x-pack[\/\\]test[\/\\]plugin_functional[\/\\]plugins[\/\\]resolver_test[\/\\]/,

packages/kbn-eslint-config/.eslintrc.js

@@ -88,7 +88,7 @@ module.exports = {
           from: 'styled-components',
           to: false,
           exclude: USES_STYLED_COMPONENTS,
-          disallowedMessage: `Prefer using @emotion/react instead. To use styled-components, ensure you plugin is enabled in @kbn/dev-utils/src/babel.ts.`
+          disallowedMessage: `Prefer using @emotion/react instead. To use styled-components, ensure you plugin is enabled in packages/kbn-babel-preset/styled_components_files.js.`
         },
         ...[
           '@elastic/eui/dist/eui_theme_light.json',

src/dev/precommit_hook/casing_check_config.js

@@ -82,7 +82,7 @@ export const IGNORE_FILE_GLOBS = [
  *
  * @type {Array}
  */
-export const KEBAB_CASE_DIRECTORY_GLOBS = ['packages/*', 'x-pack'];
+export const KEBAB_CASE_DIRECTORY_GLOBS = ['packages/*', 'x-pack', 'x-pack/packages/*'];
 
 /**
  * These patterns are matched against directories and indicate

tsconfig.base.json

@@ -602,6 +602,10 @@
       "@kbn/ebt-tools/*": ["packages/kbn-ebt-tools/*"],
       "@kbn/ecs": ["packages/kbn-ecs"],
       "@kbn/ecs/*": ["packages/kbn-ecs/*"],
+      "@kbn/ecs-data-quality-dashboard": ["x-pack/packages/kbn-ecs-data-quality-dashboard"],
+      "@kbn/ecs-data-quality-dashboard/*": ["x-pack/packages/kbn-ecs-data-quality-dashboard/*"],
+      "@kbn/ecs-data-quality-dashboard-plugin": ["x-pack/plugins/ecs_data_quality_dashboard"],
+      "@kbn/ecs-data-quality-dashboard-plugin/*": ["x-pack/plugins/ecs_data_quality_dashboard/*"],
       "@kbn/elasticsearch-client-plugin": ["test/plugin_functional/plugins/elasticsearch_client_plugin"],
       "@kbn/elasticsearch-client-plugin/*": ["test/plugin_functional/plugins/elasticsearch_client_plugin/*"],
       "@kbn/elasticsearch-client-xpack-plugin": ["x-pack/test/plugin_api_integration/plugins/elasticsearch_client"],

x-pack/packages/kbn-ecs-data-quality-dashboard/README.md

@@ -0,0 +1,12 @@
+# @kbn/ecs-data-quality-dashboard
+
+This package exports the `DataQualityPanel` React component.
+
+The `DataQualityPanel` renders the content of the _Data Quality_ dashboard.
+
+- The Data Quality dashboard checks the mappings and values in indices for [ECS](https://github.com/elastic/ecs) compliance
+- Results are added to a new case, or exported as markdown
+
+## Maintainers
+
+Maintained by the Security Solution _Threat Hunting Investigations_ team

x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/compare_fields_table/ecs_allowed_values/index.tsx

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiCode, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import React from 'react';
+import styled from 'styled-components';
+
+import { EMPTY_PLACEHOLDER } from '../helpers';
+import { CodeSuccess } from '../../styles';
+import type { AllowedValue } from '../../types';
+
+const EcsAllowedValueFlexItem = styled(EuiFlexItem)`
+  margin-bottom: ${({ theme }) => theme.eui.euiSizeXS};
+`;
+
+interface Props {
+  allowedValues: AllowedValue[] | undefined;
+}
+
+const EcsAllowedValuesComponent: React.FC<Props> = ({ allowedValues }) =>
+  allowedValues == null ? (
+    <EuiCode>{EMPTY_PLACEHOLDER}</EuiCode>
+  ) : (
+    <EuiFlexGroup direction="column" gutterSize="none">
+      {allowedValues.map((x, i) => (
+        <EcsAllowedValueFlexItem grow={false} key={`${x.name}_${i}`}>
+          <CodeSuccess>{x.name}</CodeSuccess>
+        </EcsAllowedValueFlexItem>
+      ))}
+    </EuiFlexGroup>
+  );
+
+EcsAllowedValuesComponent.displayName = 'EcsAllowedValuesComponent';
+
+export const EcsAllowedValues = React.memo(EcsAllowedValuesComponent);

x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/compare_fields_table/helpers.tsx

@@ -0,0 +1,214 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EuiTableFieldDataColumnType } from '@elastic/eui';
+import { EuiCode } from '@elastic/eui';
+import React from 'react';
+
+import { EcsAllowedValues } from './ecs_allowed_values';
+import { IndexInvalidValues } from './index_invalid_values';
+import { CodeDanger, CodeSuccess } from '../styles';
+import * as i18n from './translations';
+import type { AllowedValue, EnrichedFieldMetadata, UnallowedValueCount } from '../types';
+
+export const EMPTY_PLACEHOLDER = '--';
+
+export const getCommonTableColumns = (): Array<
+  EuiTableFieldDataColumnType<EnrichedFieldMetadata>
+> => [
+  {
+    field: 'indexFieldName',
+    name: i18n.FIELD,
+    sortable: true,
+    truncateText: false,
+    width: '20%',
+  },
+  {
+    field: 'type',
+    name: i18n.ECS_MAPPING_TYPE_EXPECTED,
+    render: (type: string) => <CodeSuccess>{type != null ? type : EMPTY_PLACEHOLDER}</CodeSuccess>,
+    sortable: true,
+    truncateText: false,
+    width: '15%',
+  },
+  {
+    field: 'indexFieldType',
+    name: i18n.INDEX_MAPPING_TYPE_ACTUAL,
+    render: (_, x) =>
+      x.type != null && x.indexFieldType !== x.type ? (
+        <CodeDanger>{x.indexFieldType}</CodeDanger>
+      ) : (
+        <CodeSuccess>{x.indexFieldType}</CodeSuccess>
+      ),
+    sortable: true,
+    truncateText: false,
+    width: '15%',
+  },
+  {
+    field: 'allowed_values',
+    name: i18n.ECS_VALUES_EXPECTED,
+    render: (allowedValues: AllowedValue[] | undefined) => (
+      <EcsAllowedValues allowedValues={allowedValues} />
+    ),
+    sortable: false,
+    truncateText: false,
+    width: '15%',
+  },
+  {
+    field: 'indexInvalidValues',
+    name: i18n.DOCUMENT_VALUES_ACTUAL,
+    render: (indexInvalidValues: UnallowedValueCount[]) => (
+      <IndexInvalidValues indexInvalidValues={indexInvalidValues} />
+    ),
+    sortable: false,
+    truncateText: false,
+    width: '15%',
+  },
+  {
+    field: 'description',
+    name: i18n.ECS_DESCRIPTION,
+    render: (description: string) =>
+      description != null ? description : <EuiCode>{EMPTY_PLACEHOLDER}</EuiCode>,
+    sortable: false,
+    truncateText: false,
+    width: '20%',
+  },
+];
+
+export const getCustomTableColumns = (): Array<
+  EuiTableFieldDataColumnType<EnrichedFieldMetadata>
+> => [
+  {
+    field: 'indexFieldName',
+    name: i18n.FIELD,
+    sortable: true,
+    truncateText: false,
+    width: '50%',
+  },
+  {
+    field: 'indexFieldType',
+    name: i18n.INDEX_MAPPING_TYPE,
+    render: (indexFieldType: string) => <EuiCode>{indexFieldType}</EuiCode>,
+    sortable: true,
+    truncateText: false,
+    width: '50%',
+  },
+];
+
+export const getEcsCompliantTableColumns = (): Array<
+  EuiTableFieldDataColumnType<EnrichedFieldMetadata>
+> => [
+  {
+    field: 'indexFieldName',
+    name: i18n.FIELD,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'type',
+    name: i18n.ECS_MAPPING_TYPE,
+    render: (type: string) =>
+      type != null ? <CodeSuccess>{type}</CodeSuccess> : <EuiCode>{EMPTY_PLACEHOLDER}</EuiCode>,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'allowed_values',
+    name: i18n.ECS_VALUES,
+    render: (allowedValues: AllowedValue[] | undefined) => (
+      <EcsAllowedValues allowedValues={allowedValues} />
+    ),
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'description',
+    name: i18n.ECS_DESCRIPTION,
+    render: (description: string) =>
+      description != null ? description : <EuiCode>{EMPTY_PLACEHOLDER}</EuiCode>,
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+];
+
+export const getIncompatibleMappingsTableColumns = (): Array<
+  EuiTableFieldDataColumnType<EnrichedFieldMetadata>
+> => [
+  {
+    field: 'indexFieldName',
+    name: i18n.FIELD,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'type',
+    name: i18n.ECS_MAPPING_TYPE_EXPECTED,
+    render: (type: string) => <CodeSuccess>{type != null ? type : EMPTY_PLACEHOLDER}</CodeSuccess>,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'indexFieldType',
+    name: i18n.INDEX_MAPPING_TYPE_ACTUAL,
+    render: (indexFieldType: string) => <CodeDanger>{indexFieldType}</CodeDanger>,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'description',
+    name: i18n.ECS_DESCRIPTION,
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+];
+
+export const getIncompatibleValuesTableColumns = (): Array<
+  EuiTableFieldDataColumnType<EnrichedFieldMetadata>
+> => [
+  {
+    field: 'indexFieldName',
+    name: i18n.FIELD,
+    sortable: true,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'allowed_values',
+    name: i18n.ECS_VALUES_EXPECTED,
+    render: (allowedValues: AllowedValue[] | undefined) => (
+      <EcsAllowedValues allowedValues={allowedValues} />
+    ),
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'indexInvalidValues',
+    name: i18n.DOCUMENT_VALUES_ACTUAL,
+    render: (indexInvalidValues: UnallowedValueCount[]) => (
+      <IndexInvalidValues indexInvalidValues={indexInvalidValues} />
+    ),
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+  {
+    field: 'description',
+    name: i18n.ECS_DESCRIPTION,
+    sortable: false,
+    truncateText: false,
+    width: '25%',
+  },
+];