[8.7] [Security Solution] Enrich Value List Telemetry (#149621) (#151076

) # Backport This will backport the following commits from `main` to `8.7`: - [[Security Solution] Enrich Value List Telemetry (#149621)](#149621)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: JD Kurma <JDKurma@gmail.com>
elastic · Feb 13, 2023 · f2ea20f · f2ea20f
1 parent 489abdc
commit f2ea20f
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 45 deletions.
diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts
@@ -15,6 +15,7 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/packag
 import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline';
 import { stubEndpointMetricsResponse } from './metrics';
 import { prebuiltRuleAlertsResponse } from './prebuilt_rule_alerts';
+import type { ESClusterInfo, ESLicense } from '../types';
 
 export const createMockTelemetryEventsSender = (
   enableTelemetry?: boolean,
@@ -37,33 +38,31 @@ export const createMockTelemetryEventsSender = (
   } as unknown as jest.Mocked<TelemetryEventsSender>;
 };
 
-const stubClusterInfo = {
-  name: 'Stub-MacBook-Pro.local',
+export const stubClusterInfo: ESClusterInfo = {
   cluster_name: 'elasticsearch',
   cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
   version: {
     number: '8.0.0-SNAPSHOT',
     build_type: 'tar',
     build_hash: '38537ab4a726b42ce8f034aad78d8fca4d4f3e51',
     build_date: moment().toISOString(),
+    build_flavor: 'DEFAULT',
     build_snapshot: true,
     lucene_version: '9.2.0',
     minimum_wire_compatibility_version: '7.17.0',
     minimum_index_compatibility_version: '7.0.0',
   },
-  tagline: 'You Know, for Search',
 };
 
-const stubLicenseInfo = {
+export const stubLicenseInfo: ESLicense = {
   status: 'active',
   uid: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
   type: 'trial',
   issue_date: moment().toISOString(),
   issue_date_in_millis: 1653299879146,
   expiry_date: moment().toISOString(),
-  expiry_date_in_millis: 1655891879146,
+  expirty_date_in_millis: 1655891879146,
   max_nodes: 1000,
-  max_resource_units: null,
   issued_to: 'elasticsearch',
   issuer: 'elasticsearch',
   start_date_in_millis: -1,

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts
@@ -6,7 +6,7 @@
  */
 
 import moment from 'moment';
-import { createMockPackagePolicy } from './__mocks__';
+import { createMockPackagePolicy, stubClusterInfo, stubLicenseInfo } from './__mocks__';
 import {
   LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
@@ -21,7 +21,7 @@ import {
   isPackagePolicyList,
   templateExceptionList,
   addDefaultAdvancedPolicyConfigSettings,
-  metricsResponseToValueListMetaData,
+  formatValueListMetaData,
   tlog,
   setIsElasticCloudDeployment,
   createTaskMetric,
@@ -805,10 +805,11 @@ describe('test advanced policy config overlap ', () => {
 
 describe('test metrics response to value list meta data', () => {
   test('can succeed when metrics response is fully populated', async () => {
+    jest.useFakeTimers().setSystemTime(new Date('2023-01-30'));
     const stubMetricResponses = {
       listMetricsResponse: {
         aggregations: {
-          total_value_list_count: 5,
+          total_value_list_count: { value: 5 },
           type_breakdown: {
             buckets: [
               {
@@ -858,8 +859,12 @@ describe('test metrics response to value list meta data', () => {
         },
       },
     };
-    const response = metricsResponseToValueListMetaData(stubMetricResponses);
+    const response = formatValueListMetaData(stubMetricResponses, stubClusterInfo, stubLicenseInfo);
     expect(response).toEqual({
+      '@timestamp': '2023-01-30T00:00:00.000Z',
+      cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
+      cluster_name: 'elasticsearch',
+      license_id: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
       total_list_count: 5,
       types: [
         {
@@ -901,8 +906,12 @@ describe('test metrics response to value list meta data', () => {
       indicatorMatchMetricsResponse: {},
     };
     // @ts-ignore
-    const response = metricsResponseToValueListMetaData(stubMetricResponses);
+    const response = formatValueListMetaData(stubMetricResponses, stubClusterInfo, stubLicenseInfo);
     expect(response).toEqual({
+      '@timestamp': '2023-01-30T00:00:00.000Z',
+      cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
+      cluster_name: 'elasticsearch',
+      license_id: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
       total_list_count: 0,
       types: [],
       lists: [],

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts
@@ -18,10 +18,7 @@ import type {
   ESLicense,
   ListTemplate,
   TelemetryEvent,
-  ValueListResponseAggregation,
-  ValueListExceptionListResponseAggregation,
-  ValueListItemsResponseAggregation,
-  ValueListIndicatorMatchResponseAggregation,
+  ValueListResponse,
   TaskMetric,
 } from './types';
 import {
@@ -241,32 +238,37 @@ export const addDefaultAdvancedPolicyConfigSettings = (policyConfig: PolicyConfi
   return merge(DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS, policyConfig);
 };
 
-export const metricsResponseToValueListMetaData = ({
-  listMetricsResponse,
-  itemMetricsResponse,
-  exceptionListMetricsResponse,
-  indicatorMatchMetricsResponse,
-}: {
-  listMetricsResponse: ValueListResponseAggregation;
-  itemMetricsResponse: ValueListItemsResponseAggregation;
-  exceptionListMetricsResponse: ValueListExceptionListResponseAggregation;
-  indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation;
-}) => ({
-  total_list_count: listMetricsResponse?.aggregations?.total_value_list_count ?? 0,
+export const formatValueListMetaData = (
+  valueListResponse: ValueListResponse,
+  clusterInfo: ESClusterInfo,
+  licenseInfo: ESLicense | undefined
+) => ({
+  '@timestamp': moment().toISOString(),
+  cluster_uuid: clusterInfo.cluster_uuid,
+  cluster_name: clusterInfo.cluster_name,
+  license_id: licenseInfo?.uid,
+  total_list_count:
+    valueListResponse.listMetricsResponse?.aggregations?.total_value_list_count?.value ?? 0,
   types:
-    listMetricsResponse?.aggregations?.type_breakdown?.buckets.map((breakdown) => ({
-      type: breakdown.key,
-      count: breakdown.doc_count,
-    })) ?? [],
+    valueListResponse.listMetricsResponse?.aggregations?.type_breakdown?.buckets.map(
+      (breakdown) => ({
+        type: breakdown.key,
+        count: breakdown.doc_count,
+      })
+    ) ?? [],
   lists:
-    itemMetricsResponse?.aggregations?.value_list_item_count?.buckets.map((itemCount) => ({
-      id: itemCount.key,
-      count: itemCount.doc_count,
-    })) ?? [],
+    valueListResponse.itemMetricsResponse?.aggregations?.value_list_item_count?.buckets.map(
+      (itemCount) => ({
+        id: itemCount.key,
+        count: itemCount.doc_count,
+      })
+    ) ?? [],
   included_in_exception_lists_count:
-    exceptionListMetricsResponse?.aggregations?.vl_included_in_exception_lists_count?.value ?? 0,
+    valueListResponse.exceptionListMetricsResponse?.aggregations
+      ?.vl_included_in_exception_lists_count?.value ?? 0,
   used_in_indicator_match_rule_count:
-    indicatorMatchMetricsResponse?.aggregations?.vl_used_in_indicator_match_rule_count?.value ?? 0,
+    valueListResponse.indicatorMatchMetricsResponse?.aggregations
+      ?.vl_used_in_indicator_match_rule_count?.value ?? 0,
 });
 
 export let isElasticCloudDeployment = false;

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts
@@ -41,7 +41,6 @@ import {
   exceptionListItemToTelemetryEntry,
   trustedApplicationToTelemetryEntry,
   ruleExceptionListItemToTelemetryEvent,
-  metricsResponseToValueListMetaData,
   tlog,
 } from './helpers';
 import { Fetcher } from '../../endpoint/routes/resolver/tree/utils/fetch';
@@ -55,7 +54,7 @@ import type {
   GetEndpointListResponse,
   RuleSearchResult,
   ExceptionListItem,
-  ValueListMetaData,
+  ValueListResponse,
   ValueListResponseAggregation,
   ValueListItemsResponseAggregation,
   ValueListExceptionListResponseAggregation,
@@ -172,7 +171,7 @@ export interface ITelemetryReceiver {
     nodeIds: string[]
   ): Promise<SearchResponse<SafeEndpointEvent, Record<string, AggregationsAggregate>>>;
 
-  fetchValueListMetaData(interval: number): Promise<ValueListMetaData>;
+  fetchValueListMetaData(interval: number): Promise<ValueListResponse>;
 }
 
 export class TelemetryReceiver implements ITelemetryReceiver {
@@ -924,12 +923,12 @@ export class TelemetryReceiver implements ITelemetryReceiver {
       exceptionListMetrics as unknown as ValueListExceptionListResponseAggregation;
     const indicatorMatchMetricsResponse =
       indicatorMatchMetrics as unknown as ValueListIndicatorMatchResponseAggregation;
-    return metricsResponseToValueListMetaData({
+    return {
       listMetricsResponse,
       itemMetricsResponse,
       exceptionListMetricsResponse,
       indicatorMatchMetricsResponse,
-    });
+    };
   }
 
   public async fetchClusterInfo(): Promise<ESClusterInfo> {

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts
@@ -18,7 +18,13 @@ import {
   TASK_METRICS_CHANNEL,
 } from '../constants';
 import type { ESClusterInfo, ESLicense } from '../types';
-import { batchTelemetryRecords, templateExceptionList, tlog, createTaskMetric } from '../helpers';
+import {
+  batchTelemetryRecords,
+  templateExceptionList,
+  tlog,
+  createTaskMetric,
+  formatValueListMetaData,
+} from '../helpers';
 import type { ITelemetryEventsSender } from '../sender';
 import type { ITelemetryReceiver } from '../receiver';
 import type { TaskExecutionPeriod } from '../task';
@@ -114,9 +120,14 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
         }
 
         // Value list meta data
-        const valueListMetaData = await receiver.fetchValueListMetaData(
+        const valueListResponse = await receiver.fetchValueListMetaData(
           FETCH_VALUE_LIST_META_DATA_INTERVAL_IN_HOURS
         );
+        const valueListMetaData = formatValueListMetaData(
+          valueListResponse,
+          clusterInfo,
+          licenseInfo
+        );
         tlog(logger, `Value List Meta Data: ${JSON.stringify(valueListMetaData)}`);
         if (valueListMetaData?.total_list_count) {
           await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, [valueListMetaData]);

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts
@@ -382,7 +382,7 @@ export interface ValueListMetaData {
 
 export interface ValueListResponseAggregation {
   aggregations: {
-    total_value_list_count: number;
+    total_value_list_count: { value: number };
     type_breakdown: {
       buckets: Array<{
         key: string;
@@ -437,3 +437,10 @@ export interface TelemetryFilterListArtifact {
   exception_lists: AllowlistFields;
   prebuilt_rules_alerts: AllowlistFields;
 }
+
+export interface ValueListResponse {
+  listMetricsResponse: ValueListResponseAggregation;
+  itemMetricsResponse: ValueListItemsResponseAggregation;
+  exceptionListMetricsResponse: ValueListExceptionListResponseAggregation;
+  indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation;
+}