From 46af5fcc2d996e003a9d078361e498b442e67c72 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Fri, 2 Oct 2020 10:10:38 -0400 Subject: [PATCH] [Security Solution][Detections] Enrich shell signals with fields common to all building blocks (#79130) * Enrich shell signals with fields common to all building blocks * PR comments + additional unit test --- .../signals/build_bulk_body.test.ts | 448 +++++++++++++++++- .../signals/build_bulk_body.ts | 53 +++ 2 files changed, 498 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 2f7dd22c0c78e7..75a7de8cd2c443 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -11,8 +11,15 @@ import { sampleIdGuid, sampleDocWithAncestors, sampleRuleSO, + sampleDocNoSortIdNoVersion, } from './__mocks__/es_results'; -import { buildBulkBody, buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; +import { + buildBulkBody, + buildSignalFromSequence, + buildSignalFromEvent, + objectPairIntersection, + objectArrayIntersection, +} from './build_bulk_body'; import { SignalHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -438,13 +445,20 @@ describe('buildBulkBody', () => { describe('buildSignalFromSequence', () => { test('builds a basic signal from a sequence of building blocks', () => { - const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]]; + const block1 = sampleDocWithAncestors().hits.hits[0]; + block1._source.new_key = 'new_key_value'; + block1._source.new_key2 = 'new_key2_value'; + const block2 = sampleDocWithAncestors().hits.hits[0]; + block2._source.new_key = 'new_key_value'; + const blocks = [block1, block2]; const ruleSO = sampleRuleSO(); const signal = buildSignalFromSequence(blocks, ruleSO); // Timestamp will potentially always be different so remove it for the test // @ts-expect-error delete signal['@timestamp']; - const expected: Omit = { + const expected: Omit & { someKey: string; new_key: string } = { + someKey: 'someValue', + new_key: 'new_key_value', event: { kind: 'signal', }, @@ -539,6 +553,96 @@ describe('buildSignalFromSequence', () => { }; expect(signal).toEqual(expected); }); + + test('builds a basic signal if there is no overlap between source events', () => { + const block1 = sampleDocNoSortIdNoVersion(); + const block2 = sampleDocNoSortIdNoVersion(); + block2._source['@timestamp'] = '2021-05-20T22:28:46+0000'; + block2._source.someKey = 'someOtherValue'; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromSequence([block1, block2], ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit = { + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + group: { + id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea', + }, + }, + }; + expect(signal).toEqual(expected); + }); }); describe('buildSignalFromEvent', () => { @@ -632,3 +736,341 @@ describe('buildSignalFromEvent', () => { expect(signal).toEqual(expected); }); }); + +describe('recursive intersection between objects', () => { + test('should treat numbers and strings as unequal', () => { + const a = { + field1: 1, + field2: 1, + }; + const b = { + field1: 1, + field2: '1', + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + }; + expect(intersection).toEqual(expected); + }); + + test('should strip unequal numbers and strings', () => { + const a = { + field1: 1, + field2: 1, + field3: 'abcd', + field4: 'abcd', + }; + const b = { + field1: 1, + field2: 100, + field3: 'abcd', + field4: 'wxyz', + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + field3: 'abcd', + }; + expect(intersection).toEqual(expected); + }); + + test('should handle null values', () => { + const a = { + field1: 1, + field2: '1', + field3: null, + }; + const b = { + field1: null, + field2: null, + field3: null, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field3: null, + }; + expect(intersection).toEqual(expected); + }); + + test('should handle explicit undefined values and return undefined if left with only undefined fields', () => { + const a = { + field1: 1, + field2: '1', + field3: undefined, + }; + const b = { + field1: undefined, + field2: undefined, + field3: undefined, + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should strip arrays out regardless of whether they are equal', () => { + const a = { + array_field1: [1, 2], + array_field2: [1, 2], + }; + const b = { + array_field1: [1, 2], + array_field2: [3, 4], + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should strip fields that are not in both objects', () => { + const a = { + field1: 1, + }; + const b = { + field2: 1, + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should work on objects within objects', () => { + const a = { + container_field: { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + nested_container_field: { + field1: 1, + field2: 1, + }, + nested_container_field2: { + field1: undefined, + }, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + container_field: { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + nested_container_field: { + field1: 1, + field2: 2, + }, + nested_container_field2: { + field1: undefined, + }, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + container_field: { + field1: 1, + field6: null, + nested_container_field: { + field1: 1, + }, + }, + }; + expect(intersection).toEqual(expected); + }); + + test('should work on objects with a variety of fields', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + field6: null, + container_field: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); +}); + +describe('objectArrayIntersection', () => { + test('should return undefined if the array is empty', () => { + const intersection = objectArrayIntersection([]); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + test('should return the initial object if there is only 1', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const intersection = objectArrayIntersection([a]); + const expected = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); + test('should work with exactly 2 objects', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectArrayIntersection([a, b]); + const expected = { + field1: 1, + field6: null, + container_field: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); + + test('should work with 3 or more objects', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const c = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + array_field: [1, 2], + container_field: { + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectArrayIntersection([a, b, c]); + const expected = { + field1: 1, + }; + expect(intersection).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index f8632a85c77e9e..8e9571fe8a445d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -129,7 +129,9 @@ export const buildSignalFromSequence = ( ): SignalHit => { const rule = buildRuleWithoutOverrides(ruleSO); const signal: Signal = buildSignal(events, rule); + const mergedEvents = objectArrayIntersection(events.map((event) => event._source)); return { + ...mergedEvents, '@timestamp': new Date().toISOString(), event: { kind: 'signal', @@ -167,3 +169,54 @@ export const buildSignalFromEvent = ( }; return signalHit; }; + +export const objectArrayIntersection = (objects: object[]) => { + if (objects.length === 0) { + return undefined; + } else if (objects.length === 1) { + return objects[0]; + } else { + return objects + .slice(1) + .reduce( + (acc: object | undefined, obj): object | undefined => objectPairIntersection(acc, obj), + objects[0] + ); + } +}; + +export const objectPairIntersection = (a: object | undefined, b: object | undefined) => { + if (a === undefined || b === undefined) { + return undefined; + } + const intersection: Record = {}; + Object.entries(a).forEach(([key, aVal]) => { + if (key in b) { + const bVal = (b as Record)[key]; + if ( + typeof aVal === 'object' && + !(aVal instanceof Array) && + aVal !== null && + typeof bVal === 'object' && + !(bVal instanceof Array) && + bVal !== null + ) { + intersection[key] = objectPairIntersection(aVal, bVal); + } else if (aVal === bVal) { + intersection[key] = aVal; + } + } + }); + // Count up the number of entries that are NOT undefined in the intersection + // If there are no keys OR all entries are undefined, return undefined + if ( + Object.values(intersection).reduce( + (acc: number, value) => (value !== undefined ? acc + 1 : acc), + 0 + ) === 0 + ) { + return undefined; + } else { + return intersection; + } +};