Skip to content

Commit

Permalink
[Security Solutions][Detection Engine] Adds threat matching API and r…
Browse files Browse the repository at this point in the history 
…ule type (#77395)

## Summary

This is the backend, first iteration of threat matching API and rule type. You see elements using the backend API on the front end but cannot use the UI to add or edit a threshold rule with this PR.

Screen shots of it running in the UI elements that do work:
<img width="1862" alt="Screen Shot 2020-09-16 at 10 34 26 AM" src="https://user-images.githubusercontent.com/1151048/93366465-6e2b9c00-f808-11ea-923b-78e8d0fdfbaa.png">

<img width="1863" alt="Screen Shot 2020-09-16 at 10 34 48 AM" src="https://user-images.githubusercontent.com/1151048/93366476-71268c80-f808-11ea-8247-d2091ff1599a.png"> 

**Usage**
Since this is only backend API work and does not have the front end add/edit at the moment, you can use the existing UI's (for the most part) to validate the work here through CURL scripts below:

Go to the folder:
```ts
/kibana/x-pack/plugins/security_solution/server/lib/detection_engine/scripts
```

And post a small ECS threat mapping to the index called `mock-threat-list`:
```ts
./create_threat_mapping.sh
```

Then to post a small number of threats that represent simple port numbers you can run:
```ts
./create_threat_data.sh
```

However, feel free to also manually create them directly in your dev tools like so:

```ts
# Posts a threat list item called some-name with an IP but change these out for valid data in your system
PUT mock-threat-list-1/_doc/9999
{
  "@timestamp": "2020-09-09T20:30:45.725Z",
  "host": {
    "name": "some-name",
    "ip": "127.0.0.1"
  }
}
```

```ts
# Posts a destination port number to watch
PUT mock-threat-list-1/_doc/10000
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "destination": {
    "port": "443"
  }
}
```

```ts
# Posts a source port number to watch
PUT mock-threat-list-1/_doc/10001
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "source": {
    "port": "443"
  }
}
```

Then you can post a threat match rule:
```ts
./post_rule.sh ./rules/queries/query_with_threat_mapping.json
```
<details>
 <summary>Click here to see Response</summary>

```ts
{
  "actions": [],
  "author": [],
  "created_at": "2020-09-16T04:25:58.041Z",
  "created_by": "yo",
  "description": "Query with a threat mapping",
  "enabled": true,
  "exceptions_list": [],
  "false_positives": [],
  "from": "now-6m",
  "id": "f4226ab0-6f88-49c3-8f09-84cf5946ee7a",
  "immutable": false,
  "interval": "5m",
  "language": "kuery",
  "max_signals": 100,
  "name": "Query with a threat mapping",
  "output_index": ".siem-signals-hassanabad3-default",
  "query": "*:*",
  "references": [],
  "risk_score": 1,
  "risk_score_mapping": [],
  "rule_id": "threat-mapping",
  "severity": "high",
  "severity_mapping": [],
  "tags": [
    "tag_1",
    "tag_2"
  ],
  "threat": [],
  "threat_index": "mock-threat-list-1",
  "threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ],
  "threat_query": "*:*",
  "throttle": "no_actions",
  "to": "now",
  "type": "threat_match",
  "updated_at": "2020-09-16T04:25:58.051Z",
  "updated_by": "yo",
  "version": 1
}
```
</details>

**Structure**

You can see the rule structure in the file:
```ts
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping.json
```
<details>
 <summary>Click here to see JSON</summary>

```ts
{
  "name": "Query with a threat mapping",
  "description": "Query with a threat mapping",
  "rule_id": "threat-mapping",
  "risk_score": 1,
  "severity": "high",
  "type": "threat_match",
  "query": "*:*",
  "tags": ["tag_1", "tag_2"],
  "threat_index": "mock-threat-list",
  "threat_query": "*:*",
  "threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ]
}
```

</details>

Structural elements that are new:

New type enum called "threat_match"
```ts
"type": "threat_match",
```

New `threat_index` string which can be set to a single threat index (This might change to an array in the near future before release):
```ts
"threat_index": "mock-threat-list"
```

New `threat_query` string which can be set any valid query to filter the threat list before executing the rule. This can be undefined, if you are only pushing in filters from the API.

```ts
"threat_query": "*:*",
```

New `threat_filters` array which can be set to any valid filter like `filters`. This can be `undefined` if you are only using the query from the API.
```ts
threat_filter": []
```

New `threat_mapping` array which can be set to a valid mapping between the threat list and the ECS list. This structure has an inner array called `entries` which represent a 2 level tree of 1st level OR elements followed by 2nd level AND elements.

For example, if you want to find all threat matches where ECS documents will match against some ${threatList} index where it would be like so:

<details>
 <summary>Click here to see array from the boolean</summary>

```ts
"threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ]
```

</details>

What that array represents in pseudo boolean logic is: 

<details>
 <summary>Click here to see pseduo logic</summary>

```ts
(host.name: ${threatList.host.name} AND host.ip: ${threatList.host.name}) OR
(destination.ip: ${threatList.destination.ip} AND destination.port: ${threatList.destination.port}) OR
(source.port ${threatList.source.port}) OR
(source.ip ${threatList.source.ip})
```

</details>

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
  • Loading branch information
@FrankHassanabad
FrankHassanabad committed Sep 20, 2020
1 parent cd51289 commit 496152e
Show file tree
Hide file tree
Showing 56 changed files with 2,719 additions and 13 deletions.
1 change: 1 addition & 0 deletions x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ export const type = t.keyof({
query: null,
saved_query: null,
threshold: null,
threat_match: null,
});
export type Type = t.TypeOf<typeof type>;

Expand Down
101 changes: 101 additions & 0 deletions ...ity_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,104 @@ export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSc
exceptions_list: [],
rule_id: 'rule-1',
});

export const getAddPrepackagedThreatMatchRulesSchemaMock = (): AddPrepackagedRulesSchema => ({
description: 'some description',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
type: 'threat_match',
risk_score: 55,
language: 'kuery',
rule_id: 'rule-1',
version: 1,
threat_query: '*:*',
threat_index: 'list-index',
threat_mapping: [
{
entries: [
{
field: 'host.name',
value: 'host.name',
type: 'mapping',
},
],
},
],
threat_filters: [
{
bool: {
must: [
{
query_string: {
query: 'host.name: linux',
analyze_wildcard: true,
time_zone: 'Zulu',
},
},
],
filter: [],
should: [],
must_not: [],
},
},
],
});

export const getAddPrepackagedThreatMatchRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({
author: [],
description: 'some description',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
severity_mapping: [],
type: 'threat_match',
risk_score: 55,
risk_score_mapping: [],
language: 'kuery',
references: [],
actions: [],
enabled: false,
false_positives: [],
from: 'now-6m',
interval: '5m',
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
to: 'now',
threat: [],
throttle: null,
version: 1,
exceptions_list: [],
rule_id: 'rule-1',
threat_query: '*:*',
threat_index: 'list-index',
threat_mapping: [
{
entries: [
{
field: 'host.name',
value: 'host.name',
type: 'mapping',
},
],
},
],
threat_filters: [
{
bool: {
must: [
{
query_string: {
query: 'host.name: linux',
analyze_wildcard: true,
time_zone: 'Zulu',
},
},
],
filter: [],
should: [],
must_not: [],
},
},
],
});
10 changes: 10 additions & 0 deletions ...security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ import {
RiskScoreMapping,
SeverityMapping,
} from '../common/schemas';
import {
threat_index,
threat_query,
threat_filters,
threat_mapping,
} from '../types/threat_mapping';

import {
DefaultStringArray,
Expand Down Expand Up @@ -116,6 +122,10 @@ export const addPrepackagedRulesSchema = t.intersection([
references: DefaultStringArray, // defaults to empty array of strings if not set during decode
note, // defaults to "undefined" if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
threat_filters, // defaults to "undefined" if not set during decode
threat_mapping, // defaults to "undefined" if not set during decode
threat_query, // defaults to "undefined" if not set during decode
threat_index, // defaults to "undefined" if not set during decode
})
),
]);
Expand Down
14 changes: 14 additions & 0 deletions ...rity_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import { left } from 'fp-ts/lib/Either';
import {
getAddPrepackagedRulesSchemaMock,
getAddPrepackagedRulesSchemaDecodedMock,
getAddPrepackagedThreatMatchRulesSchemaMock,
getAddPrepackagedThreatMatchRulesSchemaDecodedMock,
} from './add_prepackaged_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
Expand Down Expand Up @@ -1597,4 +1599,16 @@ describe('add prepackaged rules schema', () => {
expect(message.schema).toEqual(expected);
});
});

describe('threat_mapping', () => {
test('You can set a threat query, index, mapping, filters on a pre-packaged rule', () => {
const payload = getAddPrepackagedThreatMatchRulesSchemaMock();
const decoded = addPrepackagedRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
const expected = getAddPrepackagedThreatMatchRulesSchemaDecodedMock();
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(expected);
});
});
});
100 changes: 100 additions & 0 deletions ...ins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,103 @@ export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => (
exceptions_list: [],
rule_id: 'rule-1',
});

export const getCreateThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): CreateRulesSchema => ({
description: 'Detecting root and admin users',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
type: 'threat_match',
risk_score: 55,
language: 'kuery',
rule_id: ruleId,
threat_query: '*:*',
threat_index: 'list-index',
threat_mapping: [
{
entries: [
{
field: 'host.name',
value: 'host.name',
type: 'mapping',
},
],
},
],
threat_filters: [
{
bool: {
must: [
{
query_string: {
query: 'host.name: linux',
analyze_wildcard: true,
time_zone: 'Zulu',
},
},
],
filter: [],
should: [],
must_not: [],
},
},
],
});

export const getCreateThreatMatchRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({
author: [],
severity_mapping: [],
risk_score_mapping: [],
description: 'Detecting root and admin users',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
severity: 'high',
type: 'threat_match',
risk_score: 55,
language: 'kuery',
references: [],
actions: [],
enabled: true,
false_positives: [],
from: 'now-6m',
interval: '5m',
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
to: 'now',
threat: [],
throttle: null,
version: 1,
exceptions_list: [],
rule_id: 'rule-1',
threat_query: '*:*',
threat_index: 'list-index',
threat_mapping: [
{
entries: [
{
field: 'host.name',
value: 'host.name',
type: 'mapping',
},
],
},
],
threat_filters: [
{
bool: {
must: [
{
query_string: {
query: 'host.name: linux',
analyze_wildcard: true,
time_zone: 'Zulu',
},
},
],
filter: [],
should: [],
must_not: [],
},
},
],
});
14 changes: 14 additions & 0 deletions ...ins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import { left } from 'fp-ts/lib/Either';
import {
getCreateRulesSchemaMock,
getCreateRulesSchemaDecodedMock,
getCreateThreatMatchRulesSchemaMock,
getCreateThreatMatchRulesSchemaDecodedMock,
} from './create_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
Expand Down Expand Up @@ -1661,4 +1663,16 @@ describe('create rules schema', () => {
expect(message.schema).toEqual(expected);
});
});

describe('threat_mapping', () => {
test('You can set a threat query, index, mapping, filters when creating a rule', () => {
const payload = getCreateThreatMatchRulesSchemaMock();
const decoded = createRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
const expected = getCreateThreatMatchRulesSchemaDecodedMock();
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(expected);
});
});
});
10 changes: 10 additions & 0 deletions .../plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ import {
RiskScoreMapping,
SeverityMapping,
} from '../common/schemas';
import {
threat_index,
threat_query,
threat_filters,
threat_mapping,
} from '../types/threat_mapping';

import {
DefaultStringArray,
Expand Down Expand Up @@ -112,6 +118,10 @@ export const createRulesSchema = t.intersection([
note, // defaults to "undefined" if not set during decode
version: DefaultVersionNumber, // defaults to 1 if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
threat_mapping, // defaults to "undefined" if not set during decode
threat_query, // defaults to "undefined" if not set during decode
threat_filters, // defaults to "undefined" if not set during decode
threat_index, // defaults to "undefined" if not set during decode
})
),
]);
Expand Down
40 changes: 39 additions & 1 deletion ...ity_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { getCreateRulesSchemaMock } from './create_rules_schema.mock';
import {
getCreateRulesSchemaMock,
getCreateThreatMatchRulesSchemaMock,
} from './create_rules_schema.mock';
import { CreateRulesSchema } from './create_rules_schema';
import { createRuleValidateTypeDependents } from './create_rules_type_dependents';

Expand Down Expand Up @@ -87,4 +90,39 @@ describe('create_rules_type_dependents', () => {
const errors = createRuleValidateTypeDependents(schema);
expect(errors).toEqual(['"threshold.value" has to be bigger than 0']);
});

test('threat_index, threat_query, and threat_mapping are required when type is "threat_match" and validates with it', () => {
const schema: CreateRulesSchema = {
...getCreateRulesSchemaMock(),
type: 'threat_match',
};
const errors = createRuleValidateTypeDependents(schema);
expect(errors).toEqual([
'when "type" is "threat_match", "threat_index" is required',
'when "type" is "threat_match", "threat_query" is required',
'when "type" is "threat_match", "threat_mapping" is required',
]);
});

test('validates with threat_index, threat_query, and threat_mapping when type is "threat_match"', () => {
const schema = getCreateThreatMatchRulesSchemaMock();
const { threat_filters: threatFilters, ...noThreatFilters } = schema;
const errors = createRuleValidateTypeDependents(noThreatFilters);
expect(errors).toEqual([]);
});

test('does NOT validate when threat_mapping is an empty array', () => {
const schema: CreateRulesSchema = {
...getCreateThreatMatchRulesSchemaMock(),
threat_mapping: [],
};
const errors = createRuleValidateTypeDependents(schema);
expect(errors).toEqual(['threat_mapping" must have at least one element']);
});

test('validates with threat_index, threat_query, threat_mapping, and an optional threat_filters, when type is "threat_match"', () => {
const schema = getCreateThreatMatchRulesSchemaMock();
const errors = createRuleValidateTypeDependents(schema);
expect(errors).toEqual([]);
});
});
Loading

0 comments on commit 496152e

Please sign in to comment.