From 2a7245bd9f736f3fcd29542232a6abd7b5a4f763 Mon Sep 17 00:00:00 2001 From: Nikita Indik Date: Thu, 9 May 2024 16:59:27 +0200 Subject: [PATCH 1/3] Update docs for `required_fields` --- .../api/rules/rules-api-bulk-actions.asciidoc | 6 +- .../api/rules/rules-api-create.asciidoc | 88 +++++++++++++++---- .../api/rules/rules-api-find.asciidoc | 8 +- .../api/rules/rules-api-get.asciidoc | 7 +- .../api/rules/rules-api-update.asciidoc | 14 ++- 5 files changed, 97 insertions(+), 26 deletions(-) diff --git a/docs/detections/api/rules/rules-api-bulk-actions.asciidoc b/docs/detections/api/rules/rules-api-bulk-actions.asciidoc index 2cce9aa8b2..1fc09ef8f6 100644 --- a/docs/detections/api/rules/rules-api-bulk-actions.asciidoc +++ b/docs/detections/api/rules/rules-api-bulk-actions.asciidoc @@ -584,7 +584,7 @@ A rule can only be `skipped` when the bulk action to be performed on it results ], "immutable":false, "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [], "setup": "", "type":"machine_learning", "anomaly_threshold":50, @@ -752,7 +752,7 @@ If processing of any rule fails, a partial error outputs the ID and/or name of t "exceptions_list": [], "immutable": false, "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [], "setup": "", "type": "query", "language": "kuery", @@ -797,7 +797,7 @@ If processing of any rule fails, a partial error outputs the ID and/or name of t } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, `required_fields`, and `execution_summary`. +<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations` and `execution_summary`. *Example 3, Dry run* diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 621d3fe102..78c00a0f2f 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -342,6 +342,14 @@ single execution. Defaults to `100`. |references |String[] |Array containing notes about or references to relevant information about the rule. Defaults to an empty array. +|required_fields |Object[] a| Elasticsearch fields and their types that need to be present for the rule to function. The object has these fields: + + * `name` (string, required): Field name + * `type` (string, required): Field type + * `ecs` (boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. + +*NOTE*: The value of `required_fields` does not affect the rule's behavior. It can be thought of as a way to document the fields that the rule expects to be present in the data. Specifying `required_fields` incorrectly won't cause the rule to fail. + |rule_id |String |Unique ID used to identify rules. For example, when a rule is converted from a third-party security solution. Automatically created when it is not provided. @@ -815,6 +823,9 @@ POST api/detection_engine/rules } } ], + "required_fields": [ + { name: "process.parent.name", "type": "keyword" } + ], "enabled": false } -------------------------------------------------- @@ -852,6 +863,9 @@ POST api/detection_engine/rules "interval": "2m", "name": "Windows server prml-19", "query": "host.name:prml-19 and event.category:authentication and event.outcome:failure", + "required_fields": [ + { "name": "source.ip", "type": "ip" } + ], "risk_score": 30, "rule_id": "liv-win-ser-logins", "severity": "low", @@ -960,7 +974,15 @@ POST api/detection_engine/rules ], "type": "eql", "language": "eql", - "query": "sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]" + "query": "sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]", + "required_fields": [ + { "name": "event.type", "type": "keyword" }, + { "name": "process.args", "type": "keyword" }, + { "name": "process.args_count", "type": "long" }, + { "name": "process.entity_id", "type": "keyword" }, + { "name": "process.name", "type": "keyword" }, + { "name": "process.pe.original_file_name", "type": "keyword" } + ] } -------------------------------------------------- // KIBANA @@ -1014,6 +1036,11 @@ POST api/detection_engine/rules ] } ], + "required_fields": [ + { "name": "destination.ip", "type": "ip" }, + { "name": "destination.port", "type": "long" }, + { "name": "host.ip", "type": "ip" } + ], "risk_score": 50, "severity": "medium", "name": "Bad IP threat match", @@ -1049,7 +1076,11 @@ POST api/detection_engine/rules "query": "*", "new_terms_fields": ["user.id", "source.ip"], "history_window_start": "now-30d", - "index": ["auditbeat*"] + "index": ["auditbeat*"], + "required_fields": [ + { "name": "user.id", "type": "keyword" }, + { "name": "source.ip", "type": "ip" } + ] } -------------------------------------------------- // KIBANA @@ -1073,7 +1104,10 @@ POST api/detection_engine/rules "to": "now", "enabled": false, "risk_score": 21, - "severity": "low" + "severity": "low", + "required_fields": [ + { "name": "process.parent.name", "type": "keyword" } + ] } -------------------------------------------------- @@ -1180,11 +1214,13 @@ Example response for a query rule: "query": "process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE", "language": "kuery", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "process.parent.name", "type": "keyword", "ecs": true } + ], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for a {ml} job rule: @@ -1238,11 +1274,11 @@ Example response for a {ml} job rule: "anomaly_threshold": 70, "machine_learning_job_id": "linux_anomalous_network_activity_ecs", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for a threshold rule: @@ -1319,11 +1355,13 @@ Example response for a threshold rule: "value": 20 }, "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "source.ip", "type": "ip", "ecs": true } + ], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for an EQL rule: @@ -1364,11 +1402,18 @@ Example response for an EQL rule: "query": "sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]", "language": "eql", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "event.type", "type": "keyword", "ecs": true }, + { "name": "process.args", "type": "keyword", "ecs": true }, + { "name": "process.args_count", "type": "long", "ecs": true }, + { "name": "process.entity_id", "type": "keyword", "ecs": true }, + { "name": "process.name", "type": "keyword", "ecs": true }, + { "name": "process.pe.original_file_name", "type": "keyword", "ecs": true } + ], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for an indicator match rule: @@ -1436,11 +1481,15 @@ Example response for an indicator match rule: } ], "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "destination.ip", "type": "ip", "ecs": true }, + { "name": "destination.port", "type": "long", "ecs": true }, + { "name": "host.ip", "type": "ip", "ecs": true } + ], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for a new terms rule: @@ -1481,11 +1530,14 @@ Example response for a new terms rule: "new_terms_fields": ["user.id", "source.ip"], "history_window_start": "now-30d", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "user.id", "type": "keyword", "ecs": true }, + { "name": "source.ip", "type": "ip", "ecs": true } + ], "setup": "" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. Example response for an {esql} rule: @@ -1521,11 +1573,13 @@ Example response for an {esql} rule: "rule_id": "e4b53a89-debd-4a0d-a3e3-20606952e589", "immutable": false, "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "process.parent.name", "type": "keyword", "ecs": true } + ], "setup": "", "type": "esql", "language": "esql", "query": "from auditbeat-8.10.2 [metadata _id] | where process.parent.name == \"EXCEL.EXE\"" } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. +<1> dev:[] These fields are under development and their usage may change: `related_integrations`. diff --git a/docs/detections/api/rules/rules-api-find.asciidoc b/docs/detections/api/rules/rules-api-find.asciidoc index 4914b90869..8997a58f1a 100644 --- a/docs/detections/api/rules/rules-api-find.asciidoc +++ b/docs/detections/api/rules/rules-api-find.asciidoc @@ -97,7 +97,11 @@ Example response: ], "to": "now", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "event.action", "type": "keyword", "ecs": true }, + { "name": "process.name", "type": "keyword", "ecs": true }, + { "name": "process.parent.name", "type": "keyword", "ecs": true } + ], "setup": "", "type": "query", "threat": [ @@ -138,4 +142,4 @@ Example response: -------------------------------------------------- -<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, `required_fields`, and `execution_summary`. +<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations` and `execution_summary`. diff --git a/docs/detections/api/rules/rules-api-get.asciidoc b/docs/detections/api/rules/rules-api-get.asciidoc index 871feecf21..7c85435d6e 100644 --- a/docs/detections/api/rules/rules-api-get.asciidoc +++ b/docs/detections/api/rules/rules-api-get.asciidoc @@ -62,7 +62,10 @@ Example response: "interval": "1h", "rule_id": "process_started_by_ms_office_user_folder", "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "process.name", "type": "keyword", "ecs": true }, + { "name": "process.parent.name", "type": "keyword", "ecs": true } + ], "setup": "", "language": "kuery", "max_signals": 100, @@ -113,4 +116,4 @@ Example response: -------------------------------------------------- -<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, `required_fields`, and `execution_summary`. +<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, and `execution_summary`. diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index 06d267226b..e4275c526a 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -244,6 +244,14 @@ single execution. Defaults to `100`. |references |String[] |Array containing notes about or references to relevant information about the rule. Defaults to an empty array. +|required_fields |Object[] a| Elasticsearch fields and their types that need to be present for the rule to function. The object has these fields: + + * `name` (string, required): Field name + * `type` (string, required): Field type + * `ecs` (boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. + +*NOTE*: The value of `required_fields` does not affect the rule's behavior. It can be thought of as a way to document the fields that the rule expects to be present in the data. Specifying `required_fields` incorrectly won't cause the rule to fail. + |setup |String |Populates the rule's setup guide with instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly. @@ -632,7 +640,9 @@ Example response: "ms office" ], "related_integrations": [], <1> - "required_fields": [], <1> + "required_fields": [ + { "name": "process.parent.name", "type": "keyword", "ecs": true } + ], "setup": "", "type": "query", "threat": [ @@ -669,4 +679,4 @@ Example response: } -------------------------------------------------- -<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, `required_fields`, and `execution_summary`. +<1> dev:[] These fields are under development and their usage or schema may change: `related_integrations`, and `execution_summary`. From da6bc4947a7a35effe53c61ba1408dc3d9033f05 Mon Sep 17 00:00:00 2001 From: Nikita Indik Date: Fri, 10 May 2024 00:44:18 +0200 Subject: [PATCH 2/3] Update docs/detections/api/rules/rules-api-create.asciidoc Co-authored-by: Joe Peeples --- docs/detections/api/rules/rules-api-create.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 78c00a0f2f..2a0fca55be 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -342,13 +342,13 @@ single execution. Defaults to `100`. |references |String[] |Array containing notes about or references to relevant information about the rule. Defaults to an empty array. -|required_fields |Object[] a| Elasticsearch fields and their types that need to be present for the rule to function. The object has these fields: +|required_fields |Object[] a| {es} fields and their types that need to be present for the rule to function. The object has these fields: - * `name` (string, required): Field name - * `type` (string, required): Field type - * `ecs` (boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. + * `name` (string, required): The field's name. + * `type` (string, required): The field's data type. + * `ecs` (Boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. -*NOTE*: The value of `required_fields` does not affect the rule's behavior. It can be thought of as a way to document the fields that the rule expects to be present in the data. Specifying `required_fields` incorrectly won't cause the rule to fail. +*NOTE*: The value of `required_fields` does not affect the rule's behavior, and specifying it incorrectly won't cause the rule to fail. Use `required_fields` as an informational property to document the fields that the rule expects to be present in the data. |rule_id |String |Unique ID used to identify rules. For example, when a rule is converted from a third-party security solution. Automatically created when From 41c74785e5fb89e56b3950fa631557794f284b7d Mon Sep 17 00:00:00 2001 From: Nikita Indik Date: Fri, 10 May 2024 00:44:27 +0200 Subject: [PATCH 3/3] Update docs/detections/api/rules/rules-api-update.asciidoc Co-authored-by: Joe Peeples --- docs/detections/api/rules/rules-api-update.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index e4275c526a..6171e426e2 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -244,13 +244,13 @@ single execution. Defaults to `100`. |references |String[] |Array containing notes about or references to relevant information about the rule. Defaults to an empty array. -|required_fields |Object[] a| Elasticsearch fields and their types that need to be present for the rule to function. The object has these fields: +|required_fields |Object[] a| {es} fields and their types that need to be present for the rule to function. The object has these fields: - * `name` (string, required): Field name - * `type` (string, required): Field type - * `ecs` (boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. + * `name` (string, required): The field's name. + * `type` (string, required): The field's data type. + * `ecs` (Boolean, optional): Indicates whether the field is {ecs-ref}[ECS-compliant]. This property is only present in responses. Its value is computed based on field's name and type. -*NOTE*: The value of `required_fields` does not affect the rule's behavior. It can be thought of as a way to document the fields that the rule expects to be present in the data. Specifying `required_fields` incorrectly won't cause the rule to fail. +*NOTE*: The value of `required_fields` does not affect the rule's behavior, and specifying it incorrectly won't cause the rule to fail. Use `required_fields` as an informational property to document the fields that the rule expects to be present in the data. |setup |String |Populates the rule's setup guide with instructions on rule prerequisites such as required integrations, configuration steps, and anything