From 364ed7c4eaf25121fc1e0f1c127a7abc265a7b25 Mon Sep 17 00:00:00 2001 From: Brad Solomon <81818815+brsolomon-deloitte@users.noreply.github.com> Date: Thu, 5 Jan 2023 18:53:48 -0500 Subject: [PATCH 1/4] [filebeat][threatintel] Ignore bad indicator IPs for MISP fileset Closes #29949. MISP may send an Event.Attribute.value IP as a CIDR such as 146.88.240.0/24, which is not a valid IP per the Elasticsearch IP data type. --- x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index c97eb172ca5..80f7af0f5fe 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -249,6 +249,9 @@ processors: - "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}" ignore_missing: true if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" + # MISP may send a CIDR such as 1.2.3.0/22, which is not a valid Elasticsearch IP data type + # https://github.com/elastic/beats/issues/29949 + ignore_failure: true ## Email indicator operations # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp. From cf5133b596449e64f507f863dea5f727972f43f6 Mon Sep 17 00:00:00 2001 From: Brad Solomon <81818815+brsolomon-deloitte@users.noreply.github.com> Date: Thu, 5 Jan 2023 19:01:49 -0500 Subject: [PATCH 2/4] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3576b750e91..b12fc222744 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,6 +83,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Filebeat* +- Allow the `misp` fileset in the Filebeat `threatintel` module to ignore CIDR ranges for an IP field. {issue}29949[29949] {pull}34195[34195] *Auditbeat* From 43a17812f44d8a47ec3771b4cfbcb05cc29bda4d Mon Sep 17 00:00:00 2001 From: Brad Solomon <81818815+brsolomon-deloitte@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:41:16 -0500 Subject: [PATCH 3/4] remove redundant link --- x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index 80f7af0f5fe..3cdc64d8703 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -250,7 +250,6 @@ processors: ignore_missing: true if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" # MISP may send a CIDR such as 1.2.3.0/22, which is not a valid Elasticsearch IP data type - # https://github.com/elastic/beats/issues/29949 ignore_failure: true ## Email indicator operations From 3ac7a25761eb995282eb184f9964e12701fda2b2 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 23 Jan 2023 08:49:07 +1030 Subject: [PATCH 4/4] add tests case --- .../misp/test/misp_sample_cidr.ndjson.log | 1 + .../misp_sample_cidr.ndjson.log-expected.json | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log create mode 100644 x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log new file mode 100644 index 00000000000..7c5975d33c1 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log @@ -0,0 +1 @@ +{"Event":{"id":"10","orgc_id":"4","org_id":"1","date":"2020-12-09","threat_level_id":"3","info":"Recent Qakbot (Qbot) activity","published":true,"uuid":"5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16","attribute_count":"15","analysis":"2","timestamp":"1607868196","distribution":"3","proposal_email_lock":false,"locked":false,"publish_timestamp":"1610637888","sharing_group_id":"0","disable_correlation":false,"extends_uuid":"","Org":{"id":"1","name":"ORGNAME","uuid":"5877549f-ea76-4b91-91fb-c72ad682b4a5","local":true},"Orgc":{"id":"4","name":"CUDESO","uuid":"56c42374-fdb8-4544-a218-41ffc0a8ab16","local":false},"Attribute":{"id":"10686","type":"ip-dst|port","category":"Network activity","to_ids":true,"uuid":"5fd0c620-a844-4ace-9710-a37bc0a8ab16","event_id":"10","distribution":"5","timestamp":"1607517728","comment":"On port 2222","sharing_group_id":"0","deleted":false,"disable_correlation":false,"object_id":"0","object_relation":null,"first_seen":null,"last_seen":null,"value":"146.88.240.0/24","Galaxy":[],"ShadowAttribute":[]},"ShadowAttribute":[],"RelatedEvent":[],"Galaxy":[],"Object":[],"EventReport":[],"Tag":[{"id":"3","name":"tlp:white","colour":"#ffffff","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":false,"is_custom_galaxy":false,"local":0},{"id":"6","name":"misp-galaxy:banker=\"Qakbot\"","colour":"#0088cc","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":true,"is_custom_galaxy":false,"local":0}]}} diff --git a/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json new file mode 100644 index 00000000000..0770d907530 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json @@ -0,0 +1,60 @@ +[ + { + "@timestamp": "2020-12-13T14:03:16.000Z", + "event.category": "threat", + "event.dataset": "threatintel.misp", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.timezone": "-02:00", + "event.type": "indicator", + "fileset.name": "misp", + "input.type": "log", + "log.offset": 0, + "misp.attribute.category": "Network activity", + "misp.attribute.comment": "On port 2222", + "misp.attribute.deleted": false, + "misp.attribute.disable_correlation": false, + "misp.attribute.distribution": 5, + "misp.attribute.event_id": "10", + "misp.attribute.id": "10686", + "misp.attribute.object_id": "0", + "misp.attribute.sharing_group_id": "0", + "misp.attribute.timestamp": "1607517728", + "misp.attribute.to_ids": true, + "misp.attribute.type": "ip-dst|port", + "misp.attribute.uuid": "5fd0c620-a844-4ace-9710-a37bc0a8ab16", + "misp.attribute_count": 15, + "misp.date": "2020-12-09", + "misp.disable_correlation": false, + "misp.distribution": "3", + "misp.extends_uuid": "", + "misp.id": "10", + "misp.info": "Recent Qakbot (Qbot) activity", + "misp.locked": false, + "misp.org_id": "1", + "misp.orgc.id": "4", + "misp.orgc.local": false, + "misp.orgc.name": "CUDESO", + "misp.orgc.uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16", + "misp.orgc_id": "4", + "misp.proposal_email_lock": false, + "misp.publish_timestamp": "1610637888", + "misp.published": true, + "misp.sharing_group_id": "0", + "misp.threat_level_id": 3, + "misp.uuid": "5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16", + "service.type": "threatintel", + "tags": [ + "misp-galaxy:banker=Qakbot", + "tlp:white" + ], + "threat.feed.dashboard_id": "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f", + "threat.feed.name": "[Filebeat] MISP", + "threat.indicator.marking.tlp": [ + "white" + ], + "threat.indicator.provider": "misp", + "threat.indicator.scanner_stats": 2, + "threat.indicator.type": "ipv4-addr" + } +] \ No newline at end of file