diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3576b750e91..b12fc222744 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,6 +83,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Filebeat* +- Allow the `misp` fileset in the Filebeat `threatintel` module to ignore CIDR ranges for an IP field. {issue}29949[29949] {pull}34195[34195] *Auditbeat* diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index c97eb172ca5..3cdc64d8703 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -249,6 +249,8 @@ processors: - "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}" ignore_missing: true if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" + # MISP may send a CIDR such as 1.2.3.0/22, which is not a valid Elasticsearch IP data type + ignore_failure: true ## Email indicator operations # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp. diff --git a/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log new file mode 100644 index 00000000000..7c5975d33c1 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log @@ -0,0 +1 @@ +{"Event":{"id":"10","orgc_id":"4","org_id":"1","date":"2020-12-09","threat_level_id":"3","info":"Recent Qakbot (Qbot) activity","published":true,"uuid":"5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16","attribute_count":"15","analysis":"2","timestamp":"1607868196","distribution":"3","proposal_email_lock":false,"locked":false,"publish_timestamp":"1610637888","sharing_group_id":"0","disable_correlation":false,"extends_uuid":"","Org":{"id":"1","name":"ORGNAME","uuid":"5877549f-ea76-4b91-91fb-c72ad682b4a5","local":true},"Orgc":{"id":"4","name":"CUDESO","uuid":"56c42374-fdb8-4544-a218-41ffc0a8ab16","local":false},"Attribute":{"id":"10686","type":"ip-dst|port","category":"Network activity","to_ids":true,"uuid":"5fd0c620-a844-4ace-9710-a37bc0a8ab16","event_id":"10","distribution":"5","timestamp":"1607517728","comment":"On port 2222","sharing_group_id":"0","deleted":false,"disable_correlation":false,"object_id":"0","object_relation":null,"first_seen":null,"last_seen":null,"value":"146.88.240.0/24","Galaxy":[],"ShadowAttribute":[]},"ShadowAttribute":[],"RelatedEvent":[],"Galaxy":[],"Object":[],"EventReport":[],"Tag":[{"id":"3","name":"tlp:white","colour":"#ffffff","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":false,"is_custom_galaxy":false,"local":0},{"id":"6","name":"misp-galaxy:banker=\"Qakbot\"","colour":"#0088cc","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":true,"is_custom_galaxy":false,"local":0}]}} diff --git a/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json new file mode 100644 index 00000000000..0770d907530 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json @@ -0,0 +1,60 @@ +[ + { + "@timestamp": "2020-12-13T14:03:16.000Z", + "event.category": "threat", + "event.dataset": "threatintel.misp", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.timezone": "-02:00", + "event.type": "indicator", + "fileset.name": "misp", + "input.type": "log", + "log.offset": 0, + "misp.attribute.category": "Network activity", + "misp.attribute.comment": "On port 2222", + "misp.attribute.deleted": false, + "misp.attribute.disable_correlation": false, + "misp.attribute.distribution": 5, + "misp.attribute.event_id": "10", + "misp.attribute.id": "10686", + "misp.attribute.object_id": "0", + "misp.attribute.sharing_group_id": "0", + "misp.attribute.timestamp": "1607517728", + "misp.attribute.to_ids": true, + "misp.attribute.type": "ip-dst|port", + "misp.attribute.uuid": "5fd0c620-a844-4ace-9710-a37bc0a8ab16", + "misp.attribute_count": 15, + "misp.date": "2020-12-09", + "misp.disable_correlation": false, + "misp.distribution": "3", + "misp.extends_uuid": "", + "misp.id": "10", + "misp.info": "Recent Qakbot (Qbot) activity", + "misp.locked": false, + "misp.org_id": "1", + "misp.orgc.id": "4", + "misp.orgc.local": false, + "misp.orgc.name": "CUDESO", + "misp.orgc.uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16", + "misp.orgc_id": "4", + "misp.proposal_email_lock": false, + "misp.publish_timestamp": "1610637888", + "misp.published": true, + "misp.sharing_group_id": "0", + "misp.threat_level_id": 3, + "misp.uuid": "5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16", + "service.type": "threatintel", + "tags": [ + "misp-galaxy:banker=Qakbot", + "tlp:white" + ], + "threat.feed.dashboard_id": "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f", + "threat.feed.name": "[Filebeat] MISP", + "threat.indicator.marking.tlp": [ + "white" + ], + "threat.indicator.provider": "misp", + "threat.indicator.scanner_stats": 2, + "threat.indicator.type": "ipv4-addr" + } +] \ No newline at end of file