From d7fb188a69ae7434d991efb5bb00b31a40f9bcf7 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 5 Feb 2024 16:05:58 +0100 Subject: [PATCH 1/8] Adjust threat system test data to cover multiple pagination sequences and require correct timestamps after the first. --- .../_dev/deploy/docker/files/config.yml | 306 ++++++++++-------- .../_dev/test/system/test-default-config.yml | 4 +- 2 files changed, 167 insertions(+), 143 deletions(-) diff --git a/packages/ti_misp/_dev/deploy/docker/files/config.yml b/packages/ti_misp/_dev/deploy/docker/files/config.yml index e35e400564a..c2ff0944c9c 100644 --- a/packages/ti_misp/_dev/deploy/docker/files/config.yml +++ b/packages/ti_misp/_dev/deploy/docker/files/config.yml @@ -1,154 +1,34 @@ rules: - - path: /events/restSearch + - path: /events/restSearch # sequence 3, page 1 (repeats) methods: ["POST"] request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"1621599936"/ responses: - status_code: 200 body: |- { - "response": [ - { - "Event": { - "Attribute": [ - { - "Galaxy": [], - "ShadowAttribute": [], - "category": "Payload delivery", - "comment": "filename content for test event 3", - "deleted": false, - "disable_correlation": false, - "distribution": "5", - "event_id": "3633", - "first_seen": null, - "id": "266263", - "last_seen": null, - "object_id": "0", - "object_relation": null, - "sharing_group_id": "0", - "timestamp": "1621589229", - "to_ids": false, - "type": "filename", - "uuid": "3b322e1a-1dd8-490c-ab96-12e1bc3ee6a3", - "value": "thetestfile.txt" - } - ], - "EventReport": [], - "Galaxy": [], - "Object": [ - { - "Attribute": [ - { - "Galaxy": [], - "ShadowAttribute": [], - "category": "Payload delivery", - "comment": "", - "deleted": false, - "disable_correlation": false, - "distribution": "5", - "event_id": "3633", - "first_seen": null, - "id": "266265", - "last_seen": null, - "object_id": "18207", - "object_relation": "sha256", - "sharing_group_id": "0", - "timestamp": "1621589548", - "to_ids": true, - "type": "sha256", - "uuid": "657c5f2b-9d68-4ff7-a9ad-ab9e6a6c953e", - "value": "f33c27745f2bd87344be790465ef984a972fd539dc83bd4f61d4242c607ef1ee" - } - ], - "ObjectReference": [], - "comment": "File object for event 3", - "deleted": false, - "description": "File object describing a file with meta-information", - "distribution": "5", - "event_id": "3633", - "first_seen": null, - "id": "18207", - "last_seen": null, - "meta-category": "file", - "name": "file", - "sharing_group_id": "0", - "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": "22", - "timestamp": "1621589548", - "uuid": "42a88ad4-6834-46a9-a18b-aff9e078a4ea" - } - ], - "Org": { - "id": "1", - "local": true, - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "Orgc": { - "id": "1", - "local": true, - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "RelatedEvent": [ - { - "Event": { - "Org": { - "id": "1", - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "Orgc": { - "id": "1", - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "analysis": "0", - "date": "2021-05-21", - "distribution": "1", - "id": "3631", - "info": "Test event 1 just atrributes", - "org_id": "1", - "orgc_id": "1", - "published": false, - "threat_level_id": "1", - "timestamp": "1621588162", - "uuid": "8ca56ae9-3747-4172-93d2-808da1a4eaf3" - } - } - ], - "ShadowAttribute": [], - "analysis": "0", - "attribute_count": "6", - "date": "2021-05-21", - "disable_correlation": false, - "distribution": "1", - "event_creator_email": "admin@admin.test", - "extends_uuid": "", - "id": "3633", - "info": "Test event 3 objects and attributes", - "locked": false, - "org_id": "1", - "orgc_id": "1", - "proposal_email_lock": false, - "publish_timestamp": "0", - "published": false, - "sharing_group_id": "0", - "threat_level_id": "1", - "timestamp": "1621592532", - "uuid": "4edb20c7-8175-484d-bdcd-fce6872c1ef3" - } - } - ] + "response": [] } - - path: /events/restSearch + - path: /events/restSearch # sequence 2, page 2 methods: ["POST"] request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"1621592532"/ + responses: + - status_code: 200 + body: |- + { + "response": [] + } + - path: /events/restSearch # sequence 2, page 1 + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"1621592532"/ responses: - status_code: 200 body: |- @@ -239,7 +119,7 @@ rules: "published": false, "sharing_group_id": "0", "threat_level_id": "2", - "timestamp": "1621588836", + "timestamp": "1621598836", "uuid": "efbca287-edb5-4ad7-b8e4-fe9da514a763" } }, @@ -255,7 +135,7 @@ rules: "uuid": "54323f2c-e50c-4268-896c-4867950d210b", "attribute_count": "29", "analysis": "2", - "timestamp": "1412579577", + "timestamp": "1621599936", "distribution": "3", "proposal_email_lock": false, "locked": false, @@ -333,18 +213,162 @@ rules: } ] } - - path: /events/restSearch + - path: /events/restSearch # sequence 1, page 2 methods: ["POST"] request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"3","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: |- { "response": [] } + - path: /events/restSearch # sequence 1, page 1 + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": [ + { + "Event": { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Payload delivery", + "comment": "filename content for test event 3", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "266263", + "last_seen": null, + "object_id": "0", + "object_relation": null, + "sharing_group_id": "0", + "timestamp": "1621589229", + "to_ids": false, + "type": "filename", + "uuid": "3b322e1a-1dd8-490c-ab96-12e1bc3ee6a3", + "value": "thetestfile.txt" + } + ], + "EventReport": [], + "Galaxy": [], + "Object": [ + { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "266265", + "last_seen": null, + "object_id": "18207", + "object_relation": "sha256", + "sharing_group_id": "0", + "timestamp": "1621589548", + "to_ids": true, + "type": "sha256", + "uuid": "657c5f2b-9d68-4ff7-a9ad-ab9e6a6c953e", + "value": "f33c27745f2bd87344be790465ef984a972fd539dc83bd4f61d4242c607ef1ee" + } + ], + "ObjectReference": [], + "comment": "File object for event 3", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "18207", + "last_seen": null, + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "22", + "timestamp": "1621589548", + "uuid": "42a88ad4-6834-46a9-a18b-aff9e078a4ea" + } + ], + "Org": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "analysis": "0", + "date": "2021-05-21", + "distribution": "1", + "id": "3631", + "info": "Test event 1 just atrributes", + "org_id": "1", + "orgc_id": "1", + "published": false, + "threat_level_id": "1", + "timestamp": "1621588162", + "uuid": "8ca56ae9-3747-4172-93d2-808da1a4eaf3" + } + } + ], + "ShadowAttribute": [], + "analysis": "0", + "attribute_count": "6", + "date": "2021-05-21", + "disable_correlation": false, + "distribution": "1", + "event_creator_email": "admin@admin.test", + "extends_uuid": "", + "id": "3633", + "info": "Test event 3 objects and attributes", + "locked": false, + "org_id": "1", + "orgc_id": "1", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "1", + "timestamp": "1621592532", + "uuid": "4edb20c7-8175-484d-bdcd-fce6872c1ef3" + } + } + ] + } - path: /attributes/restSearch methods: ["POST"] request_headers: diff --git a/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml b/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml index c689b96f1a6..140488eddee 100644 --- a/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml +++ b/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml @@ -6,8 +6,8 @@ data_stream: preserve_original_event: true url: http://{{Hostname}}:{{Port}} api_token: test - interval: 10m - initial_interval: 10m + interval: 1s + initial_interval: 10s enable_request_tracer: true assert: hit_count: 3 From 05d618f52d93f362b0f9ee426593ad752cd89b65 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 5 Feb 2024 16:08:26 +0100 Subject: [PATCH 2/8] Correct threat timestamp logic: use cursor data correctly, don't let the starting point shift even if initial response are empty. --- .../data_stream/threat/agent/stream/httpjson.yml.hbs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs index 50415377874..8c1f3c5ed45 100644 --- a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs @@ -38,7 +38,12 @@ request.transforms: value: json - set: target: body.timestamp - value: '[[.cursor.timestamp.Unix]]' + value: >- + [[- if index .cursor "timestamp" -]] + [[- .cursor.timestamp -]] + [[- else -]] + [[- .last_response.url.params.Get "timestamp" -]] + [[- end -]] default: '[[ (now (parseDuration "-{{initial_interval}}")).Unix ]]' - set: # Ignored by MISP, set as a workaround to make it available in response.pagination. From ca8d4bb78ff18bb0272e961c3f88b799fa8ab2b5 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 5 Feb 2024 17:35:14 +0100 Subject: [PATCH 3/8] Adjust threat_attributes system test data to cover multiple pagination sequences and require correct timestamps after the first. --- .../_dev/deploy/docker/files/config.yml | 1191 +++++++++-------- .../_dev/test/system/test-default-config.yml | 4 +- 2 files changed, 614 insertions(+), 581 deletions(-) diff --git a/packages/ti_misp/_dev/deploy/docker/files/config.yml b/packages/ti_misp/_dev/deploy/docker/files/config.yml index c2ff0944c9c..b73bc9d6814 100644 --- a/packages/ti_misp/_dev/deploy/docker/files/config.yml +++ b/packages/ti_misp/_dev/deploy/docker/files/config.yml @@ -369,621 +369,654 @@ rules: } ] } - - path: /attributes/restSearch + - path: /attributes/restSearch # sequence 3, page 1 (repeats) methods: ["POST"] request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"1700667505"/ + responses: + - status_code: 200 + body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " + - path: /attributes/restSearch # sequence 2, page 2 + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"includeDecayScore":"true","limit":"10","page":"2","returnFormat":"json","timestamp":"1412320446"/ + responses: + - status_code: 200 + body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " + - path: /attributes/restSearch # sequence 2, page 1 + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"1412320446"/ responses: - status_code: 200 body: |- { "response": { - "Attribute": [ - { + "Attribute": [ + { + "id": "3", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbe-12a4-4345-b0a4-1fda950d210b", + "timestamp": "1412320447", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "https://gist.githubusercontent.com/andrewsmhay/de1cdc63d04c2bbf8c12/raw/f20402cf5a0c646c63c4521f60587703fe654443/iplist", + "Event": { + "org_id": "1", + "distribution": "3", "id": "1", - "event_id": "1", - "object_id": "0", - "object_relation": null, - "category": "External analysis", - "type": "link", - "to_ids": false, - "uuid": "542e4cbd-ee78-4a57-bfb8-1fda950d210b", - "timestamp": "1412320445", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "http://labs.opendns.com/2014/10/02/opendns-and-bash/", - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1", - "info": "OSINT ShellShock scanning IPs from OpenDNS", - "orgc_id": "2", - "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" - } + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "4", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "text", + "to_ids": false, + "uuid": "542e4ccc-b8fc-44af-959d-6ead950d210b", + "timestamp": "1412320460", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "Shellshock", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "5", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "comment", + "to_ids": false, + "uuid": "542e4ce7-6120-41c0-8793-e90e950d210b", + "timestamp": "1412320487", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "Data encoded by David André", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "266922", + "event_id": "1279", + "object_id": "20246", + "object_relation": "sha1", + "category": "Payload delivery", + "type": "sha1", + "to_ids": true, + "uuid": "84850997-631c-44ea-ac71-5f8bb4e6e1f0", + "timestamp": "1696914151", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "c514799ffdc38d48b7e90b8b6a324c354d1fd2a2", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1279", + "info": "FormBook campaign", + "orgc_id": "3", + "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" }, - { - "id": "2", - "event_id": "1", - "object_id": "0", - "object_relation": null, - "category": "External analysis", - "type": "link", - "to_ids": false, - "uuid": "542e4cbe-d560-4e14-9157-1fda950d210b", - "timestamp": "1412320446", + "Object": { + "id": "20246", "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "https://gist.github.com/andrewsmhay/de1cdc63d04c2bbf8c12", - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1", - "info": "OSINT ShellShock scanning IPs from OpenDNS", - "orgc_id": "2", - "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" - } + "sharing_group_id": "0" }, - { - "id": "3", - "event_id": "1", - "object_id": "0", - "object_relation": null, - "category": "External analysis", - "type": "link", - "to_ids": false, - "uuid": "542e4cbe-12a4-4345-b0a4-1fda950d210b", - "timestamp": "1412320446", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "https://gist.githubusercontent.com/andrewsmhay/de1cdc63d04c2bbf8c12/raw/f20402cf5a0c646c63c4521f60587703fe654443/iplist", - "Event": { - "org_id": "1", - "distribution": "3", + "Tag": [ + { + "id": "10", + "name": "type:OSINT", + "colour": "#004646", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "204", + "name": "osint:lifetime=\"perpetual\"", + "colour": "#0071c3", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "540", + "name": "osint:certainty=\"50\"", + "colour": "#0087e8", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "11", + "name": "tlp:white", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1051", + "name": "tlp:clear", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "338", + "name": "misp-galaxy:tool=\"FormBook\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1056", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "474", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + } + ] + }, + { + "id": "266790", + "event_id": "1279", + "object_id": "0", + "object_relation": null, + "category": "Network activity", + "type": "url", + "to_ids": true, + "uuid": "78f6d250-c68d-42df-8083-b55e4d20779e", + "timestamp": "1686914587", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://www.doordelivery.life/km37/", + "decay_score": [ + { + "score": -0, + "base_score": 0, + "decayed": true, + "DecayingModel": { "id": "1", - "info": "OSINT ShellShock scanning IPs from OpenDNS", - "orgc_id": "2", - "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + "name": "test-decay-model" + } + }, + { + "score": 0, + "base_score": 50, + "decayed": true, + "DecayingModel": { + "id": "2", + "name": "test-decay-model-2" + } } + ], + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1279", + "info": "FormBook campaign", + "orgc_id": "3", + "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" }, - { - "id": "4", - "event_id": "1", - "object_id": "0", - "object_relation": null, - "category": "External analysis", - "type": "text", - "to_ids": false, - "uuid": "542e4ccc-b8fc-44af-959d-6ead950d210b", - "timestamp": "1412320460", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "Shellshock", - "Event": { - "org_id": "1", - "distribution": "3", + "Tag": [ + { + "id": "10", + "name": "type:OSINT", + "colour": "#004646", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "204", + "name": "osint:lifetime=\"perpetual\"", + "colour": "#0071c3", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "540", + "name": "osint:certainty=\"50\"", + "colour": "#0087e8", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "11", + "name": "tlp:white", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1051", + "name": "tlp:clear", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "338", + "name": "misp-galaxy:tool=\"FormBook\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1056", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "474", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + } + ] + }, + { + "id": "266793", + "event_id": "1279", + "object_id": "0", + "object_relation": null, + "category": "Network activity", + "type": "url", + "to_ids": true, + "uuid": "efa8a550-bc25-4d93-abcd-1c00eaa4acdd", + "timestamp": "1686914588", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://www.blueridgebedracks.com/km37/", + "decay_score": [ + { + "score": -0, + "base_score": 0, + "decayed": false, + "DecayingModel": { "id": "1", - "info": "OSINT ShellShock scanning IPs from OpenDNS", - "orgc_id": "2", - "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + "name": "test-decay-model" + } + }, + { + "score": 0, + "base_score": 50, + "decayed": false, + "DecayingModel": { + "id": "2", + "name": "test-decay-model-2" + } } + ], + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1279", + "info": "FormBook campaign", + "orgc_id": "3", + "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" }, - { - "id": "5", - "event_id": "1", - "object_id": "0", - "object_relation": null, - "category": "External analysis", - "type": "comment", - "to_ids": false, - "uuid": "542e4ce7-6120-41c0-8793-e90e950d210b", - "timestamp": "1412320487", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "Data encoded by David André", - "Event": { - "org_id": "1", - "distribution": "3", + "Tag": [ + { + "id": "10", + "name": "type:OSINT", + "colour": "#004646", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "204", + "name": "osint:lifetime=\"perpetual\"", + "colour": "#0071c3", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "540", + "name": "osint:certainty=\"50\"", + "colour": "#0087e8", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "11", + "name": "tlp:white", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1051", + "name": "tlp:clear", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "338", + "name": "misp-galaxy:tool=\"FormBook\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1056", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "474", + "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + } + ] + }, + { + "id": "268565", + "event_id": "1294", + "object_id": "0", + "object_relation": null, + "category": "Network activity", + "type": "url", + "to_ids": true, + "uuid": "ec341f4e-0f70-4569-8ac5-e35465572726", + "timestamp": "1700667504", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://185.122.204.197/acb.sh", + "decay_score": [ + { + "score": 0, + "base_score": 0, + "decayed": true, + "DecayingModel": { "id": "1", - "info": "OSINT ShellShock scanning IPs from OpenDNS", - "orgc_id": "2", - "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + "name": "test-decay-model" + } + }, + { + "score": 49.98530793883329, + "base_score": 50, + "decayed": false, + "DecayingModel": { + "id": "2", + "name": "test-decay-model-2" + } } + ], + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1294", + "info": "CVE-2023-46604 (Apache ActiveMQ) Vulnerability Exploited to Infect Systems With Cryptominers and Rootkits", + "orgc_id": "3", + "uuid": "df7b7020-9f17-4a3c-9824-1baa4ff67cb1" }, - { - "id": "266790", - "event_id": "1279", - "object_id": "0", - "object_relation": null, - "category": "Network activity", - "type": "url", - "to_ids": true, - "uuid": "78f6d250-c68d-42df-8083-b55e4d20779e", - "timestamp": "1686914587", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "http://www.doordelivery.life/km37/", - "decay_score": [ - { - "score": -0, - "base_score": 0, - "decayed": true, - "DecayingModel": { - "id": "1", - "name": "test-decay-model" - } - }, - { - "score": 0, - "base_score": 50, - "decayed": true, - "DecayingModel": { - "id": "2", - "name": "test-decay-model-2" - } - } - ], - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1279", - "info": "FormBook campaign", - "orgc_id": "3", - "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" + "Tag": [ + { + "id": "10", + "name": "type:OSINT", + "colour": "#004646", + "numerical_value": null, + "inherited": 1 }, - "Tag": [ - { - "id": "10", - "name": "type:OSINT", - "colour": "#004646", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "204", - "name": "osint:lifetime=\"perpetual\"", - "colour": "#0071c3", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "540", - "name": "osint:certainty=\"50\"", - "colour": "#0087e8", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "11", - "name": "tlp:white", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1051", - "name": "tlp:clear", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "338", - "name": "misp-galaxy:tool=\"FormBook\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1056", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "474", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - } - ] - }, - { - "id": "268565", - "event_id": "1294", - "object_id": "0", - "object_relation": null, - "category": "Network activity", - "type": "url", - "to_ids": true, - "uuid": "ec341f4e-0f70-4569-8ac5-e35465572726", - "timestamp": "1700667504", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "http://185.122.204.197/acb.sh", - "decay_score": [ - { - "score": 0, - "base_score": 0, - "decayed": true, - "DecayingModel": { - "id": "1", - "name": "test-decay-model" - } - }, - { - "score": 49.98530793883329, - "base_score": 50, - "decayed": false, - "DecayingModel": { - "id": "2", - "name": "test-decay-model-2" - } - } - ], - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1294", - "info": "CVE-2023-46604 (Apache ActiveMQ) Vulnerability Exploited to Infect Systems With Cryptominers and Rootkits", - "orgc_id": "3", - "uuid": "df7b7020-9f17-4a3c-9824-1baa4ff67cb1" + { + "id": "204", + "name": "osint:lifetime=\"perpetual\"", + "colour": "#0071c3", + "numerical_value": null, + "inherited": 1 }, - "Tag": [ - { - "id": "10", - "name": "type:OSINT", - "colour": "#004646", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "204", - "name": "osint:lifetime=\"perpetual\"", - "colour": "#0071c3", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "11", - "name": "tlp:white", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1051", - "name": "tlp:clear", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "713", - "name": "misp-galaxy:mitre-attack-pattern=\"Resource Hijacking - T1496\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - } - ] - }, - { - "id": "266793", - "event_id": "1279", - "object_id": "0", - "object_relation": null, - "category": "Network activity", - "type": "url", - "to_ids": true, - "uuid": "efa8a550-bc25-4d93-abcd-1c00eaa4acdd", - "timestamp": "1686914587", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "http://www.blueridgebedracks.com/km37/", - "decay_score": [ - { - "score": -0, - "base_score": 0, - "decayed": false, - "DecayingModel": { - "id": "1", - "name": "test-decay-model" - } - }, - { - "score": 0, - "base_score": 50, - "decayed": false, - "DecayingModel": { - "id": "2", - "name": "test-decay-model-2" - } - } - ], - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1279", - "info": "FormBook campaign", - "orgc_id": "3", - "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" + { + "id": "11", + "name": "tlp:white", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 }, - "Tag": [ - { - "id": "10", - "name": "type:OSINT", - "colour": "#004646", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "204", - "name": "osint:lifetime=\"perpetual\"", - "colour": "#0071c3", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "540", - "name": "osint:certainty=\"50\"", - "colour": "#0087e8", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "11", - "name": "tlp:white", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1051", - "name": "tlp:clear", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "338", - "name": "misp-galaxy:tool=\"FormBook\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1056", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "474", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - } - ] - }, - { - "id": "268570", - "event_id": "1294", - "object_id": "0", - "object_relation": null, - "category": "Network activity", - "type": "url", - "to_ids": true, - "uuid": "28f55810-c61e-42d0-8565-cc7d2e7eb57c", - "timestamp": "1700667504", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "http://194.38.22.53/libsystem.so", - "decay_score": [ - { - "score": 49.98530793883329, - "base_score": 50, - "decayed": false, - "DecayingModel": { - "id": "2", - "name": "test-decay-model-2" - } - } - ], - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1294", - "info": "CVE-2023-46604 (Apache ActiveMQ) Vulnerability Exploited to Infect Systems With Cryptominers and Rootkits", - "orgc_id": "3", - "uuid": "df7b7020-9f17-4a3c-9824-1baa4ff67cb1" + { + "id": "1051", + "name": "tlp:clear", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 }, - "Tag": [ - { - "id": "10", - "name": "type:OSINT", - "colour": "#004646", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "204", - "name": "osint:lifetime=\"perpetual\"", - "colour": "#0071c3", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "11", - "name": "tlp:white", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1051", - "name": "tlp:clear", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "713", - "name": "misp-galaxy:mitre-attack-pattern=\"Resource Hijacking - T1496\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - } - ] + { + "id": "713", + "name": "misp-galaxy:mitre-attack-pattern=\"Resource Hijacking - T1496\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + } + ] + }, + { + "id": "268570", + "event_id": "1294", + "object_id": "0", + "object_relation": null, + "category": "Network activity", + "type": "url", + "to_ids": true, + "uuid": "28f55810-c61e-42d0-8565-cc7d2e7eb57c", + "timestamp": "1700667505", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://194.38.22.53/libsystem.so", + "decay_score": [ + { + "score": 49.98530793883329, + "base_score": 50, + "decayed": false, + "DecayingModel": { + "id": "2", + "name": "test-decay-model-2" + } + } + ], + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1294", + "info": "CVE-2023-46604 (Apache ActiveMQ) Vulnerability Exploited to Infect Systems With Cryptominers and Rootkits", + "orgc_id": "3", + "uuid": "df7b7020-9f17-4a3c-9824-1baa4ff67cb1" }, - { - "id": "266922", - "event_id": "1279", - "object_id": "20246", - "object_relation": "sha1", - "category": "Payload delivery", - "type": "sha1", - "to_ids": true, - "uuid": "84850997-631c-44ea-ac71-5f8bb4e6e1f0", - "timestamp": "1696914151", - "distribution": "5", - "sharing_group_id": "0", - "comment": "", - "deleted": false, - "disable_correlation": false, - "first_seen": null, - "last_seen": null, - "value": "c514799ffdc38d48b7e90b8b6a324c354d1fd2a2", - "Event": { - "org_id": "1", - "distribution": "3", - "id": "1279", - "info": "FormBook campaign", - "orgc_id": "3", - "uuid": "f45fe125-7f3f-4335-bf74-5ab61eb5b645" + "Tag": [ + { + "id": "10", + "name": "type:OSINT", + "colour": "#004646", + "numerical_value": null, + "inherited": 1 }, - "Object": { - "id": "20246", - "distribution": "5", - "sharing_group_id": "0" + { + "id": "204", + "name": "osint:lifetime=\"perpetual\"", + "colour": "#0071c3", + "numerical_value": null, + "inherited": 1 }, - "Tag": [ - { - "id": "10", - "name": "type:OSINT", - "colour": "#004646", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "204", - "name": "osint:lifetime=\"perpetual\"", - "colour": "#0071c3", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "540", - "name": "osint:certainty=\"50\"", - "colour": "#0087e8", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "11", - "name": "tlp:white", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1051", - "name": "tlp:clear", - "colour": "#ffffff", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "338", - "name": "misp-galaxy:tool=\"FormBook\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "1056", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1566.001\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - }, - { - "id": "474", - "name": "misp-galaxy:mitre-attack-pattern=\"Spearphishing Attachment - T1193\"", - "colour": "#0088cc", - "numerical_value": null, - "inherited": 1 - } - ] - } + { + "id": "11", + "name": "tlp:white", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "1051", + "name": "tlp:clear", + "colour": "#ffffff", + "numerical_value": null, + "inherited": 1 + }, + { + "id": "713", + "name": "misp-galaxy:mitre-attack-pattern=\"Resource Hijacking - T1496\"", + "colour": "#0088cc", + "numerical_value": null, + "inherited": 1 + } + ] + } ] } } - - path: /attributes/restSearch + - path: /attributes/restSearch # sequence 1, page 2 methods: ["POST"] request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"includeDecayScore":"true","limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " + - path: /attributes/restSearch # sequence 1, page 1 + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": { + "Attribute": [ + { + "id": "1", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbd-ee78-4a57-bfb8-1fda950d210b", + "timestamp": "1412320445", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://labs.opendns.com/2014/10/02/opendns-and-bash/", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "2", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbe-d560-4e14-9157-1fda950d210b", + "timestamp": "1412320446", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "https://gist.github.com/andrewsmhay/de1cdc63d04c2bbf8c12", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + } + ] + } + } diff --git a/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml b/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml index fa97cc16e46..b3996354802 100644 --- a/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml +++ b/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml @@ -6,8 +6,8 @@ data_stream: preserve_original_event: true url: http://{{Hostname}}:{{Port}} api_token: test - interval: 10m - initial_interval: 10m + interval: 1s + initial_interval: 10s enable_request_tracer: true ioc_expiration_duration: 5d assert: From fc4992099ff52d1de28bdd111d7fb88f19a866a4 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 5 Feb 2024 17:36:25 +0100 Subject: [PATCH 4/8] Correct cursor timestamp extraction expression to account for 'keep_parent: false'. --- .../data_stream/threat_attributes/agent/stream/httpjson.yml.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs index a67f3cd9d61..b13296d4611 100644 --- a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs @@ -74,7 +74,7 @@ response.pagination: value: '[[.last_response.url.params.Get "timestamp"]]' cursor: timestamp: - value: '[[.last_event.Attribute.timestamp]]' + value: '[[.last_event.timestamp]]' tags: {{#if preserve_original_event}} - preserve_original_event From f223416e24761e266e37b22dc6a28f4d14bc5f86 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 5 Feb 2024 17:37:03 +0100 Subject: [PATCH 5/8] Correct threat_attributes timestamp logic: use cursor data correctly, don't let the starting point shift even if initial response are empty. --- .../threat_attributes/agent/stream/httpjson.yml.hbs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs index b13296d4611..5704199410e 100644 --- a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs @@ -45,7 +45,12 @@ request.transforms: value: json - set: target: body.timestamp - value: '[[.cursor.timestamp.Unix]]' + value: >- + [[- if index .cursor "timestamp" -]] + [[- .cursor.timestamp -]] + [[- else -]] + [[- .last_response.url.params.Get "timestamp" -]] + [[- end -]] default: '[[ (now (parseDuration "-{{initial_interval}}")).Unix ]]' - set: # Ignored by MISP, set as a workaround to make it available in response.pagination. From 8993dd83c0d37599874c4484c86b2974aac05a18 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 6 Feb 2024 12:14:14 +0100 Subject: [PATCH 6/8] Add "order":"timestamp" parameter to MISP requests for events and attributes. --- .../_dev/deploy/docker/files/config.yml | 20 +++++++++---------- .../threat/agent/stream/httpjson.yml.hbs | 3 +++ .../agent/stream/httpjson.yml.hbs | 3 +++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/ti_misp/_dev/deploy/docker/files/config.yml b/packages/ti_misp/_dev/deploy/docker/files/config.yml index b73bc9d6814..30126c00411 100644 --- a/packages/ti_misp/_dev/deploy/docker/files/config.yml +++ b/packages/ti_misp/_dev/deploy/docker/files/config.yml @@ -4,7 +4,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"1621599936"/ + request_body: /^{"limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"1621599936"/ responses: - status_code: 200 body: |- @@ -16,7 +16,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"1621592532"/ + request_body: /^{"limit":"10","order":"timestamp","page":"2","returnFormat":"json","timestamp":"1621592532"/ responses: - status_code: 200 body: |- @@ -28,7 +28,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"1621592532"/ + request_body: /^{"limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"1621592532"/ responses: - status_code: 200 body: |- @@ -218,7 +218,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","order":"timestamp","page":"2","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: |- @@ -230,7 +230,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: |- @@ -374,7 +374,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"1700667505"/ + request_body: /^{"includeDecayScore":"true","limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"1700667505"/ responses: - status_code: 200 body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " @@ -383,7 +383,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"2","returnFormat":"json","timestamp":"1412320446"/ + request_body: /^{"includeDecayScore":"true","limit":"10","order":"timestamp","page":"2","returnFormat":"json","timestamp":"1412320446"/ responses: - status_code: 200 body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " @@ -392,7 +392,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"1412320446"/ + request_body: /^{"includeDecayScore":"true","limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"1412320446"/ responses: - status_code: 200 body: |- @@ -947,7 +947,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"includeDecayScore":"true","limit":"10","order":"timestamp","page":"2","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " @@ -956,7 +956,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"includeDecayScore":"true","limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"includeDecayScore":"true","limit":"10","order":"timestamp","page":"1","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: |- diff --git a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs index 8c1f3c5ed45..816476e4ce6 100644 --- a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs @@ -45,6 +45,9 @@ request.transforms: [[- .last_response.url.params.Get "timestamp" -]] [[- end -]] default: '[[ (now (parseDuration "-{{initial_interval}}")).Unix ]]' +- set: + target: body.order + value: timestamp - set: # Ignored by MISP, set as a workaround to make it available in response.pagination. target: url.params.timestamp diff --git a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs index 5704199410e..01dae70f968 100644 --- a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs @@ -59,6 +59,9 @@ request.transforms: - set: target: body.includeDecayScore value: true +- set: + target: body.order + value: timestamp response.split: target: body.response.Attribute From 922db359d87f6d06aed8e12a67633d539ce85b4b Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 6 Feb 2024 14:23:44 +0100 Subject: [PATCH 7/8] Fix PR link in earlier changelog entry. --- packages/ti_misp/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml index 54e883d5550..5c5ad479009 100644 --- a/packages/ti_misp/changelog.yml +++ b/packages/ti_misp/changelog.yml @@ -8,7 +8,7 @@ changes: - description: Added attribute limit option to the UI type: enhancement - link: https://github.com/elastic/integrations/pull/8943 + link: https://github.com/elastic/integrations/pull/9064 - version: "1.29.1" changes: - description: Changed owners From a788295f7b3c2f56c7349b5b93fd2adbdaae8357 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Tue, 6 Feb 2024 15:10:49 +0100 Subject: [PATCH 8/8] Change log and manifest update. --- packages/ti_misp/changelog.yml | 5 +++++ packages/ti_misp/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml index 5c5ad479009..01245fff121 100644 --- a/packages/ti_misp/changelog.yml +++ b/packages/ti_misp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.31.0" + changes: + - description: Pagination fixes + type: enhancement + link: https://github.com/elastic/integrations/pull/9073 - version: "1.30.1" changes: - description: Add recent new field to latest_ioc transform dest diff --git a/packages/ti_misp/manifest.yml b/packages/ti_misp/manifest.yml index f5f192f0a07..7dba38a701f 100644 --- a/packages/ti_misp/manifest.yml +++ b/packages/ti_misp/manifest.yml @@ -1,6 +1,6 @@ name: ti_misp title: MISP -version: "1.30.1" +version: "1.31.0" description: Ingest threat intelligence indicators from MISP platform with Elastic Agent. type: integration format_version: "3.0.0"