Skip to content

Commit

Permalink
misp: parse URIs for URI type threats
Browse files Browse the repository at this point in the history
  • Loading branch information
@efd6
efd6 committed Dec 20, 2023
1 parent c3a2f22 commit 2f0c43f
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 28 deletions.
5 changes: 5 additions & 0 deletions packages/ti_misp/changelog.yml
View file
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.27.1"
changes:
- description: Parse URIs for URI type threats.
type: bugfix
link: https://github.com/elastic/integrations/pull/8760
- version: "1.27.0"
changes:
- description: Limit request tracer log count to five.
Expand Down
1 change: 1 addition & 0 deletions ...ages/ti_misp/data_stream/threat_attributes/_dev/test/pipeline/test-misp-sample-ndjson.log
View file
Expand Up @@ -10,3 +10,4 @@
{"id":"40687","event_id":"53","object_id":"0","object_relation":null,"category":"Network activity","type":"AS","to_ids":false,"uuid":"54f86825-c80c-47cf-a795-48c1950d210b","timestamp":"1425565733","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"first_seen":null,"last_seen":null,"value":"48031","Event":{"org_id":"1","distribution":"3","id":"53","info":"OSINT Analysis of malcious CHM file by OpenDNS","orgc_id":"2","uuid":"54f8662f-c7f0-4f59-a42a-a9a9950d210b"}}
{"id":"1084","event_id":"2","object_id":"0","object_relation":null,"category":"Network activity","type":"ip-dst","to_ids":true,"uuid":"54324081-3308-4f1f-8674-4953950d210b","timestamp":"1412579457","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"first_seen":null,"last_seen":null,"value":"89.160.20.129","Event":{"org_id":"1","distribution":"3","id":"2","info":"OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks","orgc_id":"2","uuid":"54323f2c-e50c-4268-896c-4867950d210b"}}
{"id":"24749","event_id":"10","object_id":"0","object_relation":null,"category":"Network activity","type":"email-dst","to_ids":true,"uuid":"544ff4c2-914c-482f-aa29-4c43950d210b","timestamp":"1414526146","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"first_seen":null,"last_seen":null,"value":"lisa.cuddy@wind0ws.kz","Event":{"org_id":"1","distribution":"3","id":"10","info":"OSINT APT28: A Window into Russia’s Cyber Espionage Operations? blog post by FireEye","orgc_id":"2","uuid":"544fee45-f108-4fa6-ace9-3989950d210b"}}
{"Event":{"distribution":"3","id":"2578","info":"SmokeLoader з темою рахунків (CERT-UA#8091)","org_id":"1","orgc_id":"3","uuid":"9c56976c-cbdb-45e6-a185-dce19fab0b45"},"category":"Network activity","comment":"URI","deleted":false,"disable_correlation":false,"distribution":"5","event_id":"2578","first_seen":null,"id":"474314","last_seen":null,"object_id":"0","object_relation":null,"sharing_group_id":"0","timestamp":"1700488404","to_ids":true,"type":"uri","uuid":"b3edd2ca-21de-44ff-ac58-40cb99cfdba1","value":"http://druigvsegdaryadom.ir/index.php"}
62 changes: 62 additions & 0 deletions ...ata_stream/threat_attributes/_dev/test/pipeline/test-misp-sample-ndjson.log-expected.json
View file
Expand Up @@ -762,6 +762,68 @@
"type": "email-addr"
}
}
},
{
"@timestamp": "2023-11-20T13:53:24.000Z",
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"threat"
],
"kind": "enrichment",
"original": "{\"Event\":{\"distribution\":\"3\",\"id\":\"2578\",\"info\":\"SmokeLoader з темою рахунків (CERT-UA#8091)\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"uuid\":\"9c56976c-cbdb-45e6-a185-dce19fab0b45\"},\"category\":\"Network activity\",\"comment\":\"URI\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"2578\",\"first_seen\":null,\"id\":\"474314\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1700488404\",\"to_ids\":true,\"type\":\"uri\",\"uuid\":\"b3edd2ca-21de-44ff-ac58-40cb99cfdba1\",\"value\":\"http://druigvsegdaryadom.ir/index.php\"}",
"type": [
"indicator"
]
},
"misp": {
"attribute": {
"category": "Network activity",
"comment": "URI",
"deleted": false,
"disable_correlation": false,
"distribution": 5,
"event_id": "2578",
"id": "474314",
"object_id": "0",
"sharing_group_id": "0",
"to_ids": true,
"type": "uri",
"uuid": "b3edd2ca-21de-44ff-ac58-40cb99cfdba1"
},
"event": {
"distribution": 3,
"id": "2578",
"info": "SmokeLoader з темою рахунків (CERT-UA#8091)",
"orgc_id": "3",
"uuid": "9c56976c-cbdb-45e6-a185-dce19fab0b45"
}
},
"organization": {
"id": "1"
},
"tags": [
"preserve_original_event"
],
"threat": {
"feed": {
"name": "MISP"
},
"indicator": {
"provider": "misp",
"type": "url",
"url": {
"domain": "druigvsegdaryadom.ir",
"extension": "php",
"full": "http://druigvsegdaryadom.ir/index.php",
"original": "http://druigvsegdaryadom.ir/index.php",
"path": "/index.php",
"scheme": "http"
}
}
}
}
]
}
5 changes: 3 additions & 2 deletions packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml
View file
Expand Up @@ -183,12 +183,13 @@ processors:
target_field: threat.indicator.url
keep_original: true
remove_if_successful: true
if: ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'
if: ctx.threat?.indicator?.type == 'url'
ignore_failure: true
- set:
field: threat.indicator.url.full
value: "{{{threat.indicator.url.original}}}"
ignore_empty_value: true
if: "ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'"
if: ctx.threat?.indicator?.type == 'url' && ctx.threat?.indicator?.url?.original != null

## Regkey indicator operations
- set:
Expand Down
51 changes: 26 additions & 25 deletions packages/ti_misp/data_stream/threat_attributes/sample_event.json
View file
@@ -1,11 +1,11 @@
{
"@timestamp": "2014-10-03T07:14:05.000Z",
"@timestamp": "2023-11-20T13:53:24.000Z",
"agent": {
"ephemeral_id": "cdecad63-3561-4779-8f29-0f578cdbaf3e",
"id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
"ephemeral_id": "d28d6fee-2a8c-43ff-898f-561ba521d297",
"id": "8e3dcae6-8d1c-46c1-bed0-bf69fdde05e5",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "8.9.1"
"version": "8.11.1"
},
"data_stream": {
"dataset": "ti_misp.threat_attributes",
Expand All @@ -16,20 +16,20 @@
"version": "8.11.0"
},
"elastic_agent": {
"id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
"id": "8e3dcae6-8d1c-46c1-bed0-bf69fdde05e5",
"snapshot": false,
"version": "8.9.1"
"version": "8.11.1"
},
"event": {
"agent_id_status": "verified",
"category": [
"threat"
],
"created": "2023-08-28T15:44:00.019Z",
"created": "2023-12-20T07:39:40.826Z",
"dataset": "ti_misp.threat_attributes",
"ingested": "2023-08-28T15:44:02Z",
"ingested": "2023-12-20T07:39:43Z",
"kind": "enrichment",
"original": "{\"Event\":{\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT ShellShock scanning IPs from OpenDNS\",\"org_id\":\"1\",\"orgc_id\":\"2\",\"uuid\":\"542e4c9c-cadc-4f8f-bb11-6d13950d210b\"},\"category\":\"External analysis\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"1\",\"first_seen\":null,\"id\":\"1\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1412320445\",\"to_ids\":false,\"type\":\"link\",\"uuid\":\"542e4cbd-ee78-4a57-bfb8-1fda950d210b\",\"value\":\"http://labs.opendns.com/2014/10/02/opendns-and-bash/\"}",
"original": "{\"Event\":{\"distribution\":\"3\",\"id\":\"2578\",\"info\":\"SmokeLoader з темою рахунків (CERT-UA#8091)\",\"org_id\":\"1\",\"orgc_id\":\"3\",\"uuid\":\"9c56976c-cbdb-45e6-a185-dce19fab0b45\"},\"category\":\"Network activity\",\"comment\":\"URI\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"2578\",\"first_seen\":null,\"id\":\"474314\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1700488404\",\"to_ids\":true,\"type\":\"uri\",\"uuid\":\"b3edd2ca-21de-44ff-ac58-40cb99cfdba1\",\"value\":\"http://druigvsegdaryadom.ir/index.php\"}",
"type": [
"indicator"
]
Expand All @@ -39,25 +39,25 @@
},
"misp": {
"attribute": {
"category": "External analysis",
"comment": "",
"category": "Network activity",
"comment": "URI",
"deleted": false,
"disable_correlation": false,
"distribution": 5,
"event_id": "1",
"id": "1",
"event_id": "2578",
"id": "474314",
"object_id": "0",
"sharing_group_id": "0",
"to_ids": false,
"type": "link",
"uuid": "542e4cbd-ee78-4a57-bfb8-1fda950d210b"
"to_ids": true,
"type": "uri",
"uuid": "b3edd2ca-21de-44ff-ac58-40cb99cfdba1"
},
"event": {
"distribution": 3,
"id": "1",
"info": "OSINT ShellShock scanning IPs from OpenDNS",
"orgc_id": "2",
"uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b"
"id": "2578",
"info": "SmokeLoader з темою рахунків (CERT-UA#8091)",
"orgc_id": "3",
"uuid": "9c56976c-cbdb-45e6-a185-dce19fab0b45"
}
},
"organization": {
Expand All @@ -76,12 +76,13 @@
"provider": "misp",
"type": "url",
"url": {
"domain": "labs.opendns.com",
"full": "http://labs.opendns.com/2014/10/02/opendns-and-bash/",
"original": "http://labs.opendns.com/2014/10/02/opendns-and-bash/",
"path": "/2014/10/02/opendns-and-bash/",
"domain": "druigvsegdaryadom.ir",
"extension": "php",
"full": "http://druigvsegdaryadom.ir/index.php",
"original": "http://druigvsegdaryadom.ir/index.php",
"path": "/index.php",
"scheme": "http"
}
}
}
}
}
2 changes: 1 addition & 1 deletion packages/ti_misp/manifest.yml
View file
@@ -1,6 +1,6 @@
name: ti_misp
title: MISP
version: "1.27.0"
version: "1.27.1"
description: Ingest threat intelligence indicators from MISP platform with Elastic Agent.
type: integration
format_version: "3.0.0"
Expand Down

0 comments on commit 2f0c43f

Please sign in to comment.