[RecordedFuture] Add dashboards and list field (#9471)

* Add list field and dashboards

packages/ti_recordedfuture/_dev/deploy/docker/sample_logs/rf_file_default.csv

@@ -0,0 +1,3 @@
+"Name","Algorithm","Risk","RiskString","EvidenceDetails"
+"63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f","SHA-256","75","2/17","{""EvidenceDetails"": [{""Name"": ""linkedToMalware"", ""EvidenceString"": ""2 sightings on 1 source: PolySwarm. Most recent link (Mar 23, 2024): https://polyswarm.network/scan/results/file/63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f"", ""CriticalityLabel"": ""Suspicious"", ""MitigationString"": """", ""Rule"": ""Linked to Malware"", ""SourcesCount"": 1.0, ""Sources"": [""source:doLlw5""], ""Timestamp"": ""2024-03-23T17:10:20.642Z"", ""SightingsCount"": 2.0, ""Criticality"": 2.0}, {""Name"": ""positiveMalwareVerdict"", ""EvidenceString"": ""3 sightings on 3 sources: Polyswarm Sandbox Analysis, Recorded Future Triage Malware Analysis, PolySwarm. Most recent link (Mar 23, 2024): https://polyswarm.network/scan/results/file/63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f"", ""CriticalityLabel"": ""Malicious"", ""MitigationString"": """", ""Rule"": ""Positive Malware Verdict"", ""SourcesCount"": 3.0, ""Sources"": [""source:hzRhwZ"", ""source:ndy5_2"", ""source:doLlw5""], ""Timestamp"": ""2024-03-23T16:36:02.000Z"", ""SightingsCount"": 3.0, ""Criticality"": 3.0}]}"
+"ea0e23a30aa252e4c8a7a8cb92ac2bba1ffcb4e94027bca555798b2920f9bfab","SHA-256","75","2/17","{""EvidenceDetails"": [{""Name"": ""linkedToMalware"", ""EvidenceString"": ""10 sightings on 1 source: PolySwarm. Most recent link (Mar 24, 2024): https://polyswarm.network/scan/results/file/ea0e23a30aa252e4c8a7a8cb92ac2bba1ffcb4e94027bca555798b2920f9bfab"", ""CriticalityLabel"": ""Suspicious"", ""MitigationString"": """", ""Rule"": ""Linked to Malware"", ""SourcesCount"": 1.0, ""Sources"": [""source:doLlw5""], ""Timestamp"": ""2024-03-24T21:22:00.282Z"", ""SightingsCount"": 10.0, ""Criticality"": 2.0}, {""Name"": ""positiveMalwareVerdict"", ""EvidenceString"": ""31 sightings on 3 sources: Recorded Future Sandbox, Polyswarm Sandbox Analysis, PolySwarm. Malware sandbox report for ea0e23a30aa252e4c8a7a8cb92ac2bba1ffcb4e94027bca555798b2920f9bfab on March 27, 2024.   Score: 10 (Known bad). Detections: njRAT. Contains: 6 ATT\\u0026CK behaviors, 1 command and control indicator, and 12 signatures. Most recent link (Mar 24, 2024): https://polyswarm.network/scan/results/file/ea0e23a30aa252e4c8a7a8cb92ac2bba1ffcb4e94027bca555798b2920f9bfab"", ""CriticalityLabel"": ""Malicious"", ""MitigationString"": """", ""Rule"": ""Positive Malware Verdict"", ""SourcesCount"": 3.0, ""Sources"": [""source:oWAWVb"", ""source:hzRhwZ"", ""source:doLlw5""], ""Timestamp"": ""2024-03-24T20:33:10.000Z"", ""SightingsCount"": 31.0, ""Criticality"": 3.0}]}"

packages/ti_recordedfuture/_dev/deploy/docker/sample_logs/rf_url_default.csv

@@ -8,3 +8,4 @@
 "https://www.jeanninecatddns.chickenkiller.com/signin-authflow","75","3/24","{""EvidenceDetails"": [{""Rule"": ""Recently Active URL on Weaponized Domain"", ""CriticalityLabel"": ""Malicious"", ""EvidenceString"": ""1 sighting on 1 source: Recorded Future Domain Analysis URLs. Service provider: Afraid.org. Behavior observed: Malware Distribution, Phishing Techniques. Last observed on Dec 28, 2021."", ""Sources"": [""report:aRJ1CU""], ""Timestamp"": ""2021-12-28T22:15:49.631Z"", ""Name"": ""recentWeaponizedURL"", ""MitigationString"": """", ""Criticality"": 3.0}, {""Rule"": ""Recently Detected Phishing Techniques"", ""CriticalityLabel"": ""Malicious"", ""EvidenceString"": ""2 sightings on 2 sources: Bitdefender, Urlscan.io. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Dec 28, 2021."", ""Sources"": [""d3Awkm"", ""eKv4Jm""], ""Timestamp"": ""2021-12-28T00:00:00.000Z"", ""Name"": ""recentPhishingSiteDetected"", ""MitigationString"": """", ""Criticality"": 3.0}, {""Rule"": ""Recently Detected Malware Distribution"", ""CriticalityLabel"": ""Malicious"", ""EvidenceString"": ""1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Dec 28, 2021."", ""Sources"": [""d3Awkm""], ""Timestamp"": ""2021-12-28T00:00:00.000Z"", ""Name"": ""recentMalwareSiteDetected"", ""MitigationString"": """", ""Criticality"": 3.0}]}"
 "http://coollab.jp/dir/root/p/09908.js","75","3/24","{""EvidenceDetails"": [{""Rule"": ""Historically Reported as a Defanged URL"", ""CriticalityLabel"": ""Unusual"", ""EvidenceString"": ""24 sightings on 9 sources including: Malware News - Malware Analysis, News and Indicators, microsoft.com, sociabble.com, 4-traders.com, MarketScreener.com | Stock Market News. Most recent link (Aug 13, 2021): https://www.marketscreener.com/quote/stock/MICROSOFT-CORPORATION-4835/news/Microsoft-Attackers-use-Morse-code-other-encryption-methods-in-evasive-phishing-campaign-36161110/?utm_medium=RSS&utm_content=20210813"", ""Sources"": [""gBDK5G"", ""idn:microsoft.com"", ""idn:sociabble.com"", ""KBTQ2e"", ""dCotni"", ""g9rk5F"", ""Z7kln5"", ""idn:cda.ms"", ""idn:thewindowsupdate.com""], ""Timestamp"": ""2021-08-13T17:03:19.000Z"", ""Name"": ""defangedURL"", ""MitigationString"": """", ""Criticality"": 1.0}, {""Rule"": ""Historically Detected Malware Distribution"", ""CriticalityLabel"": ""Unusual"", ""EvidenceString"": ""1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Aug 13, 2021."", ""Sources"": [""d3Awkm""], ""Timestamp"": ""2021-08-13T00:00:00.000Z"", ""Name"": ""malwareSiteDetected"", ""MitigationString"": """", ""Criticality"": 1.0}, {""Rule"": ""Recently Reported by Insikt Group"", ""CriticalityLabel"": ""Malicious"", ""EvidenceString"": ""1 sighting on 1 source: Insikt Group. 1 report: Microsoft Warns of Attacks Targeting Microsoft Office 365 Users. Most recent link (Aug 12, 2021): https://app.recordedfuture.com/live/sc/4BBhpn1ApBQR"", ""Sources"": [""VKz42X""], ""Timestamp"": ""2021-08-12T00:00:00.000Z"", ""Name"": ""recentAnalystNote"", ""MitigationString"": """", ""Criticality"": 3.0}]}"
 "https://blog.br0vvnn.io","75","3/24","{""EvidenceDetails"": [{""Rule"": ""Historically Reported as a Defanged URL"", ""CriticalityLabel"": ""Unusual"", ""EvidenceString"": ""23 sightings on 9 sources including: The Official Google Blog, eccouncil.org, frsecure.com, SoyaCincau, PasteBin. Most recent tweet: Actor controlled sites and accounts Research Blog https://blog.br0vvnn[.]io. Most recent link (Jan 27, 2021): https://twitter.com/techn0m4nc3r/statuses/1354296736357953539"", ""Sources"": [""Gzt"", ""idn:eccouncil.org"", ""idn:frsecure.com"", ""J-8-Nr"", ""Jv_xrR"", ""g9rk5F"", ""cUg0pv"", ""K5LKj8"", ""fVAueu""], ""Timestamp"": ""2021-01-27T05:14:38.000Z"", ""Name"": ""defangedURL"", ""MitigationString"": """", ""Criticality"": 1.0}, {""Rule"": ""Historically Detected Phishing Techniques"", ""CriticalityLabel"": ""Unusual"", ""EvidenceString"": ""1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on May 30, 2021."", ""Sources"": [""d3Awkm""], ""Timestamp"": ""2021-05-30T00:00:00.000Z"", ""Name"": ""phishingSiteDetected"", ""MitigationString"": """", ""Criticality"": 1.0}, {""Rule"": ""Recently Reported by Insikt Group"", ""CriticalityLabel"": ""Malicious"", ""EvidenceString"": ""1 sighting on 1 source: Insikt Group. 1 report: Google Warns of Ongoing Attacks Targeting Security Researchers. Most recent link (Jan 25, 2021): https://app.recordedfuture.com/live/sc/5QCqZ2ZH4lwc"", ""Sources"": [""VKz42X""], ""Timestamp"": ""2021-01-25T00:00:00.000Z"", ""Name"": ""recentAnalystNote"", ""MitigationString"": """", ""Criticality"": 3.0}]}"
+"http://cleaning.homesecuritypc.com/packages/fduphhq_bzaeudor.bmp","66","3/35","{""EvidenceDetails"":""{""EvidenceDetails"": [{""Name"": ""malwareSiteDetected"", ""EvidenceString"": ""9 sightings on 1 source: External Sensor Data Analysis. http://cleaning.homesecuritypc.com/packages/fduphhq_bzaeudor.bmp is observed to be a malware site URL that navigates to malicious content including executables, drive-by infection sites, malicious scripts, viruses, trojans, or code."", ""CriticalityLabel"": ""Unusual"", ""MitigationString"": """", ""Rule"": ""Historically Detected Malware Distribution"", ""SourcesCount"": 1.0, ""Sources"": [""source:kBB1fk""], ""Timestamp"": ""2022-07-20T20:40:11.605Z"", ""SightingsCount"": 9.0, ""Criticality"": 1.0}, {""Name"": ""botnetUrl"", ""EvidenceString"": ""8 sightings on 1 source: External Sensor Data Analysis. http://cleaning.homesecuritypc.com/packages/fduphhq_bzaeudor.bmp is observed to be a botnet URL from which network attacks are launched. Attacks may include SPAM messages, DOS, SQL injections, proxy jacking, and other unsolicited contacts."", ""CriticalityLabel"": ""Unusual"", ""MitigationString"": """", ""Rule"": ""Historically Reported Botnet URL"", ""SourcesCount"": 1.0, ""Sources"": [""source:kBB1fk""], ""Timestamp"": ""2022-07-20T20:40:11.605Z"", ""SightingsCount"": 8.0, ""Criticality"": 1.0}, {""Name"": ""recentWeaponizedURL"", ""EvidenceString"": ""1 sighting on 1 source: Recorded Future Domain Analysis URLs. Service provider: No-IP. Behavior observed: Malware Distribution, Phishing Techniques. Last observed on Jan 12, 2024."", ""CriticalityLabel"": ""Malicious"", ""MitigationString"": """", ""Rule"": ""Recently Active URL on Weaponized Domain"", ""SourcesCount"": 1.0, ""Sources"": [""report:aRJ1CU""], ""Timestamp"": ""2024-03-28T11:46:43.868Z"", ""SightingsCount"": 1.0, ""Criticality"": 3.0}]}"

packages/ti_recordedfuture/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.0"
+  changes:
+    - description: Add dashboards and list field
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9471
 - version: "1.22.0"
   changes:
     - description: Set sensitive values as secret.

packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-csv-file-config.yml

@@ -4,3 +4,5 @@ data_stream:
   vars:
     paths:
       - "{{SERVICE_LOGS_DIR}}/*.csv"
+assert:
+  hit_count: 12

packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-fusion-file-download-config.yml

@@ -5,6 +5,7 @@ data_stream:
   vars:
     interval: 1m
     api_token: test-token
+    list: test
     custom_url: http://{{Hostname}}:{{Port}}/v2/fusion/files/?path=%2Fpublic%2Ftest-ip.csv
     preserve_original_event: true
     enable_request_tracer: true

packages/ti_recordedfuture/data_stream/threat/agent/stream/httpjson.yml.hbs

@@ -24,11 +24,15 @@ request.transforms:
     value: {{ api_token }}
 {{/if}}
 response.decode_as: text/csv
+fields_under_root: true
+fields:
+  _conf:
+    list: '{{list}}'
 tags:
 {{#if preserve_original_event}}
   - preserve_original_event
 {{/if}}
-{{#each tags as |tag i|}}
+{{#each tags as |tag|}}
   - {{tag}}
 {{/each}}
 {{#contains "forwarded" tags}}

packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml

@@ -237,6 +237,10 @@ processors:
   - rename:
       field: json
       target_field: recordedfuture
+  - rename:
+      target_field: recordedfuture.list
+      field: _conf.list
+      if: ctx._conf?.list != null
 
 #
 # Cleanup
@@ -253,6 +257,7 @@ processors:
         - recordedfuture.Name
         - recordedfuture.Risk
         - _temp_
+        - _conf
       ignore_missing: true
 on_failure:
   - set:

packages/ti_recordedfuture/data_stream/threat/fields/fields.yml

@@ -19,6 +19,11 @@
       description: >
         Details of risk rules observed.
 
+    - name: list
+      type: keyword
+      description: >
+        User-configured risklist.
+
 - name: labels.is_ioc_transform_source
   type: constant_keyword
   value: "true"

packages/ti_recordedfuture/data_stream/threat/sample_event.json

@@ -1,11 +1,11 @@
 {
-    "@timestamp": "2023-08-29T13:05:30.615Z",
+    "@timestamp": "2024-03-29T13:00:04.736Z",
     "agent": {
-        "ephemeral_id": "4d3f7527-f999-48d2-920c-3ec5a0b34414",
-        "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
+        "ephemeral_id": "fe05693b-59ec-47c6-9d5e-b0ef7c71ee65",
+        "id": "bc94f76a-cdb2-4211-9412-c5d6c5711711",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.9.1"
+        "version": "8.12.1"
     },
     "data_stream": {
         "dataset": "ti_recordedfuture.threat",
@@ -16,19 +16,19 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
+        "id": "bc94f76a-cdb2-4211-9412-c5d6c5711711",
         "snapshot": false,
-        "version": "8.9.1"
+        "version": "8.12.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
         "dataset": "ti_recordedfuture.threat",
-        "ingested": "2023-08-29T13:05:31Z",
+        "ingested": "2024-03-29T13:00:14Z",
         "kind": "enrichment",
-        "risk_score": 87,
+        "risk_score": 75,
         "timezone": "+00:00",
         "type": [
             "indicator"
@@ -39,60 +39,45 @@
     },
     "log": {
         "file": {
-            "path": "/tmp/service_logs/rf_url_default.csv"
+            "path": "/tmp/service_logs/rf_file_default.csv"
         },
-        "offset": 45
+        "offset": 57
     },
     "recordedfuture": {
         "evidence_details": [
             {
-                "Criticality": 1,
-                "CriticalityLabel": "Unusual",
-                "EvidenceString": "66 sightings on 22 sources including: Ars Technica, fook.news, urdupresss.com, HackDig Posts, apple.news. Most recent link (Jul 20, 2021): https://techsecuritenews.com/solarwinds-pirates-utilisent-nouvelle-faille-zero-day-attaques/",
+                "Criticality": 2,
+                "CriticalityLabel": "Suspicious",
+                "EvidenceString": "2 sightings on 1 source: PolySwarm. Most recent link (Mar 23, 2024): https://polyswarm.network/scan/results/file/63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f",
                 "MitigationString": "",
-                "Name": "defangedURL",
-                "Rule": "Historically Reported as a Defanged URL",
+                "Name": "linkedToMalware",
+                "Rule": "Linked to Malware",
+                "SightingsCount": 2,
                 "Sources": [
-                    "Ctq",
-                    "idn:fook.news",
-                    "idn:urdupresss.com",
-                    "POs2u-",
-                    "idn:apple.news",
-                    "idn:cryptoinfoos.com.ng",
-                    "g9rk5F",
-                    "idn:thewindowsupdate.com",
-                    "idn:nationalcybersecuritynews.today",
-                    "gBDK5G",
-                    "idn:microsoft.com",
-                    "idn:techsecuritenews.com",
-                    "idn:mblogs.info",
-                    "J6UzbO",
-                    "idn:viralamo.com",
-                    "idn:sellorbuyhomefast.com",
-                    "idn:crazyboy.tech",
-                    "idn:times24h.com",
-                    "idn:buzzfeeg.com",
-                    "idn:dsmenders.com",
-                    "WroSbs",
-                    "idn:vzonetvgh.com"
+                    "source:doLlw5"
                 ],
-                "Timestamp": "2021-07-20T00:00:00.000Z"
+                "SourcesCount": 1,
+                "Timestamp": "2024-03-23T17:10:20.642Z"
             },
             {
                 "Criticality": 3,
                 "CriticalityLabel": "Malicious",
-                "EvidenceString": "1 sighting on 1 source: Insikt Group. 1 report: SolarWinds Fixes Critical Vulnerability in Serv-U Managed File Transfer and Secure FTP Products. Most recent link (Jul 10, 2021): https://app.recordedfuture.com/live/sc/1GnDrn8zigTd",
+                "EvidenceString": "3 sightings on 3 sources: Polyswarm Sandbox Analysis, Recorded Future Triage Malware Analysis, PolySwarm. Most recent link (Mar 23, 2024): https://polyswarm.network/scan/results/file/63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f",
                 "MitigationString": "",
-                "Name": "recentAnalystNote",
-                "Rule": "Recently Reported by Insikt Group",
+                "Name": "positiveMalwareVerdict",
+                "Rule": "Positive Malware Verdict",
+                "SightingsCount": 3,
                 "Sources": [
-                    "VKz42X"
+                    "source:hzRhwZ",
+                    "source:ndy5_2",
+                    "source:doLlw5"
                 ],
-                "Timestamp": "2021-07-10T00:00:00.000Z"
+                "SourcesCount": 3,
+                "Timestamp": "2024-03-23T16:36:02.000Z"
             }
         ],
-        "name": "http://144.34.179.162/a",
-        "risk_string": "2/24"
+        "name": "63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f",
+        "risk_string": "2/17"
     },
     "tags": [
         "forwarded",
@@ -103,21 +88,17 @@
             "name": "Recorded Future"
         },
         "indicator": {
+            "file": {
+                "hash": {
+                    "sha256": "63212aa8c94098a844945ed1611389b2e1c9dc3906a5ba9d7d0d320344213f4f"
+                }
+            },
             "provider": [
-                "Ars Technica",
-                "fook.news",
-                "urdupresss.com",
-                "HackDig Posts",
-                "apple.news",
-                "Insikt Group"
+                "PolySwarm",
+                "Polyswarm Sandbox Analysis",
+                "Recorded Future Triage Malware Analysis"
             ],
-            "type": "url",
-            "url": {
-                "domain": "144.34.179.162",
-                "original": "http://144.34.179.162/a",
-                "path": "/a",
-                "scheme": "http"
-            }
+            "type": "file"
         }
     }
-}
+}