[ti_misp] Fix parsing of threat event publish_timestamp (#6575)

* Fix values in the pipeline. * Update the pipeline tests.
elastic · Jun 15, 2023 · 284f4d5 · 284f4d5
1 parent 2308873
commit 284f4d5
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 47 deletions.
diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.15.4"
+  changes:
+    - description: Fix parsing of threat event publish_timestamp.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/6575
 - version: "1.15.3"
   changes:
     - description: Fix bug where the threat_attributes data stream would not stop paginating after an empty response.

diff --git a/..._misp/data_stream/threat/_dev/test/pipeline/test-misp-attributes-ndjson.log-expected.json b/..._misp/data_stream/threat/_dev/test/pipeline/test-misp-attributes-ndjson.log-expected.json
@@ -39,7 +39,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -116,7 +116,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 2,
@@ -193,7 +193,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 2,
@@ -268,7 +268,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 2,
@@ -341,7 +341,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 2,
@@ -434,7 +434,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -540,7 +540,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -645,7 +645,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -756,7 +756,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -867,7 +867,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -959,7 +959,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 3,
@@ -1046,7 +1046,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 3,
@@ -1139,7 +1139,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 3,
@@ -1231,7 +1231,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1319,7 +1319,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1407,7 +1407,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1494,7 +1494,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1587,7 +1587,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1697,7 +1697,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1803,7 +1803,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -1909,7 +1909,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2015,7 +2015,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2121,7 +2121,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2226,7 +2226,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2337,7 +2337,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2448,7 +2448,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -2500,4 +2500,4 @@
             }
         }
     ]
-}
+}
diff --git a/...ges/ti_misp/data_stream/threat/_dev/test/pipeline/test-misp-long-ndjson.log-expected.json b/...ges/ti_misp/data_stream/threat/_dev/test/pipeline/test-misp-long-ndjson.log-expected.json
@@ -58,7 +58,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -159,7 +159,7 @@
                     "org_id": "1",
                     "orgc_id": "1",
                     "proposal_email_lock": false,
-                    "publish_timestamp": "0",
+                    "publish_timestamp": "1970-01-01T00:00:00.000Z",
                     "published": false,
                     "sharing_group_id": "0",
                     "threat_level_id": 1,
@@ -210,4 +210,4 @@
             }
         }
     ]
-}
+}