[mimecast] - update package-spec to 2.9.0 (#7165)

- Ensure email.from.address is an array - Ensure email.to.address is an array [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -ecs-version=8.8.0 -ecs-git-ref=v8.8.0 -format-version=2.9.0 packages/mimecast
elastic · Jul 28, 2023 · 0c173b0 · 0c173b0
1 parent 4e64865
commit 0c173b0
Show file tree

Hide file tree

Showing 26 changed files with 5,637 additions and 5,562 deletions.
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.13.0"
+  changes:
+    - description: Update package-spec to 2.9.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7165
 - version: "1.12.0"
   changes:
     - description: Document valid duration units.

diff --git a/packages/mimecast/data_stream/audit_events/sample_event.json b/packages/mimecast/data_stream/audit_events/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-11-16T12:01:37.000Z",
     "agent": {
-        "ephemeral_id": "318ed660-ab02-48f6-bd87-53b29acaedab",
-        "id": "8c5473c5-468b-444c-b5c0-0783fde1f55e",
+        "ephemeral_id": "67b65934-b452-4461-a076-c9b053b6da1f",
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.5.1"
+        "version": "8.9.0"
     },
     "data_stream": {
         "dataset": "mimecast.audit_events",
@@ -16,18 +16,18 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "8c5473c5-468b-444c-b5c0-0783fde1f55e",
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
         "snapshot": false,
-        "version": "8.5.1"
+        "version": "8.9.0"
     },
     "event": {
         "action": "search-action",
         "agent_id_status": "verified",
-        "created": "2023-01-16T22:59:08.657Z",
+        "created": "2023-07-27T14:56:43.376Z",
         "dataset": "mimecast.audit_events",
-        "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o",
-        "ingested": "2023-01-16T22:59:09Z",
-        "original": "{\"auditType\":\"Search Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK1o\",\"user\":\"johndoe@example.com\"}"
+        "id": "eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK2o",
+        "ingested": "2023-07-27T14:56:44Z",
+        "original": "{\"auditType\":\"Search Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Inspected Review Set Messages - Source: Review Set - Supervision - hot words, Case - GDPR/CCPA, Message Status: Pending, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSSg_xMDJPNkisSDdISQ00j0gzz44wDAtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkoaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAAjKK2o\",\"user\":\"johndoe@example.com\"}"
     },
     "input": {
         "type": "httpjson"

diff --git a/packages/mimecast/data_stream/dlp_logs/sample_event.json b/packages/mimecast/data_stream/dlp_logs/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-11-18T21:41:18.000Z",
     "agent": {
-        "ephemeral_id": "f4dde373-2ff7-464b-afdb-da94763f219b",
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
+        "ephemeral_id": "b3630060-e536-4953-a9b4-74f78c6ac6c1",
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.6.0"
+        "version": "8.9.0"
     },
     "data_stream": {
         "dataset": "mimecast.dlp_logs",
@@ -16,9 +16,9 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
-        "snapshot": true,
-        "version": "8.6.0"
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
+        "snapshot": false,
+        "version": "8.9.0"
     },
     "email": {
         "direction": "inbound",
@@ -27,7 +27,7 @@
                 "\u003c\u003e"
             ]
         },
-        "message_id": "\u003c20211118214115.B346F10021D@mail.emailsec.ninja\u003e",
+        "message_id": "\u003c20211118214115.B346F10021D-2@mail.emailsec.ninja\u003e",
         "subject": "Undelivered Mail Returned to Sender",
         "to": {
             "address": [
@@ -40,8 +40,8 @@
         "agent_id_status": "verified",
         "created": "2021-11-18T21:41:18+0000",
         "dataset": "mimecast.dlp_logs",
-        "ingested": "2023-01-13T15:05:15Z",
-        "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}"
+        "ingested": "2023-07-27T14:57:41Z",
+        "original": "{\"action\":\"notification\",\"eventTime\":\"2021-11-18T21:41:18+0000\",\"messageId\":\"\\u003c20211118214115.B346F10021D-2@mail.emailsec.ninja\\u003e\",\"policy\":\"Content Inspection - Watermark\",\"recipientAddress\":\"johndoe@example.com\",\"route\":\"inbound\",\"senderAddress\":\"\\u003c\\u003e\",\"subject\":\"Undelivered Mail Returned to Sender\"}"
     },
     "input": {
         "type": "httpjson"

diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json
@@ -56,7 +56,9 @@
                 },
                 "local_id": "29be076e-44cd-354d-a7c2-083d4a312371",
                 "to": {
-                    "address": "johndoe@example.com"
+                    "address": [
+                        "johndoe@example.com"
+                    ]
                 }
             },
             "error": {
@@ -158,7 +160,9 @@
                 "local_id": "61dfe7da-4c6d-34e1-9667-69b04f0d564f",
                 "message_id": "\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e",
                 "to": {
-                    "address": "johndoejr@example.com"
+                    "address": [
+                        "johndoejr@example.com"
+                    ]
                 }
             },
             "event": {
@@ -218,7 +222,9 @@
                 "message_id": "\u003c137188507-1634623494888@uk-mta-151.uk.mimecast.lan\u003e",
                 "subject": "You have new held messages",
                 "to": {
-                    "address": "johndoejr@example.com"
+                    "address": [
+                        "johndoejr@example.com"
+                    ]
                 }
             },
             "error": {
@@ -278,7 +284,9 @@
                 },
                 "local_id": "CYSuuaBUMjOpk3k1Xhvy_Q",
                 "to": {
-                    "address": "o365_service_account@example.com"
+                    "address": [
+                        "o365_service_account@example.com"
+                    ]
                 }
             },
             "event": {
@@ -312,7 +320,9 @@
                 "message_id": "\u003c140943948-1636373419265@uk-mta-286.uk.mimecast.lan\u003e",
                 "subject": "You have new held messages",
                 "to": {
-                    "address": "johndoejr@example.com"
+                    "address": [
+                        "johndoejr@example.com"
+                    ]
                 }
             },
             "event": {
@@ -358,7 +368,9 @@
                 },
                 "subject": "DocuSign- Contract #45576744333",
                 "to": {
-                    "address": "aorchard@twotoeight.com"
+                    "address": [
+                        "aorchard@twotoeight.com"
+                    ]
                 }
             },
             "event": {
@@ -413,7 +425,9 @@
                 "message_id": "\u003c8182967832.4@biz.net\u003e",
                 "subject": "Totally not a scam! (Honest)",
                 "to": {
-                    "address": "big.wig@biz.com"
+                    "address": [
+                        "big.wig@biz.com"
+                    ]
                 }
             },
             "event": {

diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml
@@ -57,15 +57,11 @@ processors:
       field: mimecast.MsgId
       target_field: email.message_id
       ignore_missing: true
-  - rename:
-      field: mimecast.Rcpt
-      target_field: email.to.address
-      ignore_missing: true
   - append:
       field: email.to.address
-      value: "{{{mimecast.Rcpt}}}"
+      value: '{{{mimecast.Rcpt}}}'
       allow_duplicates: false
-      if: "ctx?.mimecast?.Rcpt != null"
+      if: ctx.mimecast?.Rcpt != null
   - append:
       field: email.from.address
       value: '{{{mimecast.headerFrom}}}'
@@ -139,14 +135,11 @@ processors:
       field: mimecast.md5
       target_field: email.attachments.file.hash.md5
       ignore_missing: true
-  - rename:
-      field: mimecast.Recipient
-      target_field: email.to.address
-      ignore_missing: true  
   - append:
       field: email.to.address
-      value: "{{{mimecast.Recipient}}}"
-      if: "ctx?.mimecast?.Recipient != null"
+      value: '{{{mimecast.Recipient}}}'
+      allow_duplicates: false
+      if: ctx.mimecast?.Recipient != null
   - rename:
       field: mimecast.SenderDomain
       target_field: source.domain
@@ -195,14 +188,11 @@ processors:
       field: mimecast.reason
       target_field: event.reason
       ignore_missing: true
-  - rename:
-      field: mimecast.recipient
-      target_field: email.to.address
-      ignore_missing: true
   - append:
       field: email.to.address
-      value: "{{{mimecast.recipient}}}"
-      if: "ctx?.mimecast?.recipient != null"
+      value: '{{{mimecast.recipient}}}'
+      allow_duplicates: false
+      if: ctx.mimecast?.recipient != null
   - rename:
       field: mimecast.route
       target_field: email.direction
@@ -310,16 +300,6 @@ processors:
       field: source.as.organization_name
       target_field: source.as.organization.name
       ignore_missing: true
-  - dissect:
-       field: email.from.address
-       pattern: "<%{email.from.address}>"
-       ignore_missing: true
-       ignore_failure: true
-  - dissect:
-       field: email.to.address
-       pattern: "<%{email.to.address}>"
-       ignore_missing: true
-       ignore_failure: true
 
   # Cleanup
   - remove:
@@ -332,6 +312,8 @@ processors:
         - mimecast.log_type_part1
         - mimecast.log_type_part2
         - mimecast.log_type_parts
+        - mimecast.recipient
+        - mimecast.Rcpt
         - mimecast.sender
         - mimecast.Sender
       ignore_missing: true

diff --git a/packages/mimecast/data_stream/siem_logs/sample_event.json b/packages/mimecast/data_stream/siem_logs/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-11-12T12:15:46.000Z",
     "agent": {
-        "ephemeral_id": "f4dde373-2ff7-464b-afdb-da94763f219b",
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
+        "ephemeral_id": "c6e5221f-b305-4a75-acb4-7a43547a1e6d",
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.6.0"
+        "version": "8.9.0"
     },
     "data_stream": {
         "dataset": "mimecast.siem_logs",
@@ -16,9 +16,9 @@
         "version": "8.8.0"
     },
     "elastic_agent": {
-        "id": "5d3eee86-91a9-4afa-af92-c6b79bd866c0",
-        "snapshot": true,
-        "version": "8.6.0"
+        "id": "c0ee214c-57e5-4a60-80ba-e4dc247eb02e",
+        "snapshot": false,
+        "version": "8.9.0"
     },
     "email": {
         "direction": "internal",
@@ -29,14 +29,16 @@
         },
         "local_id": "fjihpfEgM_iRwemxhe3t_w",
         "to": {
-            "address": "o365_service_account@example.com"
+            "address": [
+                "o365_service_account@example.com"
+            ]
         }
     },
     "event": {
         "agent_id_status": "verified",
         "created": "2021-11-12T12:15:46+0000",
         "dataset": "mimecast.siem_logs",
-        "ingested": "2023-01-13T15:06:00Z",
+        "ingested": "2023-07-27T14:59:24Z",
         "original": "{\"Content-Disposition\":\"attachment; filename=\\\"jrnl_20211018093329655.json\\\"\",\"Dir\":\"Internal\",\"Rcpt\":\"o365_service_account@example.com\",\"RcptActType\":\"Jnl\",\"RcptHdrType\":\"Unknown\",\"Sender\":\"johndoe@example.com\",\"aCode\":\"fjihpfEgM_iRwemxhe3t_w\",\"acc\":\"ABC123\",\"datetime\":\"2021-11-12T12:15:46+0000\"}",
         "outcome": "unknown"
     },