carrier mail functionality

Implements processing of a carrier mail that contains email attachments. Each email attachment is converted into an individual MISP event.

mail2misp/mail2misp.py

@@ -4,6 +4,7 @@
 import re
 import syslog
 import html
+import os
 from io import BytesIO
 from ipaddress import ip_address
 from email import message_from_bytes, policy, message
@@ -50,15 +51,19 @@ def __init__(self, misp_url, misp_key, verifycert, config, offline=False, urlson
     def load_email(self, pseudofile):
         self.pseudofile = pseudofile
         self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
-        self.subject = self.original_mail.get('Subject')
+
         try:
             self.sender = self.original_mail.get('From')
         except:
             self.sender = "<unknown sender>"
-
-        # Remove words from subject
-        for removeword in self.config.removelist:
-            self.subject = re.sub(removeword, "", self.subject).strip()
+
+        try:
+            self.subject = self.original_mail.get('Subject')
+            # Remove words from subject
+            for removeword in self.config.removelist:
+                self.subject = re.sub(removeword, "", self.subject).strip()
+        except:
+            self.subject = "<subject could not be retrieved>"
 
         # Initialize the MISP event
         self.misp_event = MISPEvent()
@@ -399,3 +404,23 @@ def add_event(self):
             for value, source in self.sightings_to_add:
                 self.sighting(value, source)
         return event
+
+    def get_attached_emails(self,pseudofile):
+
+        syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
+        syslog.syslog("get_attached_emails Job started.")
+
+        forwarded_emails = []
+        self.pseudofile = pseudofile
+        self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
+        for attachment in self.original_mail.iter_attachments():
+            attachment_content = attachment.get_content()
+            filename = attachment.get_filename()
+            syslog.syslog(f'get_attached_emails: filename = {filename}')
+            # Search for email forwarded as attachment
+            # I could have more than one, attaching everything.
+            if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml':
+                # all attachments are identified as message.EmailMessage so filtering on extension for now.
+                forwarded_emails.append(BytesIO(attachment_content))
+        return forwarded_emails
+

mail_to_misp.py

@@ -33,6 +33,7 @@
             misp_key = config.misp_key
             misp_verifycert = config.misp_verifycert
             debug = config.debug
+            ignore_carrier_mail = config.ignore_carrier_mail
         except Exception as e:
             syslog.syslog(str(e))
             print("There is a problem with the configuration. A mandatory configuration variable is not set.")
@@ -55,19 +56,48 @@
         # receive data and subject through arguments
         raise Exception('This is not implemented anymore.')
 
+    syslog.syslog("About to create a mail2misp object.")
     mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
-    mail2misp.load_email(pseudofile)
-
-    if debug:
-        syslog.syslog(f'Working on {mail2misp.subject}')
-
-    if args.trap or config.spamtrap:
-        mail2misp.email_from_spamtrap()
+    attached_emails = mail2misp.get_attached_emails(pseudofile)
+    syslog.syslog(f"found {len(attached_emails)} attached emails")
+    if ignore_carrier_mail and len(attached_emails) !=0:
+        syslog.syslog("Ignoring the carrier mail.")
+        while len(attached_emails) !=0:
+            pseudofile = attached_emails.pop()
+            #Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment
+            mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
+            mail2misp.load_email(pseudofile)
+
+            if debug:
+                syslog.syslog(f'Working on {mail2misp.subject}')
+
+            if args.trap or config.spamtrap:
+                mail2misp.email_from_spamtrap()
+            else:
+                mail2misp.process_email_body()
+
+            mail2misp.process_body_iocs()
+
+            if not args.event:
+                mail2misp.add_event()
+
+        syslog.syslog("Job finished.")
     else:
-        mail2misp.process_email_body()
+        syslog.syslog("Running standard mail2misp")
+        mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
+        mail2misp.load_email(pseudofile)
+
+        if debug:
+            syslog.syslog(f'Working on {mail2misp.subject}')
 
-    mail2misp.process_body_iocs()
+        if args.trap or config.spamtrap:
+            mail2misp.email_from_spamtrap()
+        else:
+            mail2misp.process_email_body()
+
+        mail2misp.process_body_iocs()
+
+        if not args.event:
+            mail2misp.add_event()
+        syslog.syslog("Job finished.")
 
-    if not args.event:
-        mail2misp.add_event()
-    syslog.syslog("Job finished.")

mail_to_misp_config.py-example

@@ -18,6 +18,7 @@ debug = False
 nameservers = ['149.13.33.69']
 email_subject_prefix = 'M2M'
 attach_original_mail = False
+ignore_carrier_mail = False
 
 excludelist = ('google.com', 'microsoft.com')
 externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')