[ParseEmailFilesV2] S/MIME files without to/from/subject fields

Packs/CommonScripts/.secrets-ignore

@@ -111,4 +111,9 @@ http://www.meow.com
 https://www.testhxxp.com
 https://www.meow.com
 701393b3b8e6ae6e70effcda7598a8cf92d0adb1aaeb5aa91c73004519644801
-uuid.uuid5(SCO_DET_ID_NAMESPACE
+uuid.uuid5(SCO_DET_ID_NAMESPACE
+testing@gmail.com
+a853a1b0-1ffe-4e37-d9a9-a27c6bc0bd5b@gmail.com
+cipher=ECDHE-RSA-AES128-GCM-SHA256
+07.23.05.38
+multipart/signed

Packs/CommonScripts/ReleaseNotes/1_12_6.md

@@ -0,0 +1,6 @@
+
+#### Scripts
+
+##### ParseEmailFilesV2
+
+- Fixed an issue where attachments were not being saved to the war room when uploading S/MIME files that lacked the To/From/Subject fields.

Packs/CommonScripts/Scripts/ParseEmailFilesV2/ParseEmailFilesV2.py

@@ -20,7 +20,7 @@ def data_to_md(email_data, email_file_name=None, parent_email_file=None, print_o
     if email_data is None:
         return 'No data extracted from email'
 
-    md = u"### Results:\n"
+    md = "### Results:\n"
     if email_file_name:
         md = f"### {email_file_name}\n"
 
@@ -30,18 +30,18 @@ def data_to_md(email_data, email_file_name=None, parent_email_file=None, print_o
     if parent_email_file:
         md += f"### Containing email: {parent_email_file}\n"
 
-    md += u"* {0}:\t{1}\n".format('From', email_data.get('From') or "")
-    md += u"* {0}:\t{1}\n".format('To', email_data.get('To') or "")
-    md += u"* {0}:\t{1}\n".format('CC', email_data.get('CC') or "")
-    md += u"* {0}:\t{1}\n".format('Subject', email_data.get('Subject') or "")
+    md += "* {}:\t{}\n".format('From', email_data.get('From') or "")
+    md += "* {}:\t{}\n".format('To', email_data.get('To') or "")
+    md += "* {}:\t{}\n".format('CC', email_data.get('CC') or "")
+    md += "* {}:\t{}\n".format('Subject', email_data.get('Subject') or "")
     if email_data.get('Text'):
         text = email_data['Text'].replace('<', '[').replace('>', ']')
-        md += u"* {0}:\t{1}\n".format('Body/Text', text or "")
+        md += "* {}:\t{}\n".format('Body/Text', text or "")
     if email_data.get('HTML'):
-        md += u"* {0}:\t{1}\n".format('Body/HTML', email_data['HTML'] or "")
+        md += "* {}:\t{}\n".format('Body/HTML', email_data['HTML'] or "")
 
-    md += u"* {0}:\t{1}\n".format('Attachments', email_data.get('Attachments') or "")
-    md += u"\n\n" + tableToMarkdown('HeadersMap', email_data.get('HeadersMap'))
+    md += "* {}:\t{}\n".format('Attachments', email_data.get('Attachments') or "")
+    md += "\n\n" + tableToMarkdown('HeadersMap', email_data.get('HeadersMap'))
     return md
 
 
@@ -89,10 +89,7 @@ def extract_file_info(entry_id: str) -> tuple:
         file_name = result[0]['Contents']['name']
 
         dt_file_type = demisto.dt(demisto.context(), f"File(val.EntryID=='{entry_id}').Type")
-        if isinstance(dt_file_type, list):
-            file_type = dt_file_type[0]
-        else:
-            file_type = dt_file_type
+        file_type = dt_file_type[0] if isinstance(dt_file_type, list) else dt_file_type
 
     except Exception as ex:
         return_error(
@@ -151,6 +148,10 @@ def main():
                         else:
                             attachment['FileData'] = None
 
+            # probably a wrapper and we can ignore the outer "email"
+            if email.get('Format') == 'multipart/signed' and all(not email.get(field) for field in ['To', 'From', 'Subject']):
+                continue
+
             if isinstance(email.get("HTML"), bytes):
                 email['HTML'] = email.get("HTML").decode('utf-8')
 

Packs/CommonScripts/Scripts/ParseEmailFilesV2/ParseEmailFilesV2.yml

@@ -113,4 +113,4 @@ type: python
 fromversion: 5.0.0
 tests:
 - ParseEmailFilesV2-test
-dockerimage: demisto/parse-emails:1.0.0.65301
+dockerimage: devdemisto/parse-emails:1.0.0.66200

Packs/CommonScripts/Scripts/ParseEmailFilesV2/ParseEmailFilesV2_test.py

@@ -6,10 +6,10 @@
 
 
 def exec_command_for_file(
-        file_path,
-        info="RFC 822 mail text, with CRLF line terminators",
-        file_name=None,
-        file_type="",
+    file_path,
+    info="RFC 822 mail text, with CRLF line terminators",
+    file_name=None,
+    file_type="",
 ):
     """
     Return a executeCommand function which will return the passed path as an entry to the call 'getFilePath'
@@ -49,7 +49,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     return executeCommand
 
@@ -63,6 +63,7 @@ def test_eml_type(mocker):
     Then:
         - Ensure its was parsed successfully
     """
+
     def executeCommand(name, args=None):
         if name == 'getFilePath':
             return [
@@ -84,7 +85,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'})
     mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand)
@@ -114,6 +115,7 @@ def test_eml_contains_eml(mocker):
         - Ensure both files was parsed
         - Ensure the attachments was returned
     """
+
     def executeCommand(name, args=None):
         if name == 'getFilePath':
             return [
@@ -135,7 +137,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'})
     mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand)
@@ -178,6 +180,7 @@ def test_eml_contains_msg(mocker):
         - Ensure both files was parsed
         - Ensure the attachments was returned
     """
+
     def executeCommand(name, args=None):
         if name == 'getFilePath':
             return [
@@ -199,7 +202,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'})
     mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand)
@@ -238,6 +241,7 @@ def test_eml_contains_eml_depth(mocker):
         - Ensure only the first mail is parsed
         - Ensure the attachments of the first mail was returned
     """
+
     def executeCommand(name, args=None):
         if name == 'getFilePath':
             return [
@@ -259,7 +263,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'})
     mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand)
@@ -376,7 +380,7 @@ def test_md_output_empty_body_text():
         'From': 'email2@paloaltonetworks.com',
         'Text': None
     }
-    expected = u'### Results:\n' \
+    expected = '### Results:\n' \
                u'* From:\temail2@paloaltonetworks.com\n' \
                u'* To:\temail1@paloaltonetworks.com\n' \
                u'* CC:\t\n' \
@@ -392,7 +396,7 @@ def test_md_output_empty_body_text():
         'To': 'email1@paloaltonetworks.com',
         'From': 'email2@paloaltonetworks.com',
     }
-    expected = u'### Results:\n' \
+    expected = '### Results:\n' \
                u'* From:\temail2@paloaltonetworks.com\n' \
                u'* To:\temail1@paloaltonetworks.com\n' \
                u'* CC:\t\n' \
@@ -421,7 +425,7 @@ def test_md_output_with_body_text():
         'From': 'email2@paloaltonetworks.com',
         'Text': '<email text>'
     }
-    expected = u'### Results:\n' \
+    expected = '### Results:\n' \
                u'* From:\temail2@paloaltonetworks.com\n' \
                u'* To:\temail1@paloaltonetworks.com\n' \
                u'* CC:\t\n' \
@@ -463,7 +467,6 @@ def test_parse_nesting_level(nesting_level_to_return, output, res):
                                                                                         ('Outer file', 3, 0, 2),
                                                                                         ('Inner file', 1, 1, 0)])
 def test_eml_contains_eml_nesting_level(mocker, nesting_level_to_return, results_len, depth, results_index):
-
     """
     Given:
     - A eml file contains eml, nesting_level_to_return param - All files.
@@ -499,7 +502,7 @@ def executeCommand(name, args=None):
                 }
             ]
         else:
-            raise ValueError('Unimplemented command called: {}'.format(name))
+            raise ValueError(f'Unimplemented command called: {name}')
 
     mocker.patch.object(demisto, 'args', return_value={'entryid': 'test',
                                                        'nesting_level_to_return': nesting_level_to_return})
@@ -532,3 +535,49 @@ def test_eml_contains_empty_htm_not_containing_file_data(mocker):
     results = demisto.results.call_args[0]
 
     assert results[0]['EntryContext']['Email']['AttachmentsData'][0]['FileData'] is None
+
+
+def test_smime_without_to_from_subject(mocker):
+    """
+    Given:
+        multipart/signed p7m file without "To"/"From"/"Subject" fields contains an eml attachment
+    When:
+        Parsing the file
+    Then:
+        The attachment files are saved to the war-room
+    """
+    save_file = mocker.patch('ParseEmailFilesV2.save_file', return_value='mocked_file_path')
+    mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'})
+    mocker.patch.object(demisto, 'executeCommand',
+                        side_effect=exec_command_for_file('smime_without_fields.p7m',
+                                                          info="ascii text",
+                                                          file_type='multipart/signed; '
+                                                                    'protocol="application/pkcs7-signature";, ASCII text'))
+    mocker.patch.object(demisto, 'results')
+    expected_email_content = ('Return-Path: <testing@gmail.com>\n'
+                              'Received: from [172.31.255.255] ([172.31.255.255])\n'
+                              '        by smtp.gmail.com with ESMTPSA id t6sm46056484wmb.29.2019.07.23.05.38.26\n'
+                              '        for <testing@gmail.com>\n'
+                              '        (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);\n'
+                              '        Tue, 23 Jul 2019 05:38:26 -0700 (PDT)\n'
+                              'To: testing@gmail.com\n'
+                              'From: test ing <testing@gmail.com>\n'
+                              'Subject: Testing Email Attachment\n'
+                              'Message-ID: <a853a1b0-1ffe-4e37-d9a9-a27c6bc0bd5b@gmail.com>\n'
+                              'Date: Tue, 23 Jul 2019 15:38:25 +0300\n'
+                              'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:60.0)\n'
+                              ' Gecko/20100101 Thunderbird/60.8.0\n'
+                              'MIME-Version: 1.0\n'
+                              'Content-Type: text/plain; charset=utf-8; format=flowed\n'
+                              'Content-Transfer-Encoding: 7bit\n'
+                              'Content-Language: en-US\n'
+                              '\n'
+                              'This is the body of the attachment.')
+
+    main()
+
+    # Assert that save_file was called with the expected arguments
+    save_file.assert_called_once_with('Attachment.eml', expected_email_content)
+    results = demisto.results.call_args[0]
+    assert len(results) == 1
+    assert results[0]['EntryContext']['Email']['FileName'] == 'Attachment.eml'