BUG: shadowserver parser: raise required missing

Raise an error if a required field is not present
line gets dumped, won't be processed further

https://lists.cert.at/pipermail/intelmq-users/2018-July/000058.html

CHANGELOG.md

@@ -1,7 +1,6 @@
 CHANGELOG
 ==========
 
-
 1.0.6 Bugfix release (unreleased)
 ---------------------------------
 
@@ -13,6 +12,7 @@ CHANGELOG
 #### Collectors
 
 #### Parsers
+- `bots.parsers.shadowserver`: if required fields do not exist in data, an exception is raised, so the line will be dumped and not further processed.
 
 #### Experts
 - Reverse DNS Expert: ignore all invalid results and use first valid one (#1264).

intelmq/bots/parsers/shadowserver/parser.py

@@ -89,8 +89,8 @@ def parse_line(self, row, report):
         for item in conf.get('required_fields'):
             intelmqkey, shadowkey = item[:2]
             if shadowkey not in fields:  # key does not exist in data (not even in the header)
-                self.logger.warning('Required key %r not found in data. Possible change in data'
-                                    ' format or misconfiguration.', shadowkey)
+                raise ValueError('Required column %r not found in data. Possible change in data'
+                                 ' format or misconfiguration.' % shadowkey)
             if len(item) > 2:
                 conv_func = item[2]
             else:
@@ -116,7 +116,7 @@ def parse_line(self, row, report):
         for item in conf.get('optional_fields'):
             intelmqkey, shadowkey = item[:2]
             if shadowkey not in fields:  # key does not exist in data (not even in the header)
-                self.logger.warning('Optional key %r not found data. Possible change in data'
+                self.logger.warning('Optional key %r not found in data. Possible change in data'
                                     ' format or misconfiguration.', shadowkey)
                 continue
             if len(item) > 2:

intelmq/lib/test.py

@@ -396,8 +396,8 @@ def assertLogMatches(self, pattern: str, levelname: str = "ERROR"):
         Asserts if any logline matches a specific requirement.
 
         Parameters:
-            pattern: Message text which is compared
-            type: Type of logline which is asserted
+            pattern: Message text which is compared, regular expression.
+            levelname: Log level of the logline which is asserted, upper case.
         """
 
         self.assertIsNotNone(self.loglines)

intelmq/tests/bots/parsers/shadowserver/test_broken.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'),
+           "__type": "Report",
+           "time.observation": "2015-01-01T00:00:00+00:00",
+           }
+REPORT2 = {"raw": utils.base64_encode('''timestamp,ip,port
+2018-08-01T00:00:00+00,127.0.0.1,80
+'''),
+           "__type": "Report",
+           "time.observation": "2015-01-01T00:00:00+00:00",
+           }
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for a ShadowserverParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ShadowserverParserBot
+        cls.sysconfig = {'feedname': 'Accessible-CWMP'}
+
+    def test_broken(self):
+        """
+        Test a report which does not have valid fields
+        """
+        self.input_message = REPORT1
+        self.allowed_error_count = 1
+        self.run_bot()
+        self.assertLogMatches(pattern="Failed to parse line.")
+        self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in data. Possible change in data format or misconfiguration.")
+        self.assertLogMatches(pattern="Sent 0 events and found 1 error\(s\)\.",
+                              levelname="INFO")
+
+    def test_half_broken(self):
+        """
+        Test a report which does not have an optional field.
+        """
+        self.input_message = REPORT2
+        self.allowed_warning_count = 1
+        self.run_bot()
+        self.assertLogMatches(pattern="Optional key 'protocol' not found in data. Possible change in data format or misconfiguration.",
+                              levelname="WARNING")
+        self.assertLogMatches(pattern="Sent 1 events and found 0 error\(s\)\.",
+                              levelname="INFO")
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()