Merge pull request #982 from twobraids/multidump2012collector

Fixes Bug 816540, multidump support for both the old and new collectors

socorro/collector/collector_app.py

@@ -70,7 +70,7 @@ class CollectorApp(App):
         'crashstorage_class',
         doc='the source storage class',
         default='socorro.external.filesystem.crashstorage.'
-        'FileSystemRawCrashStorage',
+                'FileSystemRawCrashStorage',
         from_string_converter=class_converter
     )
 

socorro/collector/throttler.py

@@ -12,6 +12,8 @@
 
 Compiled_Regular_Expression_Type = type(re.compile(''))
 
+import re
+
 #--------------------------------------------------------------------------
 ACCEPT = 0    # save and process
 DEFER = 1     # save but don't process
@@ -26,23 +28,23 @@ class LegacyThrottler(RequiredConfig):
       'throttle_conditions',
       doc='the throttling rules',
       default=[
-        # drop browser hangs
-        ("*", lambda d: "HangID" in d and
-              d.get("ProcessType", "browser") == "browser", None),
+        # drop the browser side of all multi submission hang crashes
+        ("*", '''lambda d: "HangID" in d
+              and d.get("ProcessType", "browser") == "browser"''', None),
         # 100% of crashes with comments
-        ("Comments", "lambda x: x", 100),
-        # 100% of nightly, aurora, beta & esr
-        ("ReleaseChannel",
-         "lambda x: x in ('nightly', 'aurora', 'beta', 'esr')",
-         100),
+        ("Comments", '''lambda x: x''', 100),
+        # 100% of all aurora, beta, esr channels
+        ("ReleaseChannel", '''lambda x: x in ("aurora", "beta", "esr")''', 100),
+        # 100% of all crashes that report as being nightly
+        ("ReleaseChannel", '''lambda x: x.startswith('nightly')''', 100),
         # 10% of Firefox
         ("ProductName", 'Firefox', 10),
         # 100% of Fennec
         ("ProductName", 'Fennec', 100),
         # 100% of all alpha, beta or special
-        ("Version", "re.compile(r'\..*?[a-zA-Z]+')", 100),
+        ("Version", r'''re.compile(r'\..*?[a-zA-Z]+')''', 100),
         # 100% of Thunderbird, SeaMonkey & Camino
-        ("ProductName", "lambda x: x[0] in 'TSC'", 100),
+        ("ProductName", '''lambda x: x[0] in "TSC"''', 100),
         # reject everything else
         (None, True, 0)
       ],

socorro/collector/wsgi_collector.py

@@ -27,45 +27,43 @@ def __init__(self, config):
     #--------------------------------------------------------------------------
 
     #--------------------------------------------------------------------------
-    def make_raw_crash(self, form):
+    def _make_raw_crash_and_dumps(self, form):
+        dumps = DotDict()
         raw_crash = DotDict()
-        for name in form.keys():
-            if isinstance(form[name], basestring):
-                raw_crash[name] = form[name]
+        for name, value in form.iteritems():
+            if isinstance(value, basestring):
+                raw_crash[name] = value
+            elif hasattr(value, 'file') and hasattr(value, 'value'):
+                dumps[name] = value.value
             else:
-                raw_crash[name] = form[name].value
-        raw_crash.timestamp = time.time()
-        return raw_crash
+                raw_crash[name] = value.value
+        return raw_crash, dumps
 
     #--------------------------------------------------------------------------
     def POST(self, *args):
-        the_form = web.input()
-        dump = the_form[self.dump_field]
-
-        # Remove other submitted files from the input form, which are
-        # an indication of a multi-dump hang submission we aren't yet
-        # prepared to handle.
-        for (key, value) in web.webapi.rawinput().iteritems():
-            if hasattr(value, 'file') and hasattr(value, 'value'):
-                del the_form[key]
-
-        raw_crash = self.make_raw_crash(the_form)
+        raw_crash, dumps = \
+            self._make_raw_crash_and_dumps(web.webapi.rawinput())
 
         current_timestamp = utc_now()
         raw_crash.submitted_timestamp = current_timestamp.isoformat()
+        # legacy - ought to be removed someday
+        raw_crash.timestamp = time.time()
 
         crash_id = createNewOoid(current_timestamp)
         self.logger.info('%s received', crash_id)
 
         raw_crash.legacy_processing = self.throttler.throttle(raw_crash)
         if raw_crash.legacy_processing == DISCARD:
+            self.logger.info('%s discarded', crash_id)
             return "Discarded=1\n"
         if raw_crash.legacy_processing == IGNORE:
+            self.logger.info('%s ignored', crash_id)
             return "Unsupported=1\n"
 
         self.config.crash_storage.save_raw_crash(
           raw_crash,
-          dump,
+          dumps,
           crash_id
         )
+        self.logger.info('%s accepted', crash_id)
         return "CrashID=%s%s\n" % (self.dump_id_prefix, crash_id)