Bug 792082 - Ignore the browser side of plugin-hang reports

docs/throttling.rst

@@ -27,22 +27,30 @@ throttleConditions
 
 This option tells the collector how to route a given JSON/dump pair to
 storage for further processing or deferred storage. This consists of a
-list of conditions in this form: (JsonFileKey?, ConditionFunction?,
+list of conditions in this form: (RawCrashKey?, ConditionFunction?,
 Probability)
 
-* JsonFileKey?: a name of a field from the HTTP POST form. The
+* RawCrashKey?: a name of a field from the HTTP POST form. The
   possibilities are: "StartupTime?", "Vendor", "InstallTime?",
   "timestamp", "Add-ons", "BuildID", "SecondsSinceLastCrash?", "UserID",
   "ProductName?", "URL", "Theme", "Version", "CrashTime?"
-* ConditionFunction?: a function returning a boolean, regular
-  expression or a constant used to test the value for the
-  JsonFileKey?.
+  Alternatively, the string "*" has special meaning when the
+  ConditionFunction? is a reference to a Python function.
+* ConditionFunction?: a function accepting a single string value and
+  returning a boolean; regular expression; or a constant used for an
+  equality test with the value for the RawCrashKey?.
+  Alternatively, If the RawCrashKey? is "*" and the function will be
+  passed the entire raw crash as a dict rather than just a single
+  value of one element of the raw crash.
 * Probability: an integer between 0 and 100 inclusive. At 100, all
   JSON files, for which the ConditionFunction? returns true, will be
   saved in the database. At 0, no JSON files for which the
   ConditionFunction? returns true will be saved to the database. At 25,
   there is twenty-five percent probability that a matching JSON file
   will be written to the database.
+  Alternatively, the value can be None.  In that case, no probablity is
+  calculated and the throttler just returns the IGNORE value.  The crash
+  is not stored and "Unsupported=1" is returned to the client.
 
 There must be at least one entry in the throttleConditions list. The
 example below shows the default case.
@@ -60,5 +68,6 @@ Keep the list short to avoid bogging down the collector.::
    #("Add-ons", re.compile('inspector\@mozilla\.org\:1\..*'), 75), # queue 75% of crashes where the inspector addon is at 1.x
    #("UserID", "d6d2b6b0-c9e0-4646-8627-0b1bdd4a92bb", 100), # queue all of this user's crashes
    #("SecondsSinceLastCrash", lambda x: 300 >= int(x) >= 0, 100), # queue all crashes that happened within 5 minutes of another crash
+   ("*", lambda d: d["Product"] == "Flock" and d["Version"] == "3.0", None), # ignore Flock 3.0
    (None, True, 10) # queue 10% of what's left
  ]

scripts/config/collectorconfig.py.dist

@@ -136,6 +136,7 @@ neverDiscard.fromStringConverter = cm.booleanConverter
 
 throttleConditions = cm.Option()
 throttleConditions.default = throttleConditions.default = [
+    ("*", lambda d: "HangID" in d and d.get("ProcessType", "browser") == "browser", None),  # drop browser hangs
     ("Comments", lambda x: x, 100), # 100% of crashes with comments
     ("ReleaseChannel", lambda x: x in ("aurora", "beta", "esr"), 100),
     ("ReleaseChannel", lambda x: x.startswith('nightly'), 100),

socorro/collector/throttler.py

@@ -13,9 +13,10 @@
 Compiled_Regular_Expression_Type = type(re.compile(''))
 
 #--------------------------------------------------------------------------
-ACCEPT = 0
-DEFER = 1
-DISCARD = 2
+ACCEPT = 0    # save and process
+DEFER = 1     # save but don't process
+DISCARD = 2   # tell client to go away and not come back
+IGNORE = 3    # ignore this submission entirely
 
 
 #==============================================================================
@@ -142,7 +143,10 @@ def apply_throttle_conditions(self, raw_crash):
         for key, condition, percentage in self.processed_throttle_conditions:
             throttle_match = False
             try:
-                throttle_match = condition(raw_crash[key])
+                if key == '*':
+                    throttle_match = condition(raw_crash)
+                else:
+                    throttle_match = condition(raw_crash[key])
             except KeyError:
                 if key == None:
                     throttle_match = condition(None)
@@ -152,14 +156,24 @@ def apply_throttle_conditions(self, raw_crash):
             except IndexError:
                 pass
             if throttle_match:  # we've got a condition match - apply percent
+                if percentage is None:
+                    return None
                 random_real_percent = random.random() * 100.0
                 return random_real_percent > percentage
         # nothing matched, reject
         return True
 
     #--------------------------------------------------------------------------
     def throttle(self, raw_crash):
-        if self.apply_throttle_conditions(raw_crash):
+        throttle_result = self.apply_throttle_conditions(raw_crash)
+        if throttle_result is None:
+            self.config.logger.debug(
+              "ignoring %s %s",
+              raw_crash.ProductName,
+              raw_crash.Version
+            )
+            return IGNORE
+        if throttle_result:  # we're rejecting
             #logger.debug('yes, throttle this one')
             if (self.understands_refusal(raw_crash)
                 and not self.config.never_discard):
@@ -176,7 +190,7 @@ def throttle(self, raw_crash):
                   raw_crash.Version
                 )
                 return DEFER
-        else:
+        else:  # we're accepting
             self.config.logger.debug(
               "not throttled %s %s",
               raw_crash.ProductName,

socorro/collector/wsgi_collector.py

@@ -7,7 +7,7 @@
 
 from socorro.lib.ooid import createNewOoid
 from socorro.lib.util import DotDict
-from socorro.collector.throttler import DISCARD
+from socorro.collector.throttler import DISCARD, IGNORE
 from socorro.lib.datetimeutil import utc_now
 
 
@@ -60,6 +60,8 @@ def POST(self, *args):
         raw_crash.legacy_processing = self.throttler.throttle(raw_crash)
         if raw_crash.legacy_processing == DISCARD:
             return "Discarded=1\n"
+        if raw_crash.legacy_processing == IGNORE:
+            return "Unsupported=1\n"
 
         self.config.crash_storage.save_raw_crash(
           raw_crash,

socorro/collector/wsgicollector.py

@@ -46,6 +46,11 @@ def POST(self, *args):
     ooid = sooid.createNewOoid(currentTimestamp)
     jsonDataDictionary.legacy_processing = \
         self.legacyThrottler.throttle(jsonDataDictionary)
+
+    if jsonDataDictionary.legacy_processing == cstore.LegacyThrottler.IGNORE:
+      self.logger.info('%s ignored', ooid)
+      return "Unsupported=1\n"
+
     self.logger.info('%s received', ooid)
     result = crashStorage.save_raw(ooid,
                                    jsonDataDictionary,

socorro/storage/crashstorage.py

@@ -81,6 +81,7 @@ def __init__(self, config):
   ACCEPT = 0
   DEFER = 1
   DISCARD = 2
+  IGNORE = 3
   #-----------------------------------------------------------------------------------------------------------------
   @staticmethod
   def regexpHandlerFactory(regexp):
@@ -143,7 +144,10 @@ def applyThrottleConditions (self, jsonData):
       #logger.debug("throttle testing  %s %s %d", key, condition, percentage)
       throttleMatch = False
       try:
-        throttleMatch = condition(jsonData[key])
+        if key == '*':
+          throttleMatch = condition(jsonData)
+        else:
+          throttleMatch = condition(jsonData[key])
       except KeyError:
         if key == None:
           throttleMatch = condition(None)
@@ -153,6 +157,8 @@ def applyThrottleConditions (self, jsonData):
       except IndexError:
         pass
       if throttleMatch: #we've got a condition match - apply the throttle percentage
+        if percentage is None:
+          return None
         randomRealPercent = random.random() * 100.0
         #logger.debug("throttle: %f %f %s", randomRealPercent, percentage, randomRealPercent > percentage)
         return randomRealPercent > percentage
@@ -179,7 +185,11 @@ def applyThrottleConditions (self, jsonData):
 
   #-----------------------------------------------------------------------------------------------------------------
   def throttle (self, jsonData):
-    if self.applyThrottleConditions(jsonData):
+    result = self.applyThrottleConditions(jsonData)
+    if result is None:
+      logger.debug("ignoring %s %s", jsonData.ProductName, jsonData.Version)
+      return LegacyThrottler.IGNORE
+    if result:
       #logger.debug('yes, throttle this one')
       if self.understandsRefusal(jsonData) and not self.config.neverDiscard:
         logger.debug("discarding %s %s", jsonData.ProductName, jsonData.Version)
@@ -229,6 +239,7 @@ def makeJsonDictFromForm (self, form, tm=tm):
   DISCARDED = 2
   ERROR = 3
   RETRY = 4
+
   #-----------------------------------------------------------------------------------------------------------------
   def terminated (self, jsonData):
     return False