Fixes bug 1274345 - Added support from triming signatures missing sym…

…bols. (#3363)

* Fixes bug 1274345 - Added support from triming signatures missing symbols.

* nits

socorro/processor/signature_utilities.py

@@ -89,6 +89,7 @@ def __init__(self, config, quit_check_callback=None):
         self.irrelevant_signature_re = None
         self.prefix_signature_re = None
         self.signatures_with_line_numbers_re = None
+        self.trim_dll_signature_re = None
         self.signature_sentinels = []
 
         self.fixup_space = re.compile(r' (?=[\*&,])')
@@ -170,7 +171,7 @@ def normalize_signature(
         serve as a signature.  The parameter names of this function reflect the
         exact names of the fields from the jsonMDSW frame output.  This allows
         this function to be invoked by passing a frame as **a_frame. Sometimes,
-        a frame may already have a normalized version cached.  If that exsists,
+        a frame may already have a normalized version cached.  If that exists,
         return it instead.
         """
         if normalized is not None:
@@ -226,9 +227,10 @@ def _do_generate(self,
           - a prefix of a relevant frame: Append this element to the signature
           - a relevant frame: Append this element and stop looking
           - irrelevant: Append this element only after seeing a prefix frame
-        The signature is a ' | ' separated string of frame names
+        The signature is a ' | ' separated string of frame names.
         """
         signature_notes = []
+
         # shorten source_list to the first signatureSentinel
         sentinel_locations = []
         for a_sentinel in self.signature_sentinels:
@@ -243,17 +245,41 @@ def _do_generate(self,
         if sentinel_locations:
             source_list = source_list[min(sentinel_locations):]
 
+        # Get all the relevant frame signatures.
         new_signature_list = []
         for a_signature in source_list:
+            # If the signature matches the irrelevant signatures regex,
+            # skip to the next frame.
             if self.irrelevant_signature_re.match(a_signature):
                 continue
+
+            # If the signature matches the trim dll signatures regex,
+            # rewrite it to remove all but the module name.
+            if self.trim_dll_signature_re.match(a_signature):
+                a_signature = a_signature.split('@')[0]
+
+                # If this trimmed DLL signature is the same as the previous
+                # frame's, we do not want to add it.
+                if (
+                    new_signature_list and
+                    a_signature == new_signature_list[-1]
+                ):
+                    continue
+
             new_signature_list.append(a_signature)
+
+            # If the signature does not match the prefix signatures regex,
+            # then it is the last one we add to the list.
             if not self.prefix_signature_re.match(a_signature):
                 break
+
+        # Add a special marker for hang crash reports.
         if hang_type:
             new_signature_list.insert(0, self.hang_prefixes[hang_type])
+
         signature = delimiter.join(new_signature_list)
 
+        # Handle empty signatures to explain why we failed generating them.
         if signature == '' or signature is None:
             if crashed_thread is None:
                 signature_notes.append("CSignatureTool: No signature could be "
@@ -290,6 +316,9 @@ def __init__(self, config, quit_check_callback=None):
         self.signatures_with_line_numbers_re = re.compile(
             '|'.join(siglists.SIGNATURES_WITH_LINE_NUMBERS_RE)
         )
+        self.trim_dll_signature_re = re.compile(
+            '|'.join(siglists.TRIM_DLL_SIGNATURE_RE)
+        )
         self.signature_sentinels = siglists.SIGNATURE_SENTINELS
 
 

socorro/siglists/README.md

@@ -4,12 +4,13 @@ This folder contains lists that are used to configure the C signature generation
 
 ## Signature Generation Algorithm
 
-When generating a C signature, 4 steps are involved.
+When generating a C signature, 5 steps are involved.
 
 1. We walk the crashing thread's stack, looking for things that would match the [Signature Sentinels](#signature-sentinels). The first matching element, if any, becomes the top of the sub-stack we'll consider going forward.
 2. We walk the stack, ignoring everything that matches the [Irrelevant Signatures](#irrelevant-signatures). We consider the first non-matching element the top of the new sub-stack.
-3. We accumulate signatures that match the [Prefix Signatures](#prefix-signatures), until something doesn't match.
-4. We normalize each signature we accumulated. Signatures that match the [Signatures With Line Numbers](#signatures-with-line-numbers) have their associated code line number added to them, like this: ``signature:42``.
+3. We rewrite every signature missing symbols that matches the [Trim DLL Signatures](#trim-dll-signatures) to be the module only (the part before the first ``@`` sign). We also merge them so only one of those frames makes it to the final signature.
+4. We accumulate signatures that match the [Prefix Signatures](#prefix-signatures), until something doesn't match.
+5. We normalize each signature we accumulated. Signatures that match the [Signatures With Line Numbers](#signatures-with-line-numbers) have their associated code line number added to them, like this: ``signature:42``.
 
 The generated signature is a concatenation of all the accumulated signatures, separated with a pipe sign (`` | ``).
 
@@ -31,6 +32,20 @@ File: [``prefix_signature_re.txt``](./prefix_signature_re.txt)
 
 Prefix Signatures are regular expressions of signatures that will be combined with the following frame's signature. Signature generation stops at the first non-matching signature it finds.
 
+## Trim DLL Signatures
+
+File: [``trim_dll_signature_re.txt``](./trim_dll_signature_re.txt)
+
+Trim DLL Signatures are regular expressions of signatures that will be trimmed down to only their module name. For example, if the list contains ``foo32\.dll.*`` and a stack trace looks like this:
+
+<pre>0x0
+foo32.dll@0x2131
+foo32.dll@0x1943
+myFavoriteSig()
+</pre>
+
+The generated signature will be: ``0x0 | foo32.dll | myFavoriteSig()``.
+
 ## Signatures With Line Numbers
 
 File: [``signatures_with_line_numbers_re.txt``](./signatures_with_line_numbers_re.txt)

socorro/siglists/__init__.py

@@ -70,3 +70,4 @@ def _get_file_content(source):
 SIGNATURES_WITH_LINE_NUMBERS_RE = _get_file_content(
     'signatures_with_line_numbers_re'
 )
+TRIM_DLL_SIGNATURE_RE = _get_file_content('trim_dll_signature_re')

socorro/siglists/trim_dll_signature_re.txt

@@ -0,0 +1,6 @@
+# Trim DLL Signatures
+#
+# Regular expressions matching frame signatures that should be trimmed down to
+# only include the module name.
+
+igdumdim32\.dll.*

socorro/unittest/processor/test_signature_utilities.py

@@ -47,6 +47,7 @@ def setup_config_c_sig_tool(
         ig=['ignored1'],
         pr=['pre1', 'pre2'],
         si=['fnNeedNumber'],
+        td=['foo32\.dll.*'],
         ss=('sentinel', ('sentinel2', lambda x: 'ff' in x)),
     ):
         config = sutil.DotDict()
@@ -59,6 +60,7 @@ def setup_config_c_sig_tool(
             mocked_siglists.IRRELEVANT_SIGNATURE_RE = ig
             mocked_siglists.PREFIX_SIGNATURE_RE = pr
             mocked_siglists.SIGNATURES_WITH_LINE_NUMBERS_RE = si
+            mocked_siglists.TRIM_DLL_SIGNATURE_RE = td
             mocked_siglists.SIGNATURE_SENTINELS = ss
             s = CSignatureTool(config)
 
@@ -266,6 +268,33 @@ def test_generate_4(self):
         sig, notes = s.generate(a)
         self.assert_equal_with_nicer_output(e, sig)
 
+    #--------------------------------------------------------------------------
+    def test_generate_with_merged_dll(self):
+        generator, config = self.setup_config_c_sig_tool(
+            ['a', 'b', 'c'],
+            ['d', 'e', 'f']
+        )
+        source_list = (
+            'a',
+            'd',
+            'foo32.dll@0x231423',
+            'foo32.dll',
+            'foo32.dll@0x42',
+            'g',
+        )
+        e = 'd | foo32.dll | g'
+        sig, notes = generator.generate(source_list)
+        self.assert_equal_with_nicer_output(e, sig)
+
+        source_list = (
+            'foo32.dll',
+            'foo32.dll@0x231423',
+            'g',
+        )
+        e = 'foo32.dll | g'
+        sig, notes = generator.generate(source_list)
+        self.assert_equal_with_nicer_output(e, sig)
+
 
 #==============================================================================
 class TestJavaSignatureTool(BaseTestClass):