Merge pull request #4989 from willkg/1523278-phc

bug 1523278: add PHC field processing and searchability

socorro/external/es/super_search_fields.py

@@ -477,6 +477,76 @@ def get(self):
         "query_type": "enum",
         "storage_mapping": {"type": "string"},
     },
+    "phc_kind": {
+        "data_validation_type": "str",
+        "description": "The allocation kind, if the crash involved a bad access of a special PHC allocation.",
+        "form_field_choices": None,
+        "has_full_version": False,
+        "in_database_name": "phc_kind",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_kind",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_base_address": {
+        "data_validation_type": "str",
+        "description": "The allocation's base address, if the crash involved a bad access of a special PHC allocation. Encoded as a decimal address.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_base_address",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_base_address",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_usable_size": {
+        "data_validation_type": "int",
+        "description": "The allocation's usable size, if the crash involved a bad access of a special PHC allocation.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_usable_size",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_usable_size",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "number",
+        "storage_mapping": {"type": "long"},
+    },
+    "phc_alloc_stack": {
+        "data_validation_type": "str",
+        "description": "The allocation's allocation stack trace, if the crash involved a bad access of a special PHC allocation. Encoded as a comma-separated list of decimal addresses.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_alloc_stack",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_alloc_stack",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_free_stack": {
+        "data_validation_type": "str",
+        "description": "The allocation's free stack trace, if the crash involved a bad access of a special PHC allocation. Encoded as a comma-separated list of decimal addresses.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_free_stack",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_free_stack",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
     "PluginFilename": {
         "data_validation_type": "enum",
         "description": "",

socorro/processor/processor_pipeline.py

@@ -42,6 +42,7 @@
     MozCrashReasonRule,
     OSPrettyVersionRule,
     OutOfMemoryBinaryRule,
+    PHCRule,
     PluginContentURL,
     PluginRule,
     PluginUserComment,
@@ -205,6 +206,7 @@ def get_ruleset(self, config):
             AddonsRule(),
             DatesAndTimesRule(),
             OutOfMemoryBinaryRule(),
+            PHCRule(),
             JavaProcessRule(),
             MozCrashReasonRule(),
             # post processing of the processed crash

socorro/processor/rules/mozilla.py

@@ -775,3 +775,48 @@ def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
         processor_meta["processor_notes"].extend(ret.notes)
         # NOTE(willkg): this picks up proto_signature
         processed_crash.update(ret.extra)
+
+
+class PHCRule(Rule):
+    """Performs PHC-related annotation processing.
+
+    PHC stands for probabilistic heap checker. It adds a set of annotations
+    that need to be adjusted so as to be searchable and usable in Crash Stats.
+
+    Bug #1523278.
+
+    """
+
+    def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
+        return "PHCKind" in raw_crash
+
+    def action(self, raw_crash, raw_dumps, processed_crash, proc_meta):
+        # Add PHCKind which is a string
+        processed_crash["phc_kind"] = raw_crash["PHCKind"]
+
+        # Convert PHCBaseAddress from decimal to hex and add to processed crash
+        if "PHCBaseAddress" in raw_crash:
+            try:
+                phc_base_address = hex(int(raw_crash["PHCBaseAddress"]))
+                processed_crash["phc_base_address"] = phc_base_address
+            except ValueError:
+                pass
+
+        # Add PHCUsableSize which is an integer
+        if "PHCUsableSize" in raw_crash:
+            try:
+                processed_crash["phc_usable_size"] = int(raw_crash["PHCUsableSize"])
+            except ValueError:
+                pass
+
+        # FIXME(willkg): We should symbolicate PHCAllocStack and PHCFreeStack and
+        # put the symbolicated stacks in a new field.
+        # See bug #1523278.
+
+        # Add PHCAllocStack which is a comma-separated list of integers
+        if "PHCAllocStack" in raw_crash:
+            processed_crash["phc_alloc_stack"] = raw_crash["PHCAllocStack"]
+
+        # Add PHCFreeStack which is a comma-separated list of integers
+        if "PHCFreeStack" in raw_crash:
+            processed_crash["phc_free_stack"] = raw_crash["PHCFreeStack"]

socorro/unittest/processor/rules/test_mozilla.py

@@ -9,6 +9,7 @@
 from configman.dotdict import DotDict
 from mock import call, Mock, patch
 import requests_mock
+import pytest
 
 from socorro.lib.datetimeutil import datetime_from_isodate_string
 from socorro.lib.util import dotdict_to_dict
@@ -24,6 +25,7 @@
     MozCrashReasonRule,
     OSPrettyVersionRule,
     OutOfMemoryBinaryRule,
+    PHCRule,
     PluginContentURL,
     PluginRule,
     PluginUserComment,
@@ -1505,3 +1507,61 @@ def predicate(self, raw_crash, processed_crash):
         assert mock_get_hub.return_value.capture_exception.call_args_list == [
             call(error=(Exception, exc_value, WHATEVER))
         ]
+
+
+class TestPHCRule:
+    def test_predicate(self):
+        rule = PHCRule()
+        assert rule.predicate({}, (), {}, {}) is False
+
+    @pytest.mark.parametrize(
+        "base_address, expected",
+        [(None, None), ("", None), ("foo", None), ("10", "0xa"), ("100", "0x64")],
+    )
+    def test_phc_base_address(self, base_address, expected):
+        raw_crash = {"PHCKind": "FreedPage"}
+        if base_address is not None:
+            raw_crash["PHCBaseAddress"] = base_address
+
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        if expected is None:
+            assert "phc_base_address" not in processed_crash
+        else:
+            assert processed_crash["phc_base_address"] == expected
+
+    @pytest.mark.parametrize(
+        "usable_size, expected", [(None, None), ("", None), ("foo", None), ("10", 10)]
+    )
+    def test_phc_usable_size(self, usable_size, expected):
+        raw_crash = {"PHCKind": "FreedPage"}
+        if usable_size is not None:
+            raw_crash["PHCUsableSize"] = usable_size
+
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        if expected is None:
+            assert "phc_usable_size" not in processed_crash
+        else:
+            assert processed_crash["phc_usable_size"] == expected
+
+    def test_copied_values(self):
+        raw_crash = {
+            "PHCKind": "FreedPage",
+            "PHCUsableSize": "8",
+            "PHCBaseAddress": "10",
+            "PHCAllocStack": "100,200",
+            "PHCFreeStack": "300,400",
+        }
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        assert processed_crash == {
+            "phc_kind": "FreedPage",
+            "phc_usable_size": 8,
+            "phc_base_address": "0xa",
+            "phc_alloc_stack": "100,200",
+            "phc_free_stack": "300,400",
+        }

webapp-django/crashstats/crashstats/jinja2/crashstats/report_index.html

@@ -93,7 +93,7 @@ <h2>{{ report.product }} {{ report.version }} Crash Report [@ {{ report.signatur
           </ul>
 
           <div id="details" class="ui-tabs-panel ui-widget-content ui-corner-bottom">
-            <table class="record data-table vertical">
+            <table class="record data-table vertical hardwrapped">
               <tbody>
 
                 {# Crash report metadata #}
@@ -447,6 +447,39 @@ <h2>{{ report.product }} {{ report.version }} Crash Report [@ {{ report.signatur
                   </tr>
                 {% endif %}
 
+                {% if 'phc_kind' in report and (request.user.has_perm('crashstats.view_pii') or your_crash) %}
+                  <tr title="{{ fields_desc['processed_crash.phc_kind'] }}">
+                    <th scope="row">PHC Kind</th>
+                    <td>
+                      {{ report.phc_kind }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_usable_size'] }}">
+                    <th scope="row">PHC Usable Size</th>
+                    <td>
+                      {{ report.phc_usable_size }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_base_address'] }}">
+                    <th scope="row">PHC Base Address</th>
+                    <td>
+                      {{ report.phc_base_address }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_alloc_stack'] }}">
+                    <th scope="row">PHC Alloc Stack</th>
+                    <td>
+                      {{ report.phc_alloc_stack }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_free_stack'] }}">
+                    <th scope="row">PHC Free Stack</th>
+                    <td>
+                      {{ report.phc_free_stack }}
+                    </td>
+                  </tr>
+		            {% endif %}
+
                 {% if 'memory_measures' in report %}
                   {% if 'explicit' in report.memory_measures %}
                     <tr title="{{ fields_desc['processed_crash.memory_measures.explicit'] }}">
@@ -703,7 +736,7 @@ <h2>Thread {{ thread.thread }}{% if thread.thread_name %}, Name: {{ thread.threa
           <!-- /details -->
 
           <div id="metadata" class="ui-tabs-hide">
-            <table class="record data-table vertical">
+            <table class="record data-table vertical hardwrapped">
               <tbody>
                 {% for key in raw_keys %}
                   <tr title="{{ fields_desc.get(make_raw_crash_key(key), empty_desc) }}">

webapp-django/crashstats/crashstats/static/crashstats/css/base/tables.less

@@ -45,8 +45,6 @@ table {
     tbody {
         background-color: @white;
         tr {
-
-
             &.truncated-frame {
                 background-color: @jsonview-prop;
                 a {
@@ -65,7 +63,6 @@ table {
         }
     }
 
-
     &.captioned-data-table {
         margin: 1rem;
         width: 98%;
@@ -121,6 +118,12 @@ table {
         }
     }
 
+    &.hardwrapped {
+        td {
+            overflow-wrap: anywhere;
+        }
+    }
+
 }
 
 .tablesorter {