bug 1523278: add PHC field processing and searchability

This implements the first half of adding support for PHC fields. The
second half involves symbolication the PHCAllocStack and PHCFreeStack.
That's more involved, so I'm pushing that off for now.

Note that all of these fields should be considered security-sensitive
and require the pii permission.

socorro/external/es/super_search_fields.py

@@ -477,6 +477,76 @@ def get(self):
         "query_type": "enum",
         "storage_mapping": {"type": "string"},
     },
+    "phc_kind": {
+        "data_validation_type": "str",
+        "description": "The allocation kind, if the crash involved a bad access of a special PHC allocation.",
+        "form_field_choices": None,
+        "has_full_version": False,
+        "in_database_name": "phc_kind",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_kind",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_base_address": {
+        "data_validation_type": "str",
+        "description": "The allocation's base address, if the crash involved a bad access of a special PHC allocation. Encoded as a decimal address.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_base_address",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_base_address",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_usable_size": {
+        "data_validation_type": "int",
+        "description": "The allocation's usable size, if the crash involved a bad access of a special PHC allocation.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_usable_size",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_usable_size",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "number",
+        "storage_mapping": {"type": "long"},
+    },
+    "phc_alloc_stack": {
+        "data_validation_type": "str",
+        "description": "The allocation's allocation stack trace, if the crash involved a bad access of a special PHC allocation. Encoded as a comma-separated list of decimal addresses.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_alloc_stack",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_alloc_stack",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
+    "phc_free_stack": {
+        "data_validation_type": "str",
+        "description": "The allocation's free stack trace, if the crash involved a bad access of a special PHC allocation. Encoded as a comma-separated list of decimal addresses.",
+        "form_field_choices": [],
+        "has_full_version": False,
+        "in_database_name": "phc_free_stack",
+        "is_exposed": True,
+        "is_returned": True,
+        "name": "phc_free_stack",
+        "namespace": "processed_crash",
+        "permissions_needed": ["crashstats.view_pii"],
+        "query_type": "string",
+        "storage_mapping": None,
+    },
     "PluginFilename": {
         "data_validation_type": "enum",
         "description": "",

socorro/processor/processor_pipeline.py

@@ -42,6 +42,7 @@
     MozCrashReasonRule,
     OSPrettyVersionRule,
     OutOfMemoryBinaryRule,
+    PHCRule,
     PluginContentURL,
     PluginRule,
     PluginUserComment,
@@ -205,6 +206,7 @@ def get_ruleset(self, config):
             AddonsRule(),
             DatesAndTimesRule(),
             OutOfMemoryBinaryRule(),
+            PHCRule(),
             JavaProcessRule(),
             MozCrashReasonRule(),
             # post processing of the processed crash

socorro/processor/rules/mozilla.py

@@ -775,3 +775,48 @@ def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
         processor_meta["processor_notes"].extend(ret.notes)
         # NOTE(willkg): this picks up proto_signature
         processed_crash.update(ret.extra)
+
+
+class PHCRule(Rule):
+    """Performs PHC-related annotation processing.
+
+    PHC stands for probabilistic heap checker. It adds a set of annotations
+    that need to be adjusted so as to be searchable and usable in Crash Stats.
+
+    Bug #1523278.
+
+    """
+
+    def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
+        return "PHCKind" in raw_crash
+
+    def action(self, raw_crash, raw_dumps, processed_crash, proc_meta):
+        # Add PHCKind which is a string
+        processed_crash["phc_kind"] = raw_crash["PHCKind"]
+
+        # Convert PHCBaseAddress from decimal to hex and add to processed crash
+        if "PHCBaseAddress" in raw_crash:
+            try:
+                phc_base_address = hex(int(raw_crash["PHCBaseAddress"]))
+                processed_crash["phc_base_address"] = phc_base_address
+            except ValueError:
+                pass
+
+        # Add PHCUsableSize which is an integer
+        if "PHCUsableSize" in raw_crash:
+            try:
+                processed_crash["phc_usable_size"] = int(raw_crash["PHCUsableSize"])
+            except ValueError:
+                pass
+
+        # FIXME(willkg): We should symbolicate PHCAllocStack and PHCFreeStack and
+        # put the symbolicated stacks in a new field.
+        # See bug #1523278.
+
+        # Add PHCAllocStack which is a comma-separated list of integers
+        if "PHCAllocStack" in raw_crash:
+            processed_crash["phc_alloc_stack"] = raw_crash["PHCAllocStack"]
+
+        # Add PHCFreeStack which is a comma-separated list of integers
+        if "PHCFreeStack" in raw_crash:
+            processed_crash["phc_free_stack"] = raw_crash["PHCFreeStack"]

socorro/unittest/processor/rules/test_mozilla.py

@@ -9,6 +9,7 @@
 from configman.dotdict import DotDict
 from mock import call, Mock, patch
 import requests_mock
+import pytest
 
 from socorro.lib.datetimeutil import datetime_from_isodate_string
 from socorro.lib.util import dotdict_to_dict
@@ -24,6 +25,7 @@
     MozCrashReasonRule,
     OSPrettyVersionRule,
     OutOfMemoryBinaryRule,
+    PHCRule,
     PluginContentURL,
     PluginRule,
     PluginUserComment,
@@ -1505,3 +1507,61 @@ def predicate(self, raw_crash, processed_crash):
         assert mock_get_hub.return_value.capture_exception.call_args_list == [
             call(error=(Exception, exc_value, WHATEVER))
         ]
+
+
+class TestPHCRule:
+    def test_predicate(self):
+        rule = PHCRule()
+        assert rule.predicate({}, (), {}, {}) is False
+
+    @pytest.mark.parametrize(
+        "base_address, expected",
+        [(None, None), ("", None), ("foo", None), ("10", "0xa"), ("100", "0x64")],
+    )
+    def test_phc_base_address(self, base_address, expected):
+        raw_crash = {"PHCKind": "FreedPage"}
+        if base_address is not None:
+            raw_crash["PHCBaseAddress"] = base_address
+
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        if expected is None:
+            assert "phc_base_address" not in processed_crash
+        else:
+            assert processed_crash["phc_base_address"] == expected
+
+    @pytest.mark.parametrize(
+        "usable_size, expected", [(None, None), ("", None), ("foo", None), ("10", 10)]
+    )
+    def test_phc_usable_size(self, usable_size, expected):
+        raw_crash = {"PHCKind": "FreedPage"}
+        if usable_size is not None:
+            raw_crash["PHCUsableSize"] = usable_size
+
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        if expected is None:
+            assert "phc_usable_size" not in processed_crash
+        else:
+            assert processed_crash["phc_usable_size"] == expected
+
+    def test_copied_values(self):
+        raw_crash = {
+            "PHCKind": "FreedPage",
+            "PHCUsableSize": "8",
+            "PHCBaseAddress": "10",
+            "PHCAllocStack": "100,200",
+            "PHCFreeStack": "300,400",
+        }
+        rule = PHCRule()
+        processed_crash = {}
+        rule.action(raw_crash, (), processed_crash, {})
+        assert processed_crash == {
+            "phc_kind": "FreedPage",
+            "phc_usable_size": 8,
+            "phc_base_address": "0xa",
+            "phc_alloc_stack": "100,200",
+            "phc_free_stack": "300,400",
+        }

webapp-django/crashstats/crashstats/jinja2/crashstats/report_index.html

@@ -93,7 +93,7 @@ <h2>{{ report.product }} {{ report.version }} Crash Report [@ {{ report.signatur
           </ul>
 
           <div id="details" class="ui-tabs-panel ui-widget-content ui-corner-bottom">
-            <table class="record data-table vertical">
+            <table class="record data-table vertical hardwrapped">
               <tbody>
 
                 {# Crash report metadata #}
@@ -447,6 +447,39 @@ <h2>{{ report.product }} {{ report.version }} Crash Report [@ {{ report.signatur
                   </tr>
                 {% endif %}
 
+                {% if 'phc_kind' in report and (request.user.has_perm('crashstats.view_pii') or your_crash) %}
+                  <tr title="{{ fields_desc['processed_crash.phc_kind'] }}">
+                    <th scope="row">PHC Kind</th>
+                    <td>
+                      {{ report.phc_kind }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_usable_size'] }}">
+                    <th scope="row">PHC Usable Size</th>
+                    <td>
+                      {{ report.phc_usable_size }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_base_address'] }}">
+                    <th scope="row">PHC Base Address</th>
+                    <td>
+                      {{ report.phc_base_address }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_alloc_stack'] }}">
+                    <th scope="row">PHC Alloc Stack</th>
+                    <td>
+                      {{ report.phc_alloc_stack }}
+                    </td>
+                  </tr>
+                  <tr title="{{ fields_desc['processed_crash.phc_free_stack'] }}">
+                    <th scope="row">PHC Free Stack</th>
+                    <td>
+                      {{ report.phc_free_stack }}
+                    </td>
+                  </tr>
+		            {% endif %}
+
                 {% if 'memory_measures' in report %}
                   {% if 'explicit' in report.memory_measures %}
                     <tr title="{{ fields_desc['processed_crash.memory_measures.explicit'] }}">
@@ -703,7 +736,7 @@ <h2>Thread {{ thread.thread }}{% if thread.thread_name %}, Name: {{ thread.threa
           <!-- /details -->
 
           <div id="metadata" class="ui-tabs-hide">
-            <table class="record data-table vertical">
+            <table class="record data-table vertical hardwrapped">
               <tbody>
                 {% for key in raw_keys %}
                   <tr title="{{ fields_desc.get(make_raw_crash_key(key), empty_desc) }}">

webapp-django/crashstats/crashstats/static/crashstats/css/base/tables.less

@@ -45,8 +45,6 @@ table {
     tbody {
         background-color: @white;
         tr {
-
-
             &.truncated-frame {
                 background-color: @jsonview-prop;
                 a {
@@ -65,7 +63,6 @@ table {
         }
     }
 
-
     &.captioned-data-table {
         margin: 1rem;
         width: 98%;
@@ -121,6 +118,12 @@ table {
         }
     }
 
+    &.hardwrapped {
+        td {
+            overflow-wrap: anywhere;
+        }
+    }
+
 }
 
 .tablesorter {