Bug 1347283 - Add support decoding invalid UTF8 strings

moztelemetry/heka/message_parser.py

@@ -35,7 +35,7 @@ def _parse_heka_record(record):
             # because it contains null values for JSON fields that have been
             # split out.
             if field.name == 'submission':
-                payload = _parse_json(field.value_bytes[0].decode('utf-8'))
+                payload = _parse_json(field.value_bytes[0].decode('utf-8', 'replace'))
                 break
 
     if payload is None:

tests/heka/test_message_parser.py

@@ -78,3 +78,10 @@ def test_telemetry(data_dir):
         for r in message_parser.parse_heka_message(o):
             assert set(r.keys()) == top_keys
             assert set(r["payload"].keys()) == payload_keys
+
+
+def test_invalid_utf8(data_dir):
+    filename = "{}/test_invalid_utf8.heka".format(data_dir)
+    with open(filename, "rb") as o:
+        for r in message_parser.parse_heka_message(o):
+            assert(u'\ufffd' in r['info']['adapterDescription'])