Merge pull request #5950 from drew2a/feature/improv_sentry_event

Feature/improve sentry event

src/tribler-common/tribler_common/sentry_reporter/sentry_reporter.py

@@ -13,6 +13,7 @@
 
 from tribler_common.sentry_reporter.sentry_tools import (
     delete_item,
+    extract_dict,
     get_first_item,
     get_value,
     parse_os_environ,
@@ -169,6 +170,9 @@ def send_event(event, post_data=None, sys_info=None):
         if event is None:
             return event
 
+        post_data = post_data or dict()
+        sys_info = sys_info or dict()
+
         if CONTEXTS not in event:
             event[CONTEXTS] = {}
 
@@ -192,12 +196,18 @@ def send_event(event, post_data=None, sys_info=None):
 
         context['browser'] = {'version': version, 'name': 'Tribler'}
 
-        reporter[STACKTRACE] = parse_stacktrace(get_value(post_data, 'stack'))
+        stacktrace_parts = parse_stacktrace(get_value(post_data, 'stack'))
+        reporter[STACKTRACE] = next(stacktrace_parts, [])
+        reporter[f'{STACKTRACE}_extra'] = next(stacktrace_parts, [])
+        reporter[f'{STACKTRACE}_context'] = next(stacktrace_parts, [])
+
         reporter['comments'] = get_value(post_data, 'comments')
 
         reporter[OS_ENVIRON] = parse_os_environ(get_value(sys_info, OS_ENVIRON))
         delete_item(sys_info, OS_ENVIRON)
-        reporter[SYSINFO] = sys_info
+
+        reporter['events'] = extract_dict(sys_info, r'^(event|request)')
+        reporter[SYSINFO] = {key: sys_info[key] for key in sys_info if key not in reporter['events']}
 
         with this_sentry_strategy(SentryStrategy.SEND_ALLOWED):
             sentry_sdk.capture_event(event)

src/tribler-common/tribler_common/sentry_reporter/sentry_tools.py

@@ -1,6 +1,7 @@
 """ This a collection of tools for SentryReporter and SentryScrubber aimed to
 simplify work with several data structures.
 """
+import re
 
 
 def parse_os_environ(array):
@@ -29,21 +30,36 @@ def parse_os_environ(array):
     return result
 
 
-def parse_stacktrace(stacktrace):
+def parse_stacktrace(stacktrace, delimiters=None):
     """Parse stacktrace field.
 
+    Example of stacktrace:
+        Traceback (most recent call last):,
+              File "src\run_tribler.py", line 179, in <module>,
+            RuntimeError: ('\'utf-8\' codec can\'t decode byte 0xcd in position 0: invalid continuation byte,
+        --LONG TEXT--,
+            Traceback (most recent call last):,
+              File "<user>\\asyncio\\events.py", line 81, in _run,
+            UnicodeDecodeError: \'utf-8\' codec can\'t decode byte 0xcd in position 0: invalid continuation byte,
+        --CONTEXT--,
+            {\'message\': "Exception in callback'
     Args:
-        stacktrace: a string with '\n' delimiter.
-            Example: "line1\nline2"
+        stacktrace: the string that represents stacktrace.
+
+        delimiters: hi-level delimiters of the stacktrace.
+            ['--LONG TEXT--', '--CONTEXT--'] by default.
 
     Returns:
-        The list of strings made from the given string.
-            Example: ["line1", "line2"]
+        The generator of stacktrace parts.
     """
+    if delimiters is None:
+        delimiters = ['--LONG TEXT--', '--CONTEXT--']
+
     if not stacktrace:
-        return []
+        return
 
-    return [line for line in stacktrace.split('\n') if line]
+    for part in re.split('|'.join(delimiters), stacktrace):
+        yield [line for line in re.split(r'\\n|\n', part) if line]
 
 
 def get_first_item(items, default=None):
@@ -67,6 +83,14 @@ def get_value(d, key, default=None):
     return d.get(key, default) if d else default
 
 
+def extract_dict(d, regex_key_pattern):
+    if not d or not regex_key_pattern:
+        return dict()
+
+    matched_keys = [key for key in d if re.match(regex_key_pattern, key)]
+    return {key: d[key] for key in matched_keys}
+
+
 def modify_value(d, key, function):
     if not d or not key or not function:
         return d

src/tribler-common/tribler_common/sentry_reporter/tests/test_sentry_reporter.py

@@ -48,7 +48,15 @@ def test_send():
     assert SentryReporter.send_event({}, None, None) == {
         'contexts': {
             'browser': {'name': 'Tribler', 'version': None},
-            'reporter': {'_stacktrace': [], 'comments': None, OS_ENVIRON: {}, 'sysinfo': None},
+            'reporter': {
+                '_stacktrace': [],
+                '_stacktrace_context': [],
+                '_stacktrace_extra': [],
+                'comments': None,
+                OS_ENVIRON: {},
+                'sysinfo': {},
+                'events': {},
+            },
         },
         'tags': {'machine': None, 'os': None, 'platform': None, PLATFORM_DETAILS: None, 'version': None},
     }
@@ -61,38 +69,46 @@ def test_send():
         "timestamp": 42,
         "sysinfo": '',
         "comments": 'comment',
-        "stack": 'some\nstack',
+        "stack": 'l1\nl2--LONG TEXT--l3\nl4',
     }
 
     assert SentryReporter.send_event({'a': 'b'}, post_data, None) == {
         'a': 'b',
         'contexts': {
             'browser': {'name': 'Tribler', 'version': '0.0.0'},
-            'reporter': {'_stacktrace': ['some', 'stack'], 'comments': 'comment', 'os.environ': {}, 'sysinfo': None},
+            'reporter': {
+                '_stacktrace': ['l1', 'l2'],
+                '_stacktrace_context': [],
+                '_stacktrace_extra': ['l3', 'l4'],
+                'comments': 'comment',
+                'os.environ': {},
+                'sysinfo': {},
+                'events': {},
+            },
         },
         'tags': {'machine': 'x86_64', 'os': 'posix', 'platform': None, 'platform.details': None, 'version': '0.0.0'},
     }
 
-    # test sys_info
-    post_data = {"sysinfo": 'key\tvalue\nkey1\tvalue1\n'}
-
-    assert SentryReporter.send_event({}, post_data, None) == {
-        'contexts': {
-            'browser': {'name': 'Tribler', 'version': None},
-            'reporter': {'_stacktrace': [], 'comments': None, 'os.environ': {}, 'sysinfo': None},
-        },
-        'tags': {'machine': None, 'os': None, 'platform': None, 'platform.details': None, 'version': None},
+    sys_info = {
+        'platform': ['darwin'],
+        'platform.details': ['details'],
+        OS_ENVIRON: ['KEY:VALUE', 'KEY1:VALUE1'],
+        'event_1': [{'type': ''}],
+        'request_1': [{}],
+        'event_2': [],
+        'request_2': [],
     }
-
-    sys_info = {'platform': ['darwin'], 'platform.details': ['details'], OS_ENVIRON: ['KEY:VALUE', 'KEY1:VALUE1']}
     assert SentryReporter.send_event({}, None, sys_info) == {
         'contexts': {
             'browser': {'name': 'Tribler', 'version': None},
             'reporter': {
                 '_stacktrace': [],
+                '_stacktrace_context': [],
+                '_stacktrace_extra': [],
                 'comments': None,
-                'os.environ': {'KEY': 'VALUE', 'KEY1': 'VALUE1'},
-                'sysinfo': sys_info,
+                OS_ENVIRON: {'KEY': 'VALUE', 'KEY1': 'VALUE1'},
+                'sysinfo': {'platform': ['darwin'], 'platform.details': ['details']},
+                'events': {'event_1': [{'type': ''}], 'request_1': [{}], 'event_2': [], 'request_2': []},
             },
         },
         'tags': {'machine': None, 'os': None, 'platform': 'darwin', 'platform.details': 'details', 'version': None},

src/tribler-common/tribler_common/sentry_reporter/tests/test_sentry_tools.py

@@ -1,6 +1,7 @@
 from tribler_common.sentry_reporter.sentry_tools import (
     delete_item,
     distinct_by,
+    extract_dict,
     format_version,
     get_first_item,
     get_last_item,
@@ -63,11 +64,24 @@ def test_parse_os_environ():
 
 
 def test_parse_stacktrace():
-    assert parse_stacktrace(None) == []
-    assert parse_stacktrace('') == []
-    assert parse_stacktrace('\n') == []
-    assert parse_stacktrace('\n\n') == []
-    assert parse_stacktrace('some\n\nvalue') == ['some', 'value']
+    assert list(parse_stacktrace(None)) == []
+    assert list(parse_stacktrace('')) == []
+    assert list(parse_stacktrace('\n')) == [[]]
+    assert list(parse_stacktrace('\n\n')) == [[]]
+    assert list(parse_stacktrace('line1\n\nline2\\nline3')) == [['line1', 'line2', 'line3']]
+
+    # split --LONG TEXT-- and --CONTEXT-- parts
+    assert list(parse_stacktrace('l1\nl2--LONG TEXT--l3\nl4--CONTEXT--l5\nl6')) == [
+        ['l1', 'l2'],
+        ['l3', 'l4'],
+        ['l5', 'l6'],
+    ]
+
+    # split custom parts
+    assert list(parse_stacktrace('l1\nl2customl3\nl4', delimiters=['custom'])) == [
+        ['l1', 'l2'],
+        ['l3', 'l4'],
+    ]
 
 
 def test_modify():
@@ -118,3 +132,10 @@ def test_skip_dev_version():
     # experimental versions
     assert format_version('7.7.1-exp1-1-abcd ') == '7.7.1-exp1'
     assert format_version('7.7.1-someresearchtopic-7-abcd ') == '7.7.1-someresearchtopic'
+
+
+def test_extract_dict():
+    assert not extract_dict(None, None)
+
+    assert extract_dict({}, '') == {}
+    assert extract_dict({'k': 'v', 'k1': 'v1'}, r'\w$') == {'k': 'v'}