Fix instability detection tagging

.yamato/script/log_scripts/log_parser.py

@@ -4,7 +4,7 @@
 import requests
 import sys
 import json
-import glob
+import traceback
 import re
 from utils.execution_log import Execution_log
 from utils.utr_log import UTR_log
@@ -34,56 +34,61 @@
 def parse_failures(execution_log, logs, local):
     '''Parses each command in the execution log (and possibly UTR logs),
     recognizes any known errors, and posts additional data to Yamato.'''
-    for cmd in logs.keys():
+    for cmd_key in logs.keys():
+        cmd = logs[cmd_key]
 
         # skip parsing successful commands which have not retried, or failed tests (these get automatically parsed in yamato results)
         # TODO: do we also want to add additional yamato results for these?
-        if ((logs[cmd]['status'] == 'Success' and not logs[cmd]['retry'])
-                or any("Reason(s): One or more tests have failed." in line for line in logs[cmd]['output'])):
+        if ((cmd['status'] == 'Success' and not cmd['retry'])
+                or any("Reason(s): One or more tests have failed." in line for line in cmd['output'])):
             continue
 
-        print('\nFound failed or retried command: ', cmd, '\n')
+        print('\nFound failed or retried command: ', cmd_key, '\n')
 
         # initialize command data
-        logs[cmd]['title'] = cmd
-        logs[cmd]['conclusion'] = []
-        logs[cmd]['tags'] = []
-        logs[cmd]['summary'] = []
+        cmd['title'] = cmd_key
+        cmd['conclusion'] = []
+        cmd['tags'] = []
+        cmd['summary'] = []
 
         # check if the error matches any known pattern marked in log_patterns.py, fill the command data for each match
-        cmd_output = '\n'.join(logs[cmd]['output'])
-        recursively_match_patterns(logs, cmd, execution_log.get_patterns(), cmd_output)
+        cmd_output = '\n'.join(cmd['output'])
+        recursively_match_patterns(cmd, execution_log.get_patterns(), cmd_output) # find all amtches
+        cmd['tags'] = format_tags(cmd['tags']) # flatten and remove duplicates
+
+        # add unknown pattern if nothing else matched
+        add_unknown_pattern_if_appropriate(cmd)
 
         # post additional results to Yamato
-        post_additional_results(logs[cmd], local)
+        post_additional_results(cmd, local)
     return
 
 
-def recursively_match_patterns(logs, cmd, patterns, failure_string):
+def recursively_match_patterns(cmd, patterns, failure_string):
     '''Match the given string against any known patterns. If any of the patterns contains a 'redirect',
     parse also the directed log in a recursive fashion.'''
     matches = find_matching_patterns(patterns, failure_string)
     for pattern, match in matches:
 
-        logs[cmd]['conclusion'].append(pattern['conclusion'])
-        logs[cmd]['tags'].append(pattern['tags'])
-        logs[cmd]['summary'].append(match.group(0) if pattern['pattern'] != '.+' else 'Unknown failure: check logs for more details. ')
+        cmd['conclusion'].append(pattern['conclusion'])
+        cmd['tags'].append(pattern['tags'])
+        cmd['summary'].append(match.group(0))
 
         if pattern.get('redirect'):
-            test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
+            test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd['title'])[0]
             test_results_path = test_results_match[1] + test_results_match[2]
             for redirect in pattern['redirect']:
 
                 if redirect == UTR_LOG:
                     try:
                         df = UTR_log(test_results_path)
-                        recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                        recursively_match_patterns(cmd, df.get_patterns(), df.read_log())
                     except Exception as e:
                         print(f'! Failed to parse UTR TestResults.json: ', str(e))
                 elif redirect == UNITY_LOG:
                     try:
                         df = Unity_log(test_results_path)
-                        recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                        recursively_match_patterns(cmd, df.get_patterns(), df.read_log())
                     except Exception as e:
                         print(f'! Failed to parse UnityLog.txt', str(e))
 
@@ -102,12 +107,12 @@ def post_additional_results(cmd, local):
     data = {
         'title': cmd['title'],
         'summary': ' | '.join(list(set([s[:500] for s in cmd['summary']]))),
-        'conclusion': get_ruling_conclusion(cmd['conclusion']),
-        'tags' : list(set(flatten_tags(cmd['tags'])))
+        'conclusion': get_ruling_conclusion(cmd['conclusion'], cmd['tags']),
+        'tags' : cmd['tags']
     }
 
     if local:
-        print('\nPosting: ', json.dumps(data,indent=2), '\n')
+        print('\nPosting: ', json.dumps(data, indent=2), '\n')
     else:
         server_url = os.environ['YAMATO_REPORTING_SERVER'] + '/result'
         headers = {'Content-Type':'application/json'}
@@ -137,7 +142,8 @@ def main(argv):
             parse_failures(execution_log, logs, args.local)
 
     except Exception as e:
-        print('Failed to parse logs: ', str(e))
+        print('\nFailed to parse logs')
+        traceback.print_exc()
 
 
 if __name__ == '__main__':

.yamato/script/log_scripts/utils/constants.py

@@ -2,3 +2,9 @@
 UTR_LOG = 'utr_log'
 UNITY_LOG = 'unity_log'
 EXECUTION_LOG = 'execution_log'
+
+
+# tags
+TAG_SUCCESFUL_RETRY = 'successful-retry'
+TAG_INSTABILITY = 'instability'
+TAG_INFRASTRUCTURE = 'infrastructure'

.yamato/script/log_scripts/utils/execution_log.py

@@ -35,15 +35,15 @@ def get_patterns(self):
                 # but no working regex for matching multiline against a negative lookahead was found yet.
                 # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry)
                 'pattern': r'(Retrying)',
-                'tags': ['successful-retry'],
+                'tags': [TAG_SUCCESFUL_RETRY],
                 'conclusion': 'success',
                 'add_if': add_successful_retry_if
             },
             # Order: patterns below can be in any order, and the script can match multiple patterns
             {
                 #  Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)',
                 'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)',
-                'tags': ['packet_write_poll','instability', 'infrastructure'],
+                'tags': ['packet_write_poll',TAG_INSTABILITY, TAG_INFRASTRUCTURE],
                 'conclusion': 'failure',
             },
             {
@@ -59,7 +59,7 @@ def get_patterns(self):
             },
             {
                 'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)',
-                'tags': ['instability', 'bad-allocation', 'infrastructure'],
+                'tags': [TAG_INSTABILITY, 'bad-allocation', TAG_INFRASTRUCTURE],
                 'conclusion': 'failure',
             },
             {
@@ -75,14 +75,6 @@ def get_patterns(self):
                     UTR_LOG,
                     UNITY_LOG
                 ]
-            },
-            # Order: this matches everything and must therefore be the last item in the list
-            # If any previous pattern has been matched, this one is skipped
-            {
-                'pattern': r'.+',
-                'tags': ['unknown'],
-                'conclusion': 'failure',
-                'add_if': add_unknown_pattern_if
             }
         ]
 
@@ -96,7 +88,8 @@ def read_log(self):
             lines = [l.replace('\n','') for l in f.readlines() if l != '\n'] # remove empty lines and all newline indicators
 
         # after block index
-        after_idx = [i for i,line  in enumerate(lines) if AFTER_BLOCK_START in line][0]
+        after_idxs = [i for i,line  in enumerate(lines) if AFTER_BLOCK_START in line]
+        after_idx = after_idxs[0] if len(after_idxs) > 0 else len(lines)
 
         # all log line idx starting/ending a new command
         cmd_idxs = [i for i,line  in enumerate(lines) if COMMAND_START in line]

.yamato/script/log_scripts/utils/rules.py

@@ -1,13 +1,3 @@
-def add_unknown_pattern_if(matches):
-    '''Only add unknown pattern if no other pattern has been matched yet,
-    i.e. skip if no matches are found, or only matches indicate a retry.'''
-    if len(matches) == 0:
-        return True
-    elif len(matches) == 1:
-        if ('successful-retry' in matches[0][0]['tags'] or 'failed after' in matches[0][0]['pattern'].lower()):
-            return True
-    return False
-
 
 def add_successful_retry_if(matches):
     '''Add only if failed retry has not matched'''

.yamato/script/log_scripts/utils/shared_utils.py

@@ -1,5 +1,6 @@
 import json
 import re
+from .constants import *
 
 def load_json(file_path):
     with open(file_path) as f:
@@ -26,13 +27,19 @@ def find_matching_patterns(patterns, failure_string):
             matches.append((pattern, match))
     return matches
 
-def flatten_tags(tags):
-    '''Tags param: 2d arr of tags gathered from patterns. Returns a 1d arr.'''
-    return [tag for tag_list in tags for tag in tag_list]
+def format_tags(tags):
+    '''Flattens tags, removes duplicates, removes TAG_INSTABILITY if retry was successful
+     (latter is because we need to have either one or another, we cannot have both, so that we can distinguish them by tags)'''
+    tags = sorted(list(set([tag for tag_list in tags for tag in tag_list]))) # flatten and remove duplicates
+    if TAG_INSTABILITY in tags and TAG_SUCCESFUL_RETRY in tags:
+        tags.remove(TAG_INSTABILITY)
+    return tags
 
-def get_ruling_conclusion(conclusions):
+def get_ruling_conclusion(conclusions, tags):
     '''Pick a single conclusion out of several matches in the order of severity'''
-    if 'failure' in conclusions:
+    if TAG_SUCCESFUL_RETRY in tags:
+        return 'success'
+    elif 'failure' in conclusions:
         return 'failure'
     elif 'inconclusive' in conclusions:
         return 'inconclusive'
@@ -42,3 +49,13 @@ def get_ruling_conclusion(conclusions):
         return 'success'
     else:
         return 'failure'
+
+def add_unknown_pattern_if_appropriate(cmd):
+    '''Adds an unknown failure pattern if no patterns were matched at all, or only successful retry was matched.'''
+
+    if (len(cmd['tags']) == 0 # no pattern matched at all
+        or (len(cmd['tags']) == 1 and TAG_SUCCESFUL_RETRY in cmd['tags'])): # only successful retry pattern matched
+
+        cmd['conclusion'].append('failure')
+        cmd['tags'].append('unknown')
+        cmd['summary'].append( 'Unknown failure: check logs for more details. ')

.yamato/script/log_scripts/utils/unity_log.py

@@ -2,6 +2,7 @@
 import glob
 from .shared_utils import load_json, find_matching_patterns
 from .rules import *
+from .constants import *
 
 class Unity_log():
     ''''Handles parsing Unity log (UnityLog.txt) against known error patterns'''
@@ -19,21 +20,14 @@ def get_patterns(self):
             # {
             #       # commented out as this should always come paired with cache instability below
             #     'pattern': r'TcpProtobufSession::SendMessageAsync',
-            #     'tags': ['TcpProtobufSession', 'instability', 'infrastructure'],
+            #     'tags': ['TcpProtobufSession', TAG_INSTABILITY, TAG_INFRASTRUCTURE],
             #     'conclusion': 'failure',
             # },
             {
                 'pattern': r'AcceleratorClientConnectionCallback - disconnected - cacheserver-slo',
-                'tags': ['cache', 'instability', 'infrastructure'],
+                'tags': ['cache', TAG_INSTABILITY, TAG_INFRASTRUCTURE],
                 'conclusion': 'failure',
             },
-            {
-                # this matches everything and must therefore be the last item in the list
-                'pattern': r'.+',
-                'tags': ['unknown'],
-                'conclusion': 'failure',
-                'add_if': add_unknown_pattern_if
-            }
         ]
 
     def read_log(self):

.yamato/script/log_scripts/utils/utr_log.py

@@ -2,6 +2,7 @@
 from .shared_utils import load_json, find_matching_patterns
 from .shared_utils import *
 from .rules import *
+from .constants import *
 
 class UTR_log():
     ''''Handles parsing UTR logs (TestResults.json) against known error patterns'''
@@ -24,15 +25,8 @@ def get_patterns(self):
             # },
             {
                 'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)',
-                'tags': ['ios-deploy', 'infrastructure'], # instability?
+                'tags': ['ios-deploy', TAG_INFRASTRUCTURE], # instability?
                 'conclusion': 'failure',
-            },
-            {
-                # this matches everything and must therefore be the last item in the list
-                'pattern': r'.+',
-                'tags': ['unknown'],
-                'conclusion': 'failure',
-                'add_if': add_unknown_pattern_if
             }
         ]