Update instability detection

.yamato/script/log_scripts/log_parser.py

@@ -38,11 +38,11 @@ def parse_failures(execution_log, logs, local):
 
         # skip parsing successful commands which have not retried, or failed tests (these get automatically parsed in yamato results)
         # TODO: do we also want to add additional yamato results for these?
-        if ((logs[cmd]['status'] == 'Success' and not any("Retrying" in line for line in logs[cmd]['output']))
+        if ((logs[cmd]['status'] == 'Success' and not logs[cmd]['retry'])
                 or any("Reason(s): One or more tests have failed." in line for line in logs[cmd]['output'])):
             continue
 
-        print('\nFound failed command: ', cmd, '\n')
+        print('\nFound failed or retried command: ', cmd, '\n')
 
         # initialize command data
         logs[cmd]['title'] = cmd
@@ -67,7 +67,7 @@ def recursively_match_patterns(logs, cmd, patterns, failure_string):
 
         logs[cmd]['conclusion'].append(pattern['conclusion'])
         logs[cmd]['tags'].append(pattern['tags'])
-        logs[cmd]['summary'].append(match.group(0) if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. ')
+        logs[cmd]['summary'].append(match.group(0) if pattern['pattern'] != '.+' else 'Unknown failure: check logs for more details. ')
 
         if pattern.get('redirect'):
             test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
@@ -118,7 +118,7 @@ def post_additional_results(cmd, local):
 
 def parse_args(argv):
     parser = argparse.ArgumentParser()
-    parser.add_argument("--execution-log", required=False, help='Path to execution log file. If not specified, ../../Execution-*.log is used.', default=None)
+    parser.add_argument("--execution-log", required=False, help='Path to execution log file. If not specified, ../../Execution-*.log is used.', default="")
     parser.add_argument("--local", action='store_true', help='If specified, API call to post additional results is skipped.', default=False)
     args = parser.parse_args(argv)
     return args
@@ -131,9 +131,9 @@ def main(argv):
 
         # read execution log
         execution_log = Execution_log(args.execution_log)
-        logs, job_succeeded = execution_log.read_log()
+        logs, should_be_parsed = execution_log.read_log()
 
-        if not job_succeeded:
+        if should_be_parsed:
             parse_failures(execution_log, logs, args.local)
 
     except Exception as e:

.yamato/script/log_scripts/utils/execution_log.py

@@ -2,6 +2,7 @@
 import glob
 from .shared_utils import load_json, find_matching_patterns
 from .constants import *
+from .rules import *
 
 COMMAND_START = '################################### Running next command ###################################'
 COMMAND_END = '############################################################################################'
@@ -25,7 +26,7 @@ def get_patterns(self):
             {
                 # This is matched if all retries fail.
                 'pattern': r'(Failed after)(.+)(retries)',
-                'tags': ['retry'],
+                'tags': [],
                 'conclusion': 'failure',
             },
             {
@@ -34,35 +35,31 @@ def get_patterns(self):
                 # but no working regex for matching multiline against a negative lookahead was found yet.
                 # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry)
                 'pattern': r'(Retrying)',
-                'tags': ['retry'],
+                'tags': ['successful-retry'],
                 'conclusion': 'success',
+                'add_if': add_successful_retry_if
             },
             # Order: patterns below can be in any order, and the script can match multiple patterns
-            {
-                'pattern': r'(command not found)',
-                'tags': ['failure'],
-                'conclusion': 'failure',
-            },
             {
                 #  Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)',
                 'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)',
-                'tags': ['packet_write_poll','instability'],
-                'conclusion': 'inconclusive',
+                'tags': ['packet_write_poll','instability', 'infrastructure'],
+                'conclusion': 'failure',
             },
             {
                 # Or: r'(LTO : error: L0496: error during communication with the LTO process: The pipe has been ended)'
                 'pattern': r'(orbis-ld stderr :LLVM ERROR: out of memory)((.|\n)+)(LLVM ERROR: out of memory)',
-                'tags': ['oom','instability'],
+                'tags': ['oom'], # instability?
                 'conclusion': 'failure',
             },
             {
                 'pattern': r'(fatal: not a git repository (or any of the parent directories): .git)',
-                'tags': ['git'],
+                'tags': ['git'], # instability?
                 'conclusion': 'failure',
             },
             {
                 'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)',
-                'tags': ['instability', 'bad-allocation'],
+                'tags': ['instability', 'bad-allocation', 'infrastructure'],
                 'conclusion': 'failure',
             },
             {
@@ -71,8 +68,8 @@ def get_patterns(self):
                 'conclusion': 'failure',
             },
             {
-                'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', # if hit this, read hoarder file
-                'tags': ['non-test'],
+                'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.',
+                'tags': [],
                 'conclusion': 'failure',
                 'redirect': [
                     UTR_LOG,
@@ -85,6 +82,7 @@ def get_patterns(self):
                 'pattern': r'.+',
                 'tags': ['unknown'],
                 'conclusion': 'failure',
+                'add_if': add_unknown_pattern_if
             }
         ]
 
@@ -115,8 +113,11 @@ def read_log(self):
             logs[cmd] = {}
             logs[cmd]['output'] = output
             logs[cmd]['status'] = 'Failed' if any("Command failed" in line for line in output) else 'Success'
+            logs[cmd]['retry'] = any("Retrying" in line for line in output)
 
         # if the command block succeeded overall
         overall_status = [line for line in lines if 'Commands finished with result:' in line][0].split(']')[1].split(': ')[1]
         job_succeeded = False if 'Failed' in overall_status else True
-        return logs, job_succeeded
+        has_retries = any(logs[cmd]['retry'] for cmd in logs.keys())
+        should_be_parsed = not job_succeeded or has_retries
+        return logs, should_be_parsed

.yamato/script/log_scripts/utils/rules.py

@@ -0,0 +1,16 @@
+def add_unknown_pattern_if(matches):
+    '''Only add unknown pattern if no other pattern has been matched yet,
+    i.e. skip if no matches are found, or only matches indicate a retry.'''
+    if len(matches) == 0:
+        return True
+    elif len(matches) == 1:
+        if ('successful-retry' in matches[0][0]['tags'] or 'failed after' in matches[0][0]['pattern'].lower()):
+            return True
+    return False
+
+
+def add_successful_retry_if(matches):
+    '''Add only if failed retry has not matched'''
+    if len(matches) == 1 and 'failed after' in matches[0][0]['pattern'].lower():
+        return False
+    return True

.yamato/script/log_scripts/utils/shared_utils.py

@@ -17,9 +17,10 @@ def find_matching_patterns(patterns, failure_string):
         match = re.search(pattern['pattern'], failure_string)
         if match:
 
-            # if a matching patterns was found, skip the general unknown pattern
-            if len(matches) > 0 and not all([p['tags'][0]=='retry' for p,m in matches]) and pattern['pattern'] == '.+':
-                continue
+            # if a pattern is added conditionally, skip it if condition is not fulfilled
+            if pattern.get('add_if'):
+                if not pattern['add_if'](matches):
+                    continue
 
             print('Found match for pattern: ',  pattern['pattern'])
             matches.append((pattern, match))

.yamato/script/log_scripts/utils/unity_log.py

@@ -1,6 +1,7 @@
 import os
 import glob
 from .shared_utils import load_json, find_matching_patterns
+from .rules import *
 
 class Unity_log():
     ''''Handles parsing Unity log (UnityLog.txt) against known error patterns'''
@@ -15,21 +16,23 @@ def get_patterns(self):
         tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ...
         conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)'''
         return [
-            {
-                'pattern': r'TcpProtobufSession::SendMessageAsync',
-                'tags': ['network','instability'],
-                'conclusion': 'failure',
-            },
+            # {
+            #       # commented out as this should always come paired with cache instability below
+            #     'pattern': r'TcpProtobufSession::SendMessageAsync',
+            #     'tags': ['TcpProtobufSession', 'instability', 'infrastructure'],
+            #     'conclusion': 'failure',
+            # },
             {
                 'pattern': r'AcceleratorClientConnectionCallback - disconnected - cacheserver-slo',
-                'tags': ['cache','instability'],
+                'tags': ['cache', 'instability', 'infrastructure'],
                 'conclusion': 'failure',
             },
             {
                 # this matches everything and must therefore be the last item in the list
                 'pattern': r'.+',
                 'tags': ['unknown'],
                 'conclusion': 'failure',
+                'add_if': add_unknown_pattern_if
             }
         ]
 

.yamato/script/log_scripts/utils/utr_log.py

@@ -1,6 +1,7 @@
 import os
 from .shared_utils import load_json, find_matching_patterns
 from .shared_utils import *
+from .rules import *
 
 class UTR_log():
     ''''Handles parsing UTR logs (TestResults.json) against known error patterns'''
@@ -15,21 +16,23 @@ def get_patterns(self):
         tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ...
         conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)'''
         return [
-            {
-                'pattern': r'System.TimeoutException: Timeout while waiting',
-                'tags': ['System.TimeoutException'],
-                'conclusion': 'failure',
-            },
+            # {
+            #     # commented out for now while we focus on instabilities only
+            #     'pattern': r'System.TimeoutException: Timeout while waiting',
+            #     'tags': ['System.TimeoutException'],
+            #     'conclusion': 'failure',
+            # },
             {
                 'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)',
-                'tags': ['System.AggregateException', 'ios-deploy'],
+                'tags': ['ios-deploy', 'infrastructure'], # instability?
                 'conclusion': 'failure',
             },
             {
                 # this matches everything and must therefore be the last item in the list
                 'pattern': r'.+',
                 'tags': ['unknown'],
                 'conclusion': 'failure',
+                'add_if': add_unknown_pattern_if
             }
         ]