diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py index 95b714c0509..ed6adaf1f28 100644 --- a/.yamato/script/log_scripts/log_parser.py +++ b/.yamato/script/log_scripts/log_parser.py @@ -38,11 +38,11 @@ def parse_failures(execution_log, logs, local): # skip parsing successful commands which have not retried, or failed tests (these get automatically parsed in yamato results) # TODO: do we also want to add additional yamato results for these? - if ((logs[cmd]['status'] == 'Success' and not any("Retrying" in line for line in logs[cmd]['output'])) + if ((logs[cmd]['status'] == 'Success' and not logs[cmd]['retry']) or any("Reason(s): One or more tests have failed." in line for line in logs[cmd]['output'])): continue - print('\nFound failed command: ', cmd, '\n') + print('\nFound failed or retried command: ', cmd, '\n') # initialize command data logs[cmd]['title'] = cmd @@ -67,7 +67,7 @@ def recursively_match_patterns(logs, cmd, patterns, failure_string): logs[cmd]['conclusion'].append(pattern['conclusion']) logs[cmd]['tags'].append(pattern['tags']) - logs[cmd]['summary'].append(match.group(0) if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. ') + logs[cmd]['summary'].append(match.group(0) if pattern['pattern'] != '.+' else 'Unknown failure: check logs for more details. ') if pattern.get('redirect'): test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0] @@ -118,7 +118,7 @@ def post_additional_results(cmd, local): def parse_args(argv): parser = argparse.ArgumentParser() - parser.add_argument("--execution-log", required=False, help='Path to execution log file. If not specified, ../../Execution-*.log is used.', default=None) + parser.add_argument("--execution-log", required=False, help='Path to execution log file. If not specified, ../../Execution-*.log is used.', default="") parser.add_argument("--local", action='store_true', help='If specified, API call to post additional results is skipped.', default=False) args = parser.parse_args(argv) return args @@ -131,9 +131,9 @@ def main(argv): # read execution log execution_log = Execution_log(args.execution_log) - logs, job_succeeded = execution_log.read_log() + logs, should_be_parsed = execution_log.read_log() - if not job_succeeded: + if should_be_parsed: parse_failures(execution_log, logs, args.local) except Exception as e: diff --git a/.yamato/script/log_scripts/utils/execution_log.py b/.yamato/script/log_scripts/utils/execution_log.py index 69728add774..92e4d19fc64 100644 --- a/.yamato/script/log_scripts/utils/execution_log.py +++ b/.yamato/script/log_scripts/utils/execution_log.py @@ -2,6 +2,7 @@ import glob from .shared_utils import load_json, find_matching_patterns from .constants import * +from .rules import * COMMAND_START = '################################### Running next command ###################################' COMMAND_END = '############################################################################################' @@ -25,7 +26,7 @@ def get_patterns(self): { # This is matched if all retries fail. 'pattern': r'(Failed after)(.+)(retries)', - 'tags': ['retry'], + 'tags': [], 'conclusion': 'failure', }, { @@ -34,35 +35,31 @@ def get_patterns(self): # but no working regex for matching multiline against a negative lookahead was found yet. # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry) 'pattern': r'(Retrying)', - 'tags': ['retry'], + 'tags': ['successful-retry'], 'conclusion': 'success', + 'add_if': add_successful_retry_if }, # Order: patterns below can be in any order, and the script can match multiple patterns - { - 'pattern': r'(command not found)', - 'tags': ['failure'], - 'conclusion': 'failure', - }, { # Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)', 'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)', - 'tags': ['packet_write_poll','instability'], - 'conclusion': 'inconclusive', + 'tags': ['packet_write_poll','instability', 'infrastructure'], + 'conclusion': 'failure', }, { # Or: r'(LTO : error: L0496: error during communication with the LTO process: The pipe has been ended)' 'pattern': r'(orbis-ld stderr :LLVM ERROR: out of memory)((.|\n)+)(LLVM ERROR: out of memory)', - 'tags': ['oom','instability'], + 'tags': ['oom'], # instability? 'conclusion': 'failure', }, { 'pattern': r'(fatal: not a git repository (or any of the parent directories): .git)', - 'tags': ['git'], + 'tags': ['git'], # instability? 'conclusion': 'failure', }, { 'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)', - 'tags': ['instability', 'bad-allocation'], + 'tags': ['instability', 'bad-allocation', 'infrastructure'], 'conclusion': 'failure', }, { @@ -71,8 +68,8 @@ def get_patterns(self): 'conclusion': 'failure', }, { - 'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', # if hit this, read hoarder file - 'tags': ['non-test'], + 'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', + 'tags': [], 'conclusion': 'failure', 'redirect': [ UTR_LOG, @@ -85,6 +82,7 @@ def get_patterns(self): 'pattern': r'.+', 'tags': ['unknown'], 'conclusion': 'failure', + 'add_if': add_unknown_pattern_if } ] @@ -115,8 +113,11 @@ def read_log(self): logs[cmd] = {} logs[cmd]['output'] = output logs[cmd]['status'] = 'Failed' if any("Command failed" in line for line in output) else 'Success' + logs[cmd]['retry'] = any("Retrying" in line for line in output) # if the command block succeeded overall overall_status = [line for line in lines if 'Commands finished with result:' in line][0].split(']')[1].split(': ')[1] job_succeeded = False if 'Failed' in overall_status else True - return logs, job_succeeded + has_retries = any(logs[cmd]['retry'] for cmd in logs.keys()) + should_be_parsed = not job_succeeded or has_retries + return logs, should_be_parsed diff --git a/.yamato/script/log_scripts/utils/rules.py b/.yamato/script/log_scripts/utils/rules.py new file mode 100644 index 00000000000..134bd3a2365 --- /dev/null +++ b/.yamato/script/log_scripts/utils/rules.py @@ -0,0 +1,16 @@ +def add_unknown_pattern_if(matches): + '''Only add unknown pattern if no other pattern has been matched yet, + i.e. skip if no matches are found, or only matches indicate a retry.''' + if len(matches) == 0: + return True + elif len(matches) == 1: + if ('successful-retry' in matches[0][0]['tags'] or 'failed after' in matches[0][0]['pattern'].lower()): + return True + return False + + +def add_successful_retry_if(matches): + '''Add only if failed retry has not matched''' + if len(matches) == 1 and 'failed after' in matches[0][0]['pattern'].lower(): + return False + return True diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py index cd2851a3983..f447e3b9be9 100644 --- a/.yamato/script/log_scripts/utils/shared_utils.py +++ b/.yamato/script/log_scripts/utils/shared_utils.py @@ -17,9 +17,10 @@ def find_matching_patterns(patterns, failure_string): match = re.search(pattern['pattern'], failure_string) if match: - # if a matching patterns was found, skip the general unknown pattern - if len(matches) > 0 and not all([p['tags'][0]=='retry' for p,m in matches]) and pattern['pattern'] == '.+': - continue + # if a pattern is added conditionally, skip it if condition is not fulfilled + if pattern.get('add_if'): + if not pattern['add_if'](matches): + continue print('Found match for pattern: ', pattern['pattern']) matches.append((pattern, match)) diff --git a/.yamato/script/log_scripts/utils/unity_log.py b/.yamato/script/log_scripts/utils/unity_log.py index 3b709ad37e7..b4dcb89adf1 100644 --- a/.yamato/script/log_scripts/utils/unity_log.py +++ b/.yamato/script/log_scripts/utils/unity_log.py @@ -1,6 +1,7 @@ import os import glob from .shared_utils import load_json, find_matching_patterns +from .rules import * class Unity_log(): ''''Handles parsing Unity log (UnityLog.txt) against known error patterns''' @@ -15,14 +16,15 @@ def get_patterns(self): tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ... conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)''' return [ - { - 'pattern': r'TcpProtobufSession::SendMessageAsync', - 'tags': ['network','instability'], - 'conclusion': 'failure', - }, + # { + # # commented out as this should always come paired with cache instability below + # 'pattern': r'TcpProtobufSession::SendMessageAsync', + # 'tags': ['TcpProtobufSession', 'instability', 'infrastructure'], + # 'conclusion': 'failure', + # }, { 'pattern': r'AcceleratorClientConnectionCallback - disconnected - cacheserver-slo', - 'tags': ['cache','instability'], + 'tags': ['cache', 'instability', 'infrastructure'], 'conclusion': 'failure', }, { @@ -30,6 +32,7 @@ def get_patterns(self): 'pattern': r'.+', 'tags': ['unknown'], 'conclusion': 'failure', + 'add_if': add_unknown_pattern_if } ] diff --git a/.yamato/script/log_scripts/utils/utr_log.py b/.yamato/script/log_scripts/utils/utr_log.py index dc59271d643..4ccbb82dd21 100644 --- a/.yamato/script/log_scripts/utils/utr_log.py +++ b/.yamato/script/log_scripts/utils/utr_log.py @@ -1,6 +1,7 @@ import os from .shared_utils import load_json, find_matching_patterns from .shared_utils import * +from .rules import * class UTR_log(): ''''Handles parsing UTR logs (TestResults.json) against known error patterns''' @@ -15,14 +16,15 @@ def get_patterns(self): tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ... conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)''' return [ - { - 'pattern': r'System.TimeoutException: Timeout while waiting', - 'tags': ['System.TimeoutException'], - 'conclusion': 'failure', - }, + # { + # # commented out for now while we focus on instabilities only + # 'pattern': r'System.TimeoutException: Timeout while waiting', + # 'tags': ['System.TimeoutException'], + # 'conclusion': 'failure', + # }, { 'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)', - 'tags': ['System.AggregateException', 'ios-deploy'], + 'tags': ['ios-deploy', 'infrastructure'], # instability? 'conclusion': 'failure', }, { @@ -30,6 +32,7 @@ def get_patterns(self): 'pattern': r'.+', 'tags': ['unknown'], 'conclusion': 'failure', + 'add_if': add_unknown_pattern_if } ]