From c0e068a26064921fc2b16615e15f6a6531fb4433 Mon Sep 17 00:00:00 2001 From: Liis Kivistik Date: Wed, 1 Sep 2021 16:08:13 +0200 Subject: [PATCH 1/3] Fix successful retries --- .yamato/script/log_scripts/log_parser.py | 7 +++++-- .yamato/script/log_scripts/utils/shared_utils.py | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py index ed6adaf1f28..7d513e8255d 100644 --- a/.yamato/script/log_scripts/log_parser.py +++ b/.yamato/script/log_scripts/log_parser.py @@ -54,6 +54,9 @@ def parse_failures(execution_log, logs, local): cmd_output = '\n'.join(logs[cmd]['output']) recursively_match_patterns(logs, cmd, execution_log.get_patterns(), cmd_output) + # format all added tags + logs[cmd]['tags'] = format_tags(logs[cmd]['tags']) + # post additional results to Yamato post_additional_results(logs[cmd], local) return @@ -102,8 +105,8 @@ def post_additional_results(cmd, local): data = { 'title': cmd['title'], 'summary': ' | '.join(list(set([s[:500] for s in cmd['summary']]))), - 'conclusion': get_ruling_conclusion(cmd['conclusion']), - 'tags' : list(set(flatten_tags(cmd['tags']))) + 'conclusion': get_ruling_conclusion(cmd['conclusion'], cmd['tags']), + 'tags' : cmd['tags'] } if local: diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py index f447e3b9be9..04382c03fd7 100644 --- a/.yamato/script/log_scripts/utils/shared_utils.py +++ b/.yamato/script/log_scripts/utils/shared_utils.py @@ -26,13 +26,18 @@ def find_matching_patterns(patterns, failure_string): matches.append((pattern, match)) return matches -def flatten_tags(tags): - '''Tags param: 2d arr of tags gathered from patterns. Returns a 1d arr.''' - return [tag for tag_list in tags for tag in tag_list] +def format_tags(tags): + '''Flattens tags, removes duplicates, removes 'instability' if retry was successful''' + tags = list(set([tag for tag_list in tags for tag in tag_list])) # flatten and remove duplicates + if 'instability' and 'successful-retry' in tags: + tags.remove('instability') + return tags -def get_ruling_conclusion(conclusions): +def get_ruling_conclusion(conclusions, tags): '''Pick a single conclusion out of several matches in the order of severity''' - if 'failure' in conclusions: + if 'successful-retry' in tags: + return 'success' + elif 'failure' in conclusions: return 'failure' elif 'inconclusive' in conclusions: return 'inconclusive' From 1dad8db575ae620350819cabd5548af6a4478302 Mon Sep 17 00:00:00 2001 From: Liis Kivistik Date: Wed, 1 Sep 2021 17:01:14 +0200 Subject: [PATCH 2/3] Fix unknown patterns --- .yamato/script/log_scripts/log_parser.py | 49 ++++++++++--------- .../script/log_scripts/utils/execution_log.py | 8 --- .yamato/script/log_scripts/utils/rules.py | 10 ---- .../script/log_scripts/utils/shared_utils.py | 13 ++++- .yamato/script/log_scripts/utils/unity_log.py | 7 --- .yamato/script/log_scripts/utils/utr_log.py | 7 --- 6 files changed, 38 insertions(+), 56 deletions(-) diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py index 7d513e8255d..0cb9c7ccd43 100644 --- a/.yamato/script/log_scripts/log_parser.py +++ b/.yamato/script/log_scripts/log_parser.py @@ -4,7 +4,7 @@ import requests import sys import json -import glob +import traceback import re from utils.execution_log import Execution_log from utils.utr_log import UTR_log @@ -34,59 +34,61 @@ def parse_failures(execution_log, logs, local): '''Parses each command in the execution log (and possibly UTR logs), recognizes any known errors, and posts additional data to Yamato.''' - for cmd in logs.keys(): + for cmd_key in logs.keys(): + cmd = logs[cmd_key] # skip parsing successful commands which have not retried, or failed tests (these get automatically parsed in yamato results) # TODO: do we also want to add additional yamato results for these? - if ((logs[cmd]['status'] == 'Success' and not logs[cmd]['retry']) - or any("Reason(s): One or more tests have failed." in line for line in logs[cmd]['output'])): + if ((cmd['status'] == 'Success' and not cmd['retry']) + or any("Reason(s): One or more tests have failed." in line for line in cmd['output'])): continue - print('\nFound failed or retried command: ', cmd, '\n') + print('\nFound failed or retried command: ', cmd_key, '\n') # initialize command data - logs[cmd]['title'] = cmd - logs[cmd]['conclusion'] = [] - logs[cmd]['tags'] = [] - logs[cmd]['summary'] = [] + cmd['title'] = cmd_key + cmd['conclusion'] = [] + cmd['tags'] = [] + cmd['summary'] = [] # check if the error matches any known pattern marked in log_patterns.py, fill the command data for each match - cmd_output = '\n'.join(logs[cmd]['output']) - recursively_match_patterns(logs, cmd, execution_log.get_patterns(), cmd_output) + cmd_output = '\n'.join(cmd['output']) + recursively_match_patterns(cmd, execution_log.get_patterns(), cmd_output) # find all amtches + cmd['tags'] = format_tags(cmd['tags']) # flatten and remove duplicates - # format all added tags - logs[cmd]['tags'] = format_tags(logs[cmd]['tags']) + # add unknown pattern if nothing else matched + add_unknown_pattern_if_appropriate(cmd) # post additional results to Yamato - post_additional_results(logs[cmd], local) + post_additional_results(cmd, local) return -def recursively_match_patterns(logs, cmd, patterns, failure_string): +def recursively_match_patterns(cmd, patterns, failure_string): '''Match the given string against any known patterns. If any of the patterns contains a 'redirect', parse also the directed log in a recursive fashion.''' matches = find_matching_patterns(patterns, failure_string) for pattern, match in matches: - logs[cmd]['conclusion'].append(pattern['conclusion']) - logs[cmd]['tags'].append(pattern['tags']) - logs[cmd]['summary'].append(match.group(0) if pattern['pattern'] != '.+' else 'Unknown failure: check logs for more details. ') + cmd['conclusion'].append(pattern['conclusion']) + cmd['tags'].append(pattern['tags']) + cmd['summary'].append(match.group(0)) if pattern.get('redirect'): - test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0] + test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd['title'])[0] test_results_path = test_results_match[1] + test_results_match[2] for redirect in pattern['redirect']: if redirect == UTR_LOG: try: df = UTR_log(test_results_path) - recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log()) + recursively_match_patterns(cmd, df.get_patterns(), df.read_log()) except Exception as e: print(f'! Failed to parse UTR TestResults.json: ', str(e)) elif redirect == UNITY_LOG: try: df = Unity_log(test_results_path) - recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log()) + recursively_match_patterns(cmd, df.get_patterns(), df.read_log()) except Exception as e: print(f'! Failed to parse UnityLog.txt', str(e)) @@ -110,7 +112,7 @@ def post_additional_results(cmd, local): } if local: - print('\nPosting: ', json.dumps(data,indent=2), '\n') + print('\nPosting: ', json.dumps(data, indent=2), '\n') else: server_url = os.environ['YAMATO_REPORTING_SERVER'] + '/result' headers = {'Content-Type':'application/json'} @@ -140,7 +142,8 @@ def main(argv): parse_failures(execution_log, logs, args.local) except Exception as e: - print('Failed to parse logs: ', str(e)) + print('\nFailed to parse logs') + traceback.print_exc() if __name__ == '__main__': diff --git a/.yamato/script/log_scripts/utils/execution_log.py b/.yamato/script/log_scripts/utils/execution_log.py index 92e4d19fc64..307b9769687 100644 --- a/.yamato/script/log_scripts/utils/execution_log.py +++ b/.yamato/script/log_scripts/utils/execution_log.py @@ -75,14 +75,6 @@ def get_patterns(self): UTR_LOG, UNITY_LOG ] - }, - # Order: this matches everything and must therefore be the last item in the list - # If any previous pattern has been matched, this one is skipped - { - 'pattern': r'.+', - 'tags': ['unknown'], - 'conclusion': 'failure', - 'add_if': add_unknown_pattern_if } ] diff --git a/.yamato/script/log_scripts/utils/rules.py b/.yamato/script/log_scripts/utils/rules.py index 134bd3a2365..c246d63cbd0 100644 --- a/.yamato/script/log_scripts/utils/rules.py +++ b/.yamato/script/log_scripts/utils/rules.py @@ -1,13 +1,3 @@ -def add_unknown_pattern_if(matches): - '''Only add unknown pattern if no other pattern has been matched yet, - i.e. skip if no matches are found, or only matches indicate a retry.''' - if len(matches) == 0: - return True - elif len(matches) == 1: - if ('successful-retry' in matches[0][0]['tags'] or 'failed after' in matches[0][0]['pattern'].lower()): - return True - return False - def add_successful_retry_if(matches): '''Add only if failed retry has not matched''' diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py index 04382c03fd7..e4619ea4970 100644 --- a/.yamato/script/log_scripts/utils/shared_utils.py +++ b/.yamato/script/log_scripts/utils/shared_utils.py @@ -27,7 +27,8 @@ def find_matching_patterns(patterns, failure_string): return matches def format_tags(tags): - '''Flattens tags, removes duplicates, removes 'instability' if retry was successful''' + '''Flattens tags, removes duplicates, removes 'instability' if retry was successful + (latter is because we need to have either one or another, we cannot have both, so that we can distinguish them by tags)''' tags = list(set([tag for tag_list in tags for tag in tag_list])) # flatten and remove duplicates if 'instability' and 'successful-retry' in tags: tags.remove('instability') @@ -47,3 +48,13 @@ def get_ruling_conclusion(conclusions, tags): return 'success' else: return 'failure' + +def add_unknown_pattern_if_appropriate(cmd): + '''Adds an unknown failure pattern if no patterns were matched at all, or only successful retry was matched.''' + + if (len(cmd['tags']) == 0 # no pattern matched at all + or (len(cmd['tags']) == 1 and 'successful-retry' in cmd['tags'])): # only successful retry pattern matched + + cmd['conclusion'].append('failure') + cmd['tags'].append('unknown') + cmd['summary'].append( 'Unknown failure: check logs for more details. ') diff --git a/.yamato/script/log_scripts/utils/unity_log.py b/.yamato/script/log_scripts/utils/unity_log.py index b4dcb89adf1..ef6e0e94047 100644 --- a/.yamato/script/log_scripts/utils/unity_log.py +++ b/.yamato/script/log_scripts/utils/unity_log.py @@ -27,13 +27,6 @@ def get_patterns(self): 'tags': ['cache', 'instability', 'infrastructure'], 'conclusion': 'failure', }, - { - # this matches everything and must therefore be the last item in the list - 'pattern': r'.+', - 'tags': ['unknown'], - 'conclusion': 'failure', - 'add_if': add_unknown_pattern_if - } ] def read_log(self): diff --git a/.yamato/script/log_scripts/utils/utr_log.py b/.yamato/script/log_scripts/utils/utr_log.py index 4ccbb82dd21..86b04010069 100644 --- a/.yamato/script/log_scripts/utils/utr_log.py +++ b/.yamato/script/log_scripts/utils/utr_log.py @@ -26,13 +26,6 @@ def get_patterns(self): 'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)', 'tags': ['ios-deploy', 'infrastructure'], # instability? 'conclusion': 'failure', - }, - { - # this matches everything and must therefore be the last item in the list - 'pattern': r'.+', - 'tags': ['unknown'], - 'conclusion': 'failure', - 'add_if': add_unknown_pattern_if } ] From 7d648754e0db35612b9e8f2788eae0649d85f48f Mon Sep 17 00:00:00 2001 From: Liis Kivistik Date: Wed, 1 Sep 2021 17:29:50 +0200 Subject: [PATCH 3/3] Move some tags to const --- .yamato/script/log_scripts/utils/constants.py | 6 ++++++ .yamato/script/log_scripts/utils/execution_log.py | 9 +++++---- .yamato/script/log_scripts/utils/shared_utils.py | 13 +++++++------ .yamato/script/log_scripts/utils/unity_log.py | 5 +++-- .yamato/script/log_scripts/utils/utr_log.py | 3 ++- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.yamato/script/log_scripts/utils/constants.py b/.yamato/script/log_scripts/utils/constants.py index 29d350efc30..a788f75bf95 100644 --- a/.yamato/script/log_scripts/utils/constants.py +++ b/.yamato/script/log_scripts/utils/constants.py @@ -2,3 +2,9 @@ UTR_LOG = 'utr_log' UNITY_LOG = 'unity_log' EXECUTION_LOG = 'execution_log' + + +# tags +TAG_SUCCESFUL_RETRY = 'successful-retry' +TAG_INSTABILITY = 'instability' +TAG_INFRASTRUCTURE = 'infrastructure' diff --git a/.yamato/script/log_scripts/utils/execution_log.py b/.yamato/script/log_scripts/utils/execution_log.py index 307b9769687..94cb1d7f921 100644 --- a/.yamato/script/log_scripts/utils/execution_log.py +++ b/.yamato/script/log_scripts/utils/execution_log.py @@ -35,7 +35,7 @@ def get_patterns(self): # but no working regex for matching multiline against a negative lookahead was found yet. # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry) 'pattern': r'(Retrying)', - 'tags': ['successful-retry'], + 'tags': [TAG_SUCCESFUL_RETRY], 'conclusion': 'success', 'add_if': add_successful_retry_if }, @@ -43,7 +43,7 @@ def get_patterns(self): { # Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)', 'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)', - 'tags': ['packet_write_poll','instability', 'infrastructure'], + 'tags': ['packet_write_poll',TAG_INSTABILITY, TAG_INFRASTRUCTURE], 'conclusion': 'failure', }, { @@ -59,7 +59,7 @@ def get_patterns(self): }, { 'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)', - 'tags': ['instability', 'bad-allocation', 'infrastructure'], + 'tags': [TAG_INSTABILITY, 'bad-allocation', TAG_INFRASTRUCTURE], 'conclusion': 'failure', }, { @@ -88,7 +88,8 @@ def read_log(self): lines = [l.replace('\n','') for l in f.readlines() if l != '\n'] # remove empty lines and all newline indicators # after block index - after_idx = [i for i,line in enumerate(lines) if AFTER_BLOCK_START in line][0] + after_idxs = [i for i,line in enumerate(lines) if AFTER_BLOCK_START in line] + after_idx = after_idxs[0] if len(after_idxs) > 0 else len(lines) # all log line idx starting/ending a new command cmd_idxs = [i for i,line in enumerate(lines) if COMMAND_START in line] diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py index e4619ea4970..4fe53888cef 100644 --- a/.yamato/script/log_scripts/utils/shared_utils.py +++ b/.yamato/script/log_scripts/utils/shared_utils.py @@ -1,5 +1,6 @@ import json import re +from .constants import * def load_json(file_path): with open(file_path) as f: @@ -27,16 +28,16 @@ def find_matching_patterns(patterns, failure_string): return matches def format_tags(tags): - '''Flattens tags, removes duplicates, removes 'instability' if retry was successful + '''Flattens tags, removes duplicates, removes TAG_INSTABILITY if retry was successful (latter is because we need to have either one or another, we cannot have both, so that we can distinguish them by tags)''' - tags = list(set([tag for tag_list in tags for tag in tag_list])) # flatten and remove duplicates - if 'instability' and 'successful-retry' in tags: - tags.remove('instability') + tags = sorted(list(set([tag for tag_list in tags for tag in tag_list]))) # flatten and remove duplicates + if TAG_INSTABILITY in tags and TAG_SUCCESFUL_RETRY in tags: + tags.remove(TAG_INSTABILITY) return tags def get_ruling_conclusion(conclusions, tags): '''Pick a single conclusion out of several matches in the order of severity''' - if 'successful-retry' in tags: + if TAG_SUCCESFUL_RETRY in tags: return 'success' elif 'failure' in conclusions: return 'failure' @@ -53,7 +54,7 @@ def add_unknown_pattern_if_appropriate(cmd): '''Adds an unknown failure pattern if no patterns were matched at all, or only successful retry was matched.''' if (len(cmd['tags']) == 0 # no pattern matched at all - or (len(cmd['tags']) == 1 and 'successful-retry' in cmd['tags'])): # only successful retry pattern matched + or (len(cmd['tags']) == 1 and TAG_SUCCESFUL_RETRY in cmd['tags'])): # only successful retry pattern matched cmd['conclusion'].append('failure') cmd['tags'].append('unknown') diff --git a/.yamato/script/log_scripts/utils/unity_log.py b/.yamato/script/log_scripts/utils/unity_log.py index ef6e0e94047..b05976bd044 100644 --- a/.yamato/script/log_scripts/utils/unity_log.py +++ b/.yamato/script/log_scripts/utils/unity_log.py @@ -2,6 +2,7 @@ import glob from .shared_utils import load_json, find_matching_patterns from .rules import * +from .constants import * class Unity_log(): ''''Handles parsing Unity log (UnityLog.txt) against known error patterns''' @@ -19,12 +20,12 @@ def get_patterns(self): # { # # commented out as this should always come paired with cache instability below # 'pattern': r'TcpProtobufSession::SendMessageAsync', - # 'tags': ['TcpProtobufSession', 'instability', 'infrastructure'], + # 'tags': ['TcpProtobufSession', TAG_INSTABILITY, TAG_INFRASTRUCTURE], # 'conclusion': 'failure', # }, { 'pattern': r'AcceleratorClientConnectionCallback - disconnected - cacheserver-slo', - 'tags': ['cache', 'instability', 'infrastructure'], + 'tags': ['cache', TAG_INSTABILITY, TAG_INFRASTRUCTURE], 'conclusion': 'failure', }, ] diff --git a/.yamato/script/log_scripts/utils/utr_log.py b/.yamato/script/log_scripts/utils/utr_log.py index 86b04010069..2ad95f3ad82 100644 --- a/.yamato/script/log_scripts/utils/utr_log.py +++ b/.yamato/script/log_scripts/utils/utr_log.py @@ -2,6 +2,7 @@ from .shared_utils import load_json, find_matching_patterns from .shared_utils import * from .rules import * +from .constants import * class UTR_log(): ''''Handles parsing UTR logs (TestResults.json) against known error patterns''' @@ -24,7 +25,7 @@ def get_patterns(self): # }, { 'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)', - 'tags': ['ios-deploy', 'infrastructure'], # instability? + 'tags': ['ios-deploy', TAG_INFRASTRUCTURE], # instability? 'conclusion': 'failure', } ]