From ed4b50ffc2e4fe232734c9110d6b69051b21a9fa Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Thu, 19 Aug 2021 17:55:16 +0300
Subject: [PATCH 1/6] Allow to redirect logs

---
 .yamato/script/log_scripts/log_parser.py      | 173 +++++-----------
 .../utils/execution_log_patterns.py           | 186 +++++++++++-------
 .../script/log_scripts/utils/shared_utils.py  |  43 ++++
 .../log_scripts/utils/utr_log_patterns.py     |  58 ++++--
 4 files changed, 244 insertions(+), 216 deletions(-)
 create mode 100644 .yamato/script/log_scripts/utils/shared_utils.py

diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py
index 37781528199..88bdec303c2 100644
--- a/.yamato/script/log_scripts/log_parser.py
+++ b/.yamato/script/log_scripts/log_parser.py
@@ -1,13 +1,14 @@
 
 import argparse
 import os
-import sys
 import requests
+import sys
 import json
 import glob
 import re
-from utils.execution_log_patterns import execution_log_patterns
-from utils.utr_log_patterns import utr_log_patterns
+from utils.execution_log_patterns import Execution_log
+from utils.utr_log_patterns import UTR_log
+from utils.shared_utils import *
 
 '''
 This script runs for extended Yamato reporting. It
@@ -30,55 +31,9 @@
     --execution-log "<path to execution log file>"
 '''
 
-def load_json(file_path):
-    with open(file_path) as f:
-        json_data = f.readlines()
-        json_data[0] = json_data[0].replace('let testData = ','') # strip the beginning
-        json_data[-1] = json_data[-1][:-1] # strip the ; from end
-        return json.loads(' '.join(json_data))
-        #return json.load(f)
-
-def get_execution_log():
-    '''Returns the path to execution log file.'''
-    path_to_execution_log = os.path.join(os.path.dirname(os.path.dirname(os.getcwd())),'Execution-*.log')
-    print('Searching for logs in: ', path_to_execution_log)
-
-    execution_log_file = glob.glob(path_to_execution_log)[0]
-    print('Reading log: ', execution_log_file)
-    return execution_log_file
-
-
-def read_execution_log(log_file_path):
-    '''Reads execution logs and returns:
-    logs: dictionary with keys corresponding to commands, and values containing log output and status
-    job_succeeded: boolean indicating if the job succeeded
-    '''
-
-    with open(log_file_path, encoding='utf-8') as f:
-        lines = [l.replace('\n','') for l in f.readlines() if l != '\n'] # remove empty lines and all newline indicators
-
-    # all log line idx starting/ending a new command
-    command_idxs = [i for i,line  in enumerate(lines) if '################################### Running next command ###################################' in line]
-    command_idxs_end = [i for i,line  in enumerate(lines) if '############################################################################################' in line]
-    command_idxs.append(len(lines)) # add dummy idx to handle the last command
-
-    # get output (list of lines) for each command
-    logs = {}
-    for i, command_idx in enumerate(command_idxs):
-        if command_idx == len(lines):
-            break
-        command = '\n'.join(lines[command_idx+1: command_idxs_end[i]])
-        output = lines[command_idx+3: command_idxs[i+1]-1]
-        logs[command] = {}
-        logs[command]['output'] = output
-        logs[command]['status'] = 'Failed' if any("Command failed" in line for line in output) else 'Success'
-
-    # if the command block succeeded overall
-    overall_status = [line for line in lines if 'Commands finished with result:' in line][0].split(']')[1].split(': ')[1]
-    job_succeeded = False if 'Failed' in overall_status else True
-    return logs, job_succeeded
-
-def parse_failures(logs, local):
+
+
+def parse_failures(execution_log, logs, local):
     '''Parses each command in the execution log (and possibly UTR logs),
     recognizes any known errors, and posts additional data to Yamato.'''
     for cmd in logs.keys():
@@ -87,101 +42,61 @@ def parse_failures(logs, local):
         # TODO: do we also want to add additional yamato results for these?
         if ((logs[cmd]['status'] == 'Success' and not any("Retrying" in line for line in logs[cmd]['output']))
                 or any("Reason(s): One or more tests have failed." in line for line in logs[cmd]['output'])):
-            print('Skipping: ', cmd)
             continue
 
-        print('\nReading: ', cmd)
-
-        # check if the error matches any known pattern marked in log_patterns.py
-        output = '\n'.join(logs[cmd]['output'])
-        matching_patterns, tags, conclusion = find_matching_patterns(execution_log_patterns, output)
+        print('\nFound failed command: ', cmd, '\n')
 
+        # initialize command data
         logs[cmd]['title'] = cmd
-        logs[cmd]['conclusion'] = conclusion
-        logs[cmd]['tags'] = tags
-        logs[cmd]['summary'] = ''
-        for pattern,match in matching_patterns: # update the command log with input from the matched pattern
-            logs[cmd]['summary'] +=  match.group(0) + ' | ' if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. '
+        logs[cmd]['conclusion'] = []
+        logs[cmd]['tags'] = []
+        logs[cmd]['summary'] = []
 
-
-
-        # if it is an UTR non-test related error message not shown in Execution log but in test-results, append that to summary
-        if  'non-test' in logs[cmd]['tags']:
-            test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
-            test_results_path = test_results_match[1] + test_results_match[2]
-            # utr_failures, utr_tags, utr_conclusion = read_hoarder_log(os.path.join(test_results_path,'HoarderData.json'))
-            utr_failures, utr_tags, utr_conclusion = read_test_results_json(os.path.join(test_results_path,'TestResults.json'))
-            logs[cmd]['summary'] += utr_failures + ' | '
-            logs[cmd]['tags'].extend(utr_tags)
-            logs[cmd]['conclusion'] = utr_conclusion
+        # check if the error matches any known pattern marked in log_patterns.py, fill the command data for each match
+        cmd_output = '\n'.join(logs[cmd]['output'])
+        recursively_match_patterns(logs, cmd, execution_log.get_patterns(), cmd_output)
 
         # post additional results to Yamato
         post_additional_results(logs[cmd], local)
     return
 
 
-def find_matching_patterns(patterns, failure_string):
-    '''Finds a matching pattern from a specified list of pattern objects for a specified failure string (e.g. command output).
-    Returns the matching pattern object, and the matched substring.'''
-    matches = []
-    for pattern in patterns:
-            match = re.search(pattern['pattern'], failure_string)
-            if match:
-
-                # if a matching patterns was found, skip the general unknown pattern
-                if len(matches) > 0 and pattern['pattern'] == '.+':
-                    continue
-
-                print('Found match for pattern: ',  pattern['pattern'])
-                matches.append((pattern, match))
+def recursively_match_patterns(logs, cmd, patterns, failure_string):
+    matches = find_matching_patterns(patterns, failure_string)
+    for pattern, match in matches:
 
-    tags = [pattern['tags'] for (pattern,_) in matches]
-    flattened_tags = [tag for tag_list in tags for tag in tag_list]
+        logs[cmd]['conclusion'].append(pattern['conclusion'])
+        logs[cmd]['tags'].append(pattern['tags'])
+        logs[cmd]['summary'].append(match.group(0) if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. ')
 
-    conclusion = get_ruling_conclusion([pattern['conclusion'] for (pattern,_) in matches])
-
-    return matches, flattened_tags, conclusion
-
-
-def get_ruling_conclusion(conclusions):
-    '''Pick a single conclusion out of several matches in the order of severity'''
-    if 'failure' in conclusions:
-        return 'failure'
-    elif 'inconclusive' in conclusions:
-        return 'inconclusive'
-    elif 'cancelled' in conclusions:
-        return 'cancelled'
-    elif 'success' in conclusions:
-        return 'success'
-    else:
-        return 'failure'
-
-
-def read_test_results_json(log_file_path):
-    '''Reads error messages from TestResults.json.'''
-    logs = load_json(log_file_path)
-    error_logs = [log for log in logs if log.get('rootCause')]
-    if len(error_logs) > 0:
-        failure_reasons = ' '.join(error_logs[0].get('errors',['']))
-
-        _, tags, conclusion = find_matching_patterns(utr_log_patterns, failure_reasons)
-        return failure_reasons, tags, conclusion
-    else:
-        return [], [], ''
+        if pattern.get('direct_further'):
+            for direct_further in pattern['direct_further']:
+                if direct_further == 'utr_log':
+                    test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
+                    test_results_path = test_results_match[1] + test_results_match[2]
+                    df = UTR_log(test_results_path)
+                    recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                else:
+                    print('! Invalid redirect: ', direct_further)
 
 
 def post_additional_results(cmd, local):
-    '''Posts additional results to Yamato reporting server'''
+    '''Posts additional results to Yamato reporting server:
+        - title: command itself
+        - summary: concatenated summary of all matched patterns, each capped at 500 char
+        - tags: non-duplicate tags of all matched patterns
+        - conclusion: most severe conclusion of all matched patterns
+        '''
 
     data = {
         'title': cmd['title'],
-        'summary': cmd['summary'][:500] + '...',
-        'conclusion': cmd['conclusion'],
-        'tags' : list(set(cmd['tags']))
+        'summary': ' | '.join([s[:500] for s in cmd['summary']]),
+        'conclusion': get_ruling_conclusion(cmd['conclusion']),
+        'tags' : list(set(flatten_tags(cmd['tags'])))
     }
 
     if local:
-        print('Posting: ', json.dumps(data,indent=2), '\n')
+        print('\nPosting: ', json.dumps(data,indent=2), '\n')
     else:
         server_url = os.environ['YAMATO_REPORTING_SERVER'] + '/result'
         headers = {'Content-Type':'application/json'}
@@ -203,12 +118,12 @@ def main(argv):
     try:
         args = parse_args(argv)
 
-        # read execution logs
-        execution_log_file = get_execution_log() if not args.execution_log else args.execution_log
-        logs, job_succeeded = read_execution_log(execution_log_file)
+        # read execution log
+        execution_log = Execution_log(args.execution_log)
+        logs, job_succeeded = execution_log.read_log()
 
         if not job_succeeded:
-            parse_failures(logs, args.local)
+            parse_failures(execution_log, logs, args.local)
 
     except Exception as e:
         print('Failed to parse logs: ', str(e))
diff --git a/.yamato/script/log_scripts/utils/execution_log_patterns.py b/.yamato/script/log_scripts/utils/execution_log_patterns.py
index 1dc52a402be..b605f401657 100644
--- a/.yamato/script/log_scripts/utils/execution_log_patterns.py
+++ b/.yamato/script/log_scripts/utils/execution_log_patterns.py
@@ -1,70 +1,118 @@
-# Contains patterns to match in the execution log
-#
-# Conclusion can be either: success, failure, cancelled, inconclusive.
-# Conclusions of utr_log_patterns overwrite conclusions of execution_log_patterns.
-# Tags of utr_log_patterns get appended to tags of execution_log_patterns.
+import os
+import glob
+from .utr_log_patterns import UTR_log
+from .shared_utils import load_json, find_matching_patterns
 
-execution_log_patterns = [
-    # Order: retry blocks must be matched first
-    # If either retry pattern is matched, the failures are further parsed to match any specific failure patterns written below
-    {
-        # This is matched if all retries fail.
-        'pattern': r'(Failed after)(.+)(retries)',
-        'tags': ['retry'],
-        'conclusion': 'failure',
-    },
-    {
-        # This matches both successful/failed retries.
-        # Successful retries have no concrete pattern to match: the only difference with failed retry is that it does not contain 'Failed after n retries',
-        # but no working regex for matching multiline against a negative lookahead was found yet.
-        # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry)
-        'pattern': r'(Retrying)',
-        'tags': ['retry'],
-        'conclusion': 'success',
-    },
-    # Order: patterns below can be in any order, and the script can match multiple patterns
-    {
-        'pattern': r'(command not found)',
-        'tags': ['failure'],
-        'conclusion': 'failure',
-    },
-    {
-        #  Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)',
-        'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)',
-        'tags': ['packet_write_poll','instability'],
-        'conclusion': 'inconclusive',
-    },
-    {
-        # Or: r'(LTO : error: L0496: error during communication with the LTO process: The pipe has been ended)'
-        'pattern': r'(orbis-ld stderr :LLVM ERROR: out of memory)((.|\n)+)(LLVM ERROR: out of memory)',
-        'tags': ['oom','instability'],
-        'conclusion': 'failure',
-    },
-    {
-        'pattern': r'(fatal: not a git repository (or any of the parent directories): .git)',
-        'tags': ['git'],
-        'conclusion': 'failure',
-    },
-    {
-        'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)',
-        'tags': ['instability', 'bad-allocation'],
-        'conclusion': 'failure',
-    },
-    {
-        'pattern': r'Reason\(s\): One or more tests have failed.', # this one is unused right now since yamato does it automatically
-        'tags': ['tests'],
-        'conclusion': 'failure',
-    },
-    {
-        'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', # if hit this, read hoarder file
-        'tags': ['non-test'],
-        'conclusion': 'failure',
-    },
-    # Order: this matches everything and must therefore be the last item in the list
-    # If any previous pattern has been matched, this one is skipped
-    {
-        'pattern': r'.+',
-        'tags': ['unknown'],
-        'conclusion': 'failure',
-    }
-]
+
+class Execution_log():
+    '''Handles log parsing and error matching of the execution log'''
+
+    def __init__(self, path_to_log=""):
+        self.path = glob.glob(os.path.join(os.path.dirname(os.path.dirname(os.getcwd())),'Execution-*.log'))[0] if path_to_log=="" else path_to_log
+        self.patterns = self.get_patterns()
+
+    def get_patterns(self):
+        '''Returns error patterns to match against. Each pattern has:
+        pattern: regex to match some string against
+        tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ...
+        conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)'''
+        return [
+            # Order: retry blocks must be matched first
+            # If either retry pattern is matched, the failures are further parsed to match any specific failure patterns written below
+            {
+                # This is matched if all retries fail.
+                'pattern': r'(Failed after)(.+)(retries)',
+                'tags': ['retry'],
+                'conclusion': 'failure',
+            },
+            {
+                # This matches both successful/failed retries.
+                # Successful retries have no concrete pattern to match: the only difference with failed retry is that it does not contain 'Failed after n retries',
+                # but no working regex for matching multiline against a negative lookahead was found yet.
+                # Therefore, this pattern must come after failed retry pattern (python logic will handle recognizing this block as a successful retry)
+                'pattern': r'(Retrying)',
+                'tags': ['retry'],
+                'conclusion': 'success',
+            },
+            # Order: patterns below can be in any order, and the script can match multiple patterns
+            {
+                'pattern': r'(command not found)',
+                'tags': ['failure'],
+                'conclusion': 'failure',
+            },
+            {
+                #  Or with newlines: r'(packet_write_poll: Connection to)((.|\n)+)(Operation not permitted)((.|\n)+)(lost connection)',
+                'pattern': r'(packet_write_poll: Connection to)(.+)(Operation not permitted)',
+                'tags': ['packet_write_poll','instability'],
+                'conclusion': 'inconclusive',
+            },
+            {
+                # Or: r'(LTO : error: L0496: error during communication with the LTO process: The pipe has been ended)'
+                'pattern': r'(orbis-ld stderr :LLVM ERROR: out of memory)((.|\n)+)(LLVM ERROR: out of memory)',
+                'tags': ['oom','instability'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'(fatal: not a git repository (or any of the parent directories): .git)',
+                'tags': ['git'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'(LTO : error: L0492: LTOP internal error: bad allocation)',
+                'tags': ['instability', 'bad-allocation'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'Reason\(s\): One or more tests have failed.', # this one is unused right now since yamato does it automatically
+                'tags': ['tests'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', # if hit this, read hoarder file
+                'tags': ['non-test'],
+                'conclusion': 'failure',
+                'direct_further': [
+                    'utr_log'
+                ]
+            },
+            # Order: this matches everything and must therefore be the last item in the list
+            # If any previous pattern has been matched, this one is skipped
+            {
+                'pattern': r'.+',
+                'tags': ['unknown'],
+                'conclusion': 'failure',
+            }
+        ]
+
+    def read_log(self):
+        '''Reads execution logs and returns:
+        logs: dictionary with keys corresponding to commands, and values containing log output and status
+        job_succeeded: boolean indicating if the job succeeded
+        '''
+
+        with open(self.path, encoding='utf-8') as f:
+            lines = [l.replace('\n','') for l in f.readlines() if l != '\n'] # remove empty lines and all newline indicators
+
+        # after block index
+        after_idx = [i for i,line  in enumerate(lines) if 'Starting After Block' in line][0]
+
+        # all log line idx starting/ending a new command
+        cmd_idxs = [i for i,line  in enumerate(lines) if '################################### Running next command ###################################' in line]
+        cmd_idxs_end = [i for i,line  in enumerate(lines) if '############################################################################################' in line]
+        cmd_idxs.append(len(lines)) # add dummy idx to handle the last command
+
+        # get output (list of lines) for each command
+        logs = {}
+        for i, cmd_idx in enumerate(cmd_idxs):
+            if cmd_idx == len(lines) or cmd_idx >= after_idx:
+                break
+            cmd = '\n'.join(lines[cmd_idx+1: cmd_idxs_end[i]])
+            output = lines[cmd_idx+3: cmd_idxs[i+1]-1]
+            logs[cmd] = {}
+            logs[cmd]['output'] = output
+            logs[cmd]['status'] = 'Failed' if any("Command failed" in line for line in output) else 'Success'
+
+        # if the command block succeeded overall
+        overall_status = [line for line in lines if 'Commands finished with result:' in line][0].split(']')[1].split(': ')[1]
+        job_succeeded = False if 'Failed' in overall_status else True
+        return logs, job_succeeded
diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py
new file mode 100644
index 00000000000..3aa328b81fc
--- /dev/null
+++ b/.yamato/script/log_scripts/utils/shared_utils.py
@@ -0,0 +1,43 @@
+import json
+import re
+
+def load_json(file_path):
+    with open(file_path) as f:
+        json_data = f.readlines()
+        json_data[0] = json_data[0].replace('let testData = ','') # strip the beginning
+        json_data[-1] = json_data[-1][:-1] # strip the ; from end
+        return json.loads(' '.join(json_data))
+        #return json.load(f)
+
+def find_matching_patterns(patterns, failure_string):
+    '''Finds a matching pattern from a specified list of pattern objects for a specified failure string (e.g. command output).
+    Returns the matching pattern object, and the matched substring.'''
+    matches = []
+    for pattern in patterns:
+        match = re.search(pattern['pattern'], failure_string)
+        if match:
+
+            # if a matching patterns was found, skip the general unknown pattern
+            if len(matches) > 0 and pattern['pattern'] == '.+':
+                continue
+
+            print('Found match for pattern: ',  pattern['pattern'])
+            matches.append((pattern, match))
+    return matches
+
+def flatten_tags(tags):
+    '''Tags param: 2d arr of tags gathered from patterns. Returns a 1d arr.'''
+    return [tag for tag_list in tags for tag in tag_list]
+
+def get_ruling_conclusion(conclusions):
+    '''Pick a single conclusion out of several matches in the order of severity'''
+    if 'failure' in conclusions:
+        return 'failure'
+    elif 'inconclusive' in conclusions:
+        return 'inconclusive'
+    elif 'cancelled' in conclusions:
+        return 'cancelled'
+    elif 'success' in conclusions:
+        return 'success'
+    else:
+        return 'failure'
diff --git a/.yamato/script/log_scripts/utils/utr_log_patterns.py b/.yamato/script/log_scripts/utils/utr_log_patterns.py
index 77502e04cd8..e583cc62d00 100644
--- a/.yamato/script/log_scripts/utils/utr_log_patterns.py
+++ b/.yamato/script/log_scripts/utils/utr_log_patterns.py
@@ -1,20 +1,42 @@
-# Contains patterns to match errors in the hoarder json suites.failureReasons in case of non-test related errors
-#
-# Conclusion can be either: success, failure, cancelled, inconclusive.
-# Conclusions of utr_log_patterns overwrite conclusions of execution_log_patterns.
-# Tags of utr_log_patterns get appended to tags of execution_log_patterns.
+import os
+from .shared_utils import load_json, find_matching_patterns
 
-utr_log_patterns = [
-    {
-        'pattern': r'System.TimeoutException: Timeout while waiting',
-        'tags': ['System.TimeoutException'],
-        'conclusion': 'failure',
-    },
-    {
-        # this matches everything and must therefore be the last item in the list
-        'pattern': r'.+',
-        'tags': ['unknown'],
-        'conclusion': 'failure',
-    }
+class UTR_log():
+    ''''Handles parsing UTR logs (TestResults.json) against known error patterns'''
 
-]
+    def __init__(self, path_to_log):
+        self.path = os.path.join(path_to_log,'TestResults.json')
+        self.patterns = self.get_patterns()
+
+    def get_patterns(self):
+        '''Returns error patterns to match against. Each pattern has:
+        pattern: regex to match some string against
+        tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ...
+        conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)'''
+        return [
+            {
+                'pattern': r'System.TimeoutException: Timeout while waiting',
+                'tags': ['System.TimeoutException'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'System.AggregateException: One or more errors occurred. \(Detected that ios-deploy is not running when attempting to establish player connection.\)',
+                'tags': ['System.AggregateException', 'ios-deploy'],
+                'conclusion': 'failure',
+            },
+            {
+                # this matches everything and must therefore be the last item in the list
+                'pattern': r'.+',
+                'tags': ['unknown'],
+                'conclusion': 'failure',
+            }
+        ]
+
+    def read_log(self):
+        '''Returns the string output, which is then matched against known patterns.
+        In this case, the output corresponds to the error messages of the object marked as rootcause in TestResults.json'''
+        logs = load_json(self.path)
+        error_logs = [log for log in logs if log.get('rootCause')]
+        if len(error_logs) > 0:
+            return ' '.join(error_logs[0].get('errors',['']))
+        return ''

From b79d0a30bd420ec55ca4dc9d71b71aa6df93b58e Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Thu, 19 Aug 2021 18:06:04 +0300
Subject: [PATCH 2/6] Some small fixes

---
 .yamato/script/log_scripts/log_parser.py      | 22 +++++++++----------
 ...ution_log_patterns.py => execution_log.py} |  3 +--
 .../utils/{utr_log_patterns.py => utr_log.py} |  0
 3 files changed, 11 insertions(+), 14 deletions(-)
 rename .yamato/script/log_scripts/utils/{execution_log_patterns.py => execution_log.py} (98%)
 rename .yamato/script/log_scripts/utils/{utr_log_patterns.py => utr_log.py} (100%)

diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py
index 88bdec303c2..1eb832de88f 100644
--- a/.yamato/script/log_scripts/log_parser.py
+++ b/.yamato/script/log_scripts/log_parser.py
@@ -6,24 +6,20 @@
 import json
 import glob
 import re
-from utils.execution_log_patterns import Execution_log
-from utils.utr_log_patterns import UTR_log
+from utils.execution_log import Execution_log
+from utils.utr_log import UTR_log
 from utils.shared_utils import *
 
 '''
 This script runs for extended Yamato reporting. It
 1. Parses the execution log and for each command and its output (except successful non-retries and failed tests)
-    - finds matching patterns from execution_log_patterns
+    - finds matching patterns from execution_log.py patterns
+    - for each matched pattern, recursively finds matches also for any pattern with 'redirect' specified
     - sends to Yamato extended reporting server
         title: command itself
         summary: substring(s) of the command output matching the specified pattern(s)
         tags: all (distinct) tags beloging to the matched pattern(s)
         conclusion: failure/inconclusive/cancelled/success, which applies to the parsed command (not the whole job status)
-2. In case non-test related failure from UTR is matched, the script also reads TestResults.json and
-    - finds matching patterns from utr_log_patterns
-    - extends the data sent to Yamato extended reporting server in the previous point, by
-        extending the summary with matched substrings from the utr pattern and by
-        appending the utr pattern tags to the list of previous tags
 
 By default, the script requires no parameters and uses default execution log location in Yamato.
 To run it locally, specify
@@ -62,6 +58,8 @@ def parse_failures(execution_log, logs, local):
 
 
 def recursively_match_patterns(logs, cmd, patterns, failure_string):
+    '''Match the given string against any known patterns. If any of the patterns contains a 'redirect',
+    parse also the directed log in a recursive fashion.'''
     matches = find_matching_patterns(patterns, failure_string)
     for pattern, match in matches:
 
@@ -69,15 +67,15 @@ def recursively_match_patterns(logs, cmd, patterns, failure_string):
         logs[cmd]['tags'].append(pattern['tags'])
         logs[cmd]['summary'].append(match.group(0) if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. ')
 
-        if pattern.get('direct_further'):
-            for direct_further in pattern['direct_further']:
-                if direct_further == 'utr_log':
+        if pattern.get('redirect'):
+            for redirect in pattern['redirect']:
+                if redirect == 'utr_log':
                     test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
                     test_results_path = test_results_match[1] + test_results_match[2]
                     df = UTR_log(test_results_path)
                     recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
                 else:
-                    print('! Invalid redirect: ', direct_further)
+                    print('! Invalid redirect: ', redirect)
 
 
 def post_additional_results(cmd, local):
diff --git a/.yamato/script/log_scripts/utils/execution_log_patterns.py b/.yamato/script/log_scripts/utils/execution_log.py
similarity index 98%
rename from .yamato/script/log_scripts/utils/execution_log_patterns.py
rename to .yamato/script/log_scripts/utils/execution_log.py
index b605f401657..efe0ed9067c 100644
--- a/.yamato/script/log_scripts/utils/execution_log_patterns.py
+++ b/.yamato/script/log_scripts/utils/execution_log.py
@@ -1,6 +1,5 @@
 import os
 import glob
-from .utr_log_patterns import UTR_log
 from .shared_utils import load_json, find_matching_patterns
 
 
@@ -71,7 +70,7 @@ def get_patterns(self):
                 'pattern': r'Reason\(s\): One or more non-test related errors or failures occurred.', # if hit this, read hoarder file
                 'tags': ['non-test'],
                 'conclusion': 'failure',
-                'direct_further': [
+                'redirect': [
                     'utr_log'
                 ]
             },
diff --git a/.yamato/script/log_scripts/utils/utr_log_patterns.py b/.yamato/script/log_scripts/utils/utr_log.py
similarity index 100%
rename from .yamato/script/log_scripts/utils/utr_log_patterns.py
rename to .yamato/script/log_scripts/utils/utr_log.py

From 29cad5629a21c64b6f81df96db6ff85a56a654be Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Thu, 19 Aug 2021 18:23:38 +0300
Subject: [PATCH 3/6] Add parsing for Unity log

---
 .yamato/script/log_scripts/log_parser.py      | 11 ++++-
 .../script/log_scripts/utils/execution_log.py |  3 +-
 .yamato/script/log_scripts/utils/unity_log.py | 40 +++++++++++++++++++
 3 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 .yamato/script/log_scripts/utils/unity_log.py

diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py
index 1eb832de88f..d7e9d3b1f40 100644
--- a/.yamato/script/log_scripts/log_parser.py
+++ b/.yamato/script/log_scripts/log_parser.py
@@ -8,6 +8,7 @@
 import re
 from utils.execution_log import Execution_log
 from utils.utr_log import UTR_log
+from utils.unity_log import Unity_log
 from utils.shared_utils import *
 
 '''
@@ -68,12 +69,18 @@ def recursively_match_patterns(logs, cmd, patterns, failure_string):
         logs[cmd]['summary'].append(match.group(0) if pattern['tags'][0] != 'unknown' else 'Unknown failure: check logs for more details. ')
 
         if pattern.get('redirect'):
+            test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
+            test_results_path = test_results_match[1] + test_results_match[2]
             for redirect in pattern['redirect']:
+
                 if redirect == 'utr_log':
-                    test_results_match = re.findall(r'(--artifacts_path=)(.+)(test-results)', cmd)[0]
-                    test_results_path = test_results_match[1] + test_results_match[2]
                     df = UTR_log(test_results_path)
                     recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+
+                elif redirect == 'unity_log':
+                    df = Unity_log(test_results_path)
+                    recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+
                 else:
                     print('! Invalid redirect: ', redirect)
 
diff --git a/.yamato/script/log_scripts/utils/execution_log.py b/.yamato/script/log_scripts/utils/execution_log.py
index efe0ed9067c..29c82b6ae82 100644
--- a/.yamato/script/log_scripts/utils/execution_log.py
+++ b/.yamato/script/log_scripts/utils/execution_log.py
@@ -71,7 +71,8 @@ def get_patterns(self):
                 'tags': ['non-test'],
                 'conclusion': 'failure',
                 'redirect': [
-                    'utr_log'
+                    'utr_log',
+                    'unity_log'
                 ]
             },
             # Order: this matches everything and must therefore be the last item in the list
diff --git a/.yamato/script/log_scripts/utils/unity_log.py b/.yamato/script/log_scripts/utils/unity_log.py
new file mode 100644
index 00000000000..3b709ad37e7
--- /dev/null
+++ b/.yamato/script/log_scripts/utils/unity_log.py
@@ -0,0 +1,40 @@
+import os
+import glob
+from .shared_utils import load_json, find_matching_patterns
+
+class Unity_log():
+    ''''Handles parsing Unity log (UnityLog.txt) against known error patterns'''
+
+    def __init__(self, path_to_log):
+        self.path = glob.glob(os.path.join(path_to_log,"*/**/",'UnityLog.txt'))[0]
+        self.patterns = self.get_patterns()
+
+    def get_patterns(self):
+        '''Returns error patterns to match against. Each pattern has:
+        pattern: regex to match some string against
+        tags: tags to be added to Yamato additional results, typically one as identifier, and one as category such as instability, ...
+        conclusion: success/failure/cancelled/inconclusive (if many patterns are matched for a command, most severe is chosen in the end)'''
+        return [
+            {
+                'pattern': r'TcpProtobufSession::SendMessageAsync',
+                'tags': ['network','instability'],
+                'conclusion': 'failure',
+            },
+            {
+                'pattern': r'AcceleratorClientConnectionCallback - disconnected - cacheserver-slo',
+                'tags': ['cache','instability'],
+                'conclusion': 'failure',
+            },
+            {
+                # this matches everything and must therefore be the last item in the list
+                'pattern': r'.+',
+                'tags': ['unknown'],
+                'conclusion': 'failure',
+            }
+        ]
+
+    def read_log(self):
+        '''Returns the string output, which is then matched against known patterns.
+        In this case, returns the whole UnityLog.txt contents.'''
+        with open(self.path, encoding='utf-8') as f:
+            return f.read()

From 321f0f1960344c58020ee12558e18c94f3c706cf Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Mon, 23 Aug 2021 15:45:58 +0200
Subject: [PATCH 4/6] Fix recognizing unknown patterns

---
 .yamato/script/log_scripts/utils/shared_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.yamato/script/log_scripts/utils/shared_utils.py b/.yamato/script/log_scripts/utils/shared_utils.py
index 3aa328b81fc..cd2851a3983 100644
--- a/.yamato/script/log_scripts/utils/shared_utils.py
+++ b/.yamato/script/log_scripts/utils/shared_utils.py
@@ -18,7 +18,7 @@ def find_matching_patterns(patterns, failure_string):
         if match:
 
             # if a matching patterns was found, skip the general unknown pattern
-            if len(matches) > 0 and pattern['pattern'] == '.+':
+            if len(matches) > 0 and not all([p['tags'][0]=='retry' for p,m in matches]) and pattern['pattern'] == '.+':
                 continue
 
             print('Found match for pattern: ',  pattern['pattern'])

From 1c910deef29e907127b20ce0dcd1a8f60a085a63 Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Mon, 23 Aug 2021 16:51:26 +0200
Subject: [PATCH 5/6] Add try-catch to reading logs

---
 .yamato/script/log_scripts/log_parser.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/.yamato/script/log_scripts/log_parser.py b/.yamato/script/log_scripts/log_parser.py
index d7e9d3b1f40..7caedf4944d 100644
--- a/.yamato/script/log_scripts/log_parser.py
+++ b/.yamato/script/log_scripts/log_parser.py
@@ -74,12 +74,17 @@ def recursively_match_patterns(logs, cmd, patterns, failure_string):
             for redirect in pattern['redirect']:
 
                 if redirect == 'utr_log':
-                    df = UTR_log(test_results_path)
-                    recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
-
+                    try:
+                        df = UTR_log(test_results_path)
+                        recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                    except Exception as e:
+                        print('! Failed to parse UTR TestResults.json: ', e)
                 elif redirect == 'unity_log':
-                    df = Unity_log(test_results_path)
-                    recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                    try:
+                        df = Unity_log(test_results_path)
+                        recursively_match_patterns(logs, cmd, df.get_patterns(), df.read_log())
+                    except Exception as e:
+                        print('! Failed to parse UnityLog.txt', e)
 
                 else:
                     print('! Invalid redirect: ', redirect)
@@ -95,7 +100,7 @@ def post_additional_results(cmd, local):
 
     data = {
         'title': cmd['title'],
-        'summary': ' | '.join([s[:500] for s in cmd['summary']]),
+        'summary': ' | '.join(list(set([s[:500] for s in cmd['summary']]))),
         'conclusion': get_ruling_conclusion(cmd['conclusion']),
         'tags' : list(set(flatten_tags(cmd['tags'])))
     }

From 6abc7d0cf5996c21ca320569a2048474bd5f9750 Mon Sep 17 00:00:00 2001
From: Liis Kivistik <liis.kivistik@unity3d.com>
Date: Tue, 24 Aug 2021 13:24:47 +0200
Subject: [PATCH 6/6] Move strings to vars

---
 .yamato/script/log_scripts/utils/constants.py     |  4 ++++
 .yamato/script/log_scripts/utils/execution_log.py | 14 +++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)
 create mode 100644 .yamato/script/log_scripts/utils/constants.py

diff --git a/.yamato/script/log_scripts/utils/constants.py b/.yamato/script/log_scripts/utils/constants.py
new file mode 100644
index 00000000000..29d350efc30
--- /dev/null
+++ b/.yamato/script/log_scripts/utils/constants.py
@@ -0,0 +1,4 @@
+# Following constants correspond to which log parser class to use
+UTR_LOG = 'utr_log'
+UNITY_LOG = 'unity_log'
+EXECUTION_LOG = 'execution_log'
diff --git a/.yamato/script/log_scripts/utils/execution_log.py b/.yamato/script/log_scripts/utils/execution_log.py
index 29c82b6ae82..69728add774 100644
--- a/.yamato/script/log_scripts/utils/execution_log.py
+++ b/.yamato/script/log_scripts/utils/execution_log.py
@@ -1,7 +1,11 @@
 import os
 import glob
 from .shared_utils import load_json, find_matching_patterns
+from .constants import *
 
+COMMAND_START = '################################### Running next command ###################################'
+COMMAND_END = '############################################################################################'
+AFTER_BLOCK_START = 'Starting After Block'
 
 class Execution_log():
     '''Handles log parsing and error matching of the execution log'''
@@ -71,8 +75,8 @@ def get_patterns(self):
                 'tags': ['non-test'],
                 'conclusion': 'failure',
                 'redirect': [
-                    'utr_log',
-                    'unity_log'
+                    UTR_LOG,
+                    UNITY_LOG
                 ]
             },
             # Order: this matches everything and must therefore be the last item in the list
@@ -94,11 +98,11 @@ def read_log(self):
             lines = [l.replace('\n','') for l in f.readlines() if l != '\n'] # remove empty lines and all newline indicators
 
         # after block index
-        after_idx = [i for i,line  in enumerate(lines) if 'Starting After Block' in line][0]
+        after_idx = [i for i,line  in enumerate(lines) if AFTER_BLOCK_START in line][0]
 
         # all log line idx starting/ending a new command
-        cmd_idxs = [i for i,line  in enumerate(lines) if '################################### Running next command ###################################' in line]
-        cmd_idxs_end = [i for i,line  in enumerate(lines) if '############################################################################################' in line]
+        cmd_idxs = [i for i,line  in enumerate(lines) if COMMAND_START in line]
+        cmd_idxs_end = [i for i,line  in enumerate(lines) if COMMAND_END in line]
         cmd_idxs.append(len(lines)) # add dummy idx to handle the last command
 
         # get output (list of lines) for each command