-
Notifications
You must be signed in to change notification settings - Fork 965
/
output_checker.py
214 lines (188 loc) · 9.31 KB
/
output_checker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
import re
import traceback
from logging import getLogger
from galaxy.tools.parser.error_level import StdioErrorLevel
from galaxy.util import unicodify
from galaxy.util.bunch import Bunch
log = getLogger(__name__)
DETECTED_JOB_STATE = Bunch(
OK='ok',
OUT_OF_MEMORY_ERROR='oom_error',
GENERIC_ERROR='generic_error',
)
ERROR_PEAK = 2000
def check_output_regex(job_id_tag, regex, stream, stream_name, job_messages, max_error_level):
"""
check a single regex against a stream
regex the regex to check
stream the stream to search in
job_messages a list where the descriptions of the detected regexes can be appended
max_error_level the maximum error level that has been detected so far
returns the max of the error_level of the regex and the given max_error_level
"""
regex_match = re.search(regex.match, stream, re.IGNORECASE)
if regex_match:
reason = __regex_err_msg(regex_match, stream_name, regex)
job_messages.append(reason)
return max(max_error_level, regex.error_level)
return max_error_level
def check_output_regex_byline(job_id_tag, regex, stream, stream_append, max_error_level):
"""
check a single regex against a stream line by line, since errors
are expected to appear in the end of the stream we start with
the last line
returns the max of the error_level of the regex and the given max_error_level
"""
for line in reversed(stream.split("\n")):
regex_match = re.search(regex.match, line, re.IGNORECASE)
if regex_match:
rexmsg = __regex_err_msg(regex_match, regex)
log.info("Job %s: %s" % (job_id_tag, rexmsg))
stream_append.append(rexmsg)
return max(max_error_level, regex.error_level)
return max_error_level
def check_output(stdio_regexes, stdio_exit_codes, stdout, stderr, tool_exit_code, job_id_tag):
"""
Check the output of a tool - given the stdout, stderr, and the tool's
exit code, return DETECTED_JOB_STATE.OK if the tool exited succesfully or
error type otherwise. No exceptions should be thrown. If this code encounters
an exception, it returns OK so that the workflow can continue;
otherwise, a bug in this code could halt workflow progress.
Note that, if the tool did not define any exit code handling or
any stdio/stderr handling, then it reverts back to previous behavior:
if stderr contains anything, then False is returned.
"""
# By default, the tool succeeded. This covers the case where the code
# has a bug but the tool was ok, and it lets a workflow continue.
state = DETECTED_JOB_STATE.OK
stdout = unicodify(stdout)
stderr = unicodify(stderr)
# messages (descriptions of the detected exit_code and regexes)
# to be prepended to the stdout/stderr after all exit code and regex tests
# are done (otherwise added messages are searched again).
# messages are added it the order of detection
# If job is failed, track why.
job_messages = []
try:
# Check exit codes and match regular expressions against stdout and
# stderr if this tool was configured to do so.
# If there is a regular expression for scanning stdout/stderr,
# then we assume that the tool writer overwrote the default
# behavior of just setting an error if there is *anything* on
# stderr.
if len(stdio_regexes) > 0 or len(stdio_exit_codes) > 0:
# Check the exit code ranges in the order in which
# they were specified. Each exit_code is a StdioExitCode
# that includes an applicable range. If the exit code was in
# that range, then apply the error level and add a message.
# If we've reached a fatal error rule, then stop.
max_error_level = StdioErrorLevel.NO_ERROR
if tool_exit_code is not None:
for stdio_exit_code in stdio_exit_codes:
if (tool_exit_code >= stdio_exit_code.range_start and
tool_exit_code <= stdio_exit_code.range_end):
# Tack on a generic description of the code
# plus a specific code description. For example,
# this might prepend "Job 42: Warning (Out of Memory)\n".
code_desc = stdio_exit_code.desc
if None is code_desc:
code_desc = ""
desc = "%s: Exit code %d (%s)" % (
StdioErrorLevel.desc(stdio_exit_code.error_level),
tool_exit_code,
code_desc)
reason = {
'type': 'exit_code',
'desc': desc,
'exit_code': tool_exit_code,
'code_desc': code_desc,
'error_level': stdio_exit_code.error_level,
}
log.info("Job %s: %s" % (job_id_tag, reason))
job_messages.append(reason)
max_error_level = max(max_error_level,
stdio_exit_code.error_level)
if max_error_level >= StdioErrorLevel.MAX:
break
if max_error_level < StdioErrorLevel.FATAL_OOM:
# We'll examine every regex. Each regex specifies whether
# it is to be run on stdout, stderr, or both. (It is
# possible for neither stdout nor stderr to be scanned,
# but those regexes won't be used.) We record the highest
# error level, which are currently "warning" and "fatal".
# If fatal, then we set the job's state to ERROR.
# If warning, then we still set the job's state to OK
# but include a message. We'll do this if we haven't seen
# a fatal error yet
for regex in stdio_regexes:
# If ( this regex should be matched against stdout )
# - Run the regex's match pattern against stdout
# - If it matched, then determine the error level.
# o If it was fatal, then we're done - break.
if regex.stderr_match:
max_error_level = check_output_regex(job_id_tag, regex, stderr, 'stderr', job_messages, max_error_level)
if max_error_level >= StdioErrorLevel.MAX:
break
if regex.stdout_match:
max_error_level = check_output_regex(job_id_tag, regex, stdout, 'stdout', job_messages, max_error_level)
if max_error_level >= StdioErrorLevel.MAX:
break
# If we encountered a fatal error, then we'll need to set the
# job state accordingly. Otherwise the job is ok:
if max_error_level == StdioErrorLevel.FATAL_OOM:
state = DETECTED_JOB_STATE.OUT_OF_MEMORY_ERROR
elif max_error_level >= StdioErrorLevel.FATAL:
log.debug("Tool exit code indicates an error, failing job.")
state = DETECTED_JOB_STATE.GENERIC_ERROR
else:
state = DETECTED_JOB_STATE.OK
# When there are no regular expressions and no exit codes to check,
# default to the previous behavior: when there's anything on stderr
# the job has an error, and the job is ok otherwise.
else:
# TODO: Add in the tool and job id:
# log.debug( "Tool did not define exit code or stdio handling; "
# + "checking stderr for success" )
if stderr:
state = DETECTED_JOB_STATE.GENERIC_ERROR
else:
state = DETECTED_JOB_STATE.OK
if DETECTED_JOB_STATE != DETECTED_JOB_STATE.OK and stderr:
if stderr:
peak = stderr[0:ERROR_PEAK]
log.debug("job failed, standard error is - [%s]" % peak)
# On any exception, return True.
except Exception:
tb = traceback.format_exc()
log.warning("Tool check encountered unexpected exception; " +
"assuming tool was successful: " + tb)
state = DETECTED_JOB_STATE.OK
return state, stdout, stderr, job_messages
def __regex_err_msg(match, stream, regex):
"""
Return a message about the match on tool output using the given
ToolStdioRegex regex object. The regex_match is a MatchObject
that will contain the string matched on.
"""
# Get the description for the error level:
desc = StdioErrorLevel.desc(regex.error_level) + ": "
mstart = match.start()
mend = match.end()
if mend - mstart > 256:
match_str = match.string[mstart : mstart + 256] + "..."
else:
match_str = match.string[mstart: mend]
# If there's a description for the regular expression, then use it.
# Otherwise, we'll take the first 256 characters of the match.
if regex.desc is not None:
desc += regex.desc
else:
desc += "Matched on %s" % match_str
return {
"type": "regex",
"stream": stream,
"desc": desc,
"code_desc": regex.desc,
"match": match_str,
"error_level": regex.error_level,
}