Skip to content
Permalink
Browse files

Merge pull request #13437 from milljm/rework-failed-tests-13424

Properly restoring tester status methods
  • Loading branch information...
permcody committed Jun 3, 2019
2 parents 3ed1345 + e7f6ac8 commit ee93ae39a650f96bfee5d9838fc23870fea17d02
@@ -50,7 +50,7 @@ def getJobsAndAdvance(self):

# delete finished jobs
next_jobs = set([])
for job in list(self.__job_dag.ind_nodes()):
for job in self.__job_dag.ind_nodes():
if job.isFinished():
next_jobs.add(job)
self.__job_dag.delete_node(job)
@@ -103,12 +103,50 @@ def _doMakeDependencies(self):
except KeyError:
job.setStatus(job.error, 'unknown dependency')

def _hasDownStreamsWithFailures(self, job):
""" Return True if any dependents of job has previous failures """
for d_job in self.__job_dag.all_downstreams(job):
status, message, caveats = d_job.previousTesterStatus(self.options)
if status in d_job.job_status.getFailingStatuses():
return True

def _doPreviouslyFailed(self, job):
"""
Set up statuses for jobs contained within the DAG for use with failed-tests option
"""
tester = job.getTester()
status, message, caveats = job.previousTesterStatus(self.options)

# This job passed, but one of its dependents has not
if status == tester.success and self._hasDownStreamsWithFailures(job):
tester.setStatus(tester.success)
# Do we care? I figure, we should at least mention something. This job is not actually going to re-execute an app.
tester.addCaveats('previous results: {}'.format(status.status))
job.setStatus(job.finished)

# This job was skipped, passed or silent
elif status in job.job_status.getSuccessStatuses():
tester.setStatus(tester.silent)
job.setStatus(job.finished)

# Remaining independent 'skipped' jobs we don't want to print output for
elif not job.getRunnable():
tester.setStatus(tester.silent)
job.setStatus(job.finished)

# Remaining jobs are failures of some sort. Append the previous result as a caveat.
if message:
tester.addCaveats('previous results: {}'.format(message))

def _doSkippedDependencies(self):
""" Determine which jobs in the DAG should be skipped """
for job in list(self.__job_dag.topological_sort()):
tester = job.getTester()
for job in self.__job_dag.topological_sort():
dep_jobs = set([])
if not job.getRunnable() or self._haltDescent(job):

if self.options.failed_tests:
self._doPreviouslyFailed(job)

if not job.getRunnable() or job.isFail() or job.isSkip():
job.setStatus(job.skip)
dep_jobs.update(self.__job_dag.all_downstreams(job))

@@ -118,7 +156,7 @@ def _doSkippedDependencies(self):

for d_job in dep_jobs:
d_tester = d_job.getTester()
if tester.isSilent() and not d_job.getRunnable():
if job.isSilent() and not d_job.getRunnable():
d_tester.setStatus(d_tester.silent)
elif not self._skipPrereqs():
d_job.setStatus(d_job.skip)
@@ -156,17 +194,6 @@ def _doRaceConditions(self):
job.setOutput('Output file will over write pre-existing output file:\n\t%s\n' % (outfile))
job.setStatus(job.error, 'OUTFILE RACE CONDITION')

def _haltDescent(self, job):
""" return boolean if this job should not allow its children to run """
tester = job.getTester()
if (job.isError()
or job.isSkip()
or tester.isFail()
or tester.isSkip()
or tester.isSilent()
or tester.isDeleted()):
return True

def _skipPrereqs(self):
"""
Method to return boolean to skip dependency prerequisites checks.
@@ -87,6 +87,20 @@ class StatusSystem(object):
running,
finished]

__exit_nonzero_statuses = [fail,
diff,
deleted,
error,
timeout]

__exit_zero_statuses = [success,
skip,
silent]

__pending_statuses = [hold,
queued,
running]

def __init__(self):
self.__status = self.no_status

@@ -102,6 +116,22 @@ def getStatus(self):
"""
return self.__status

def getAllStatuses(self):
""" return list of named tuples containing all status types """
return self.__all_statuses

def getFailingStatuses(self):
""" return list of named tuples containing failing status types """
return self.__exit_nonzero_statuses

def getSuccessStatuses(self):
""" return list of named tuples containing exit code zero status types """
return self.__exit_zero_statuses

def getPendingStatuses(self):
""" return list of named tuples containing pending status types """
return self.__pending_statuses

def setStatus(self, status=no_status):
"""
Set the current status to status. If status is not supplied, 'no_status' is implied.
@@ -712,7 +712,10 @@ def initialize(self, argv, app_name):
self.options.results_storage = json.load(f)

# Adhere to previous input file syntax, or set the default
self.options.input_file_name = self.options.results_storage.get('INPUT_FILE_NAME', 'tests')
_input_file_name = 'tests'
if self.options.input_file_name:
_input_file_name = self.options.input_file_name
self.options.input_file_name = self.options.results_storage.get('INPUT_FILE_NAME', _input_file_name)

except ValueError:
# This is a hidden file, controled by the TestHarness. So we probably shouldn't error
@@ -856,6 +859,9 @@ def checkAndUpdateCLArgs(self):
if opts.queue_source_command and not os.path.exists(opts.queue_source_command):
print('ERROR: pre-source supplied but path does not exist')
sys.exit(1)
if opts.failed_tests and not os.path.exists(self.results_storage):
print('ERROR: --failed-tests could not detect a previous run')
sys.exit(1)

# Flatten input_file_name from ['tests', 'speedtests'] to just tests if none supplied
# We can not support running two spec files during one launch into a third party queue manager.
@@ -267,15 +267,18 @@ def setStatus(self, status, message=''):
def createStatus(self):
return self.job_status.createStatus()

def previousTesterStatus(self, options, previous_results=None):
return self.__tester.previousTesterStatus(options, previous_results)

def getStatusMessage(self):
return self.__job_message

### Boolean status comparisons based on current Job _and_ Tester status. All finalized status calls
### should now call job.isSomeStatus for the definitive answer.
# the following are more job related...
def isError(self):
_status = self.getStatus()
return _status == self.error or _status == self.timeout
return self.getStatus() in self.job_status.getFailingStatuses()

def isSkip(self):
_status = self.getStatus()
return (_status == self.finished and self.__tester.isSkip()) \
@@ -242,34 +242,28 @@ def _isProcessReady(self, job_data):
else:
for job in job_data.jobs.getJobs():
tester = job.getTester()
(status, caveats, status_message) = self._getTesterStatusAndCaveatsFromSession(job_data.json_data, tester)
status, message, caveats = job.previousTesterStatus(self.options, job_data.json_data)
tester.setStatus(status, message)
if caveats:
tester.addCaveats(caveats)
status_message = tester.getStatusMessage()

# This single job will enter the runner thread pool
if status_message == "LAUNCHING":
tester.setStatus(tester.queued)
tester.addCaveats(caveats)

# This single job will be skipped for some reason
else:
tester.addCaveats(caveats)
tester.setStatus(status, status_message)
is_ready = False

# Job group not originally launched
else:
for job in job_data.jobs.getJobs():
tester = job.getTester()
# Entire job group was skipped, deleted, silent, etc
if job_data.json_data and job_data.json_data.get(job_data.job_dir, {}):
(status, caveats, status_message) = self._getTesterStatusAndCaveatsFromSession(job_data.json_data, tester)
status, message, caveats = job.previousTesterStatus(self.options, job_data.json_data)
tester.setStatus(status, message)
if caveats:
tester.addCaveats(caveats)
if status == tester.skip:
tester.setStatus(tester.skip, status_message)
else:
tester.setStatus(tester.silent, status_message)

# Entire job group did not previously enter the Scheduler
else:
if tester.isNoStatus():
tester.setStatus(tester.silent)
is_ready = False

@@ -350,9 +344,10 @@ def _setJobStatus(self, job_data):

if group_results.get(job.getTestName(), {}):
job_results = group_results[job.getTestName()]
(status, caveats, status_message) = self._getTesterStatusAndCaveatsFromSession(results, tester)
tester.addCaveats(caveats)
tester.setStatus(status, status_message)
status, message, caveats = job.previousTesterStatus(self.options, results)
tester.setStatus(status, message)
if caveats:
tester.addCaveats(caveats)

# Recover useful job information from job results
job.setPreviousTime(job_results['TIMING'])
@@ -363,18 +358,6 @@ def _setJobStatus(self, job_data):
tester.addCaveats('not originally launched')
tester.setStatus(tester.skip)

def _getTesterStatusAndCaveatsFromSession(self, session, tester):
""" re-create status for tester using the session file """
restored_status = tester.test_status.createStatus()
status_message = ''
caveats = []
if session and session.get(tester.getTestDir(), {}).get(tester.getTestName(), {}):
status_key = session[tester.getTestDir()][tester.getTestName()]['STATUS'].encode('ascii', 'ignore')
restored_status = tester.test_status.createStatus(status_key)
status_message = session[tester.getTestDir()][tester.getTestName()]['STATUS_MESSAGE'].encode('ascii', 'ignore')
caveats = session[tester.getTestDir()][tester.getTestName()]['CAVEATS']
return (restored_status, caveats, status_message)

def _cleanupFiles(self, Jobs):
""" Silence all Jobs and perform cleanup operations """
job_list = Jobs.getJobs()
@@ -144,6 +144,20 @@ def setStatus(self, status, message=''):
def createStatus(self):
return self.test_status.createStatus()

# Return a tuple (status, message, caveats) for this tester as found
# in the .previous_test_results.json file (or supplied json object)
def previousTesterStatus(self, options, previous_results=None):
if not previous_results:
previous_results = options.results_storage

status_exists = previous_results.get(self.getTestDir(), {}).get(self.getTestName(), None)
status = (self.test_status.createStatus(), '', '')
if status_exists:
status = (self.test_status.createStatus(status_exists['STATUS'].encode('ascii', 'ignore')),
status_exists['STATUS_MESSAGE'].encode('ascii', 'ignore'),
status_exists['CAVEATS'])
return (status)

def getStatusMessage(self):
return self.__tester_message

@@ -387,18 +401,10 @@ def checkRunnableBase(self, options):
self.setStatus(self.silent)
return False

# If something has already deemed this test a failure or is silent, return now
if self.isFail() or self.isSilent():
# If something has already deemed this test a failure
if self.isFail():
return False

# Check if we only want to run failed tests
if options.failed_tests and options.results_storage is not None:
result_key = options.results_storage.get(self.getTestDir(), {})
status = result_key.get(self.getTestName(), {}).get('FAIL', '')
if not status:
self.setStatus(self.silent)
return False

# Check if we only want to run syntax tests
if options.check_input and not self.specs['check_input']:
self.setStatus(self.silent)
@@ -65,9 +65,6 @@ def delete_node(self, node_name, graph=None):
if node_name not in graph:
raise KeyError('node %s does not exist' % node_name)

# cache current graph before we delete a node
self.__cacheGraph()

graph.pop(node_name)

for node, edges in graph.iteritems():
@@ -105,9 +102,6 @@ def delete_edge(self, ind_node, dep_node, graph=None):
if not graph:
graph = self.graph

# cache current graph before we delete an edge
self.__cacheGraph()

if dep_node not in graph.get(ind_node, []):
raise KeyError('this edge does not exist in graph')
graph[ind_node].remove(dep_node)
@@ -117,9 +111,6 @@ def rename_edges(self, old_task_name, new_task_name, graph=None):
if not graph:
graph = self.graph

# cache current graph before we rename an edge
self.__cacheGraph()

for node, edges in graph.items():

if node == old_task_name:
@@ -266,13 +257,9 @@ def size(self):
# Added by the MOOSE group
def getOriginalDAG(self):
"""
Returns a clone of the graph as it existed before any nodes
were deleted.
Adding nodes again later, will allow the original graph to
'reset' and grow.
Returns a clone of the graph as it existed before calling the first ind_nodes
"""
return self.__cacheGraph()
return self.__cacheGraph().graph

# Added by the MOOSE group
def clone(self):
@@ -304,8 +291,6 @@ def reverse_clone(self, graph=None):
# Added by the MOOSE group
def reverse_edges(self, graph=None):
""" Reversed dependencies in current graph. """
# Create clone of original graph
self.__cacheGraph()

new_graph = self.reverse_clone()
self.graph = new_graph.graph
@@ -316,9 +301,6 @@ def delete_edge_if_exists(self, ind_node, dep_node, graph=None):
if not graph:
graph = self.graph

# cache current graph before we delete an edge
self.__cacheGraph()

if dep_node not in graph.get(ind_node, []):
return
graph[ind_node].remove(dep_node)

0 comments on commit ee93ae3

Please sign in to comment.
You can’t perform that action at this time.