diff --git a/.travis.yml b/.travis.yml index 51c3e5e..0c0a3ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,12 @@ language: python -python: 2.7 -dist: trusty +matrix: + include: + - dist: trusty + python: 2.7 + - dist: xenial + python: 3.7 + env: + - nodoc=true services: - mongodb env: TMPDIR=$PWD/tmp diff --git a/docs/api.rst b/docs/api.rst index 15eeeb8..86a3985 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -16,7 +16,6 @@ From the URL root index, users will be able to directly access the following: - :ref:`global-view-ref` - :ref:`new-user-ref` - :ref:`reset-pass-ref` -- :ref:`show-logs-ref` .. _global-view-ref: @@ -53,13 +52,6 @@ Reporting Completed Actions .. automethod:: workflowtools.WorkflowTools.reportaction -.. _show-logs-ref: - -Viewing Workflow Logs -~~~~~~~~~~~~~~~~~~~~~ - -.. automethod:: workflowtools.WorkflowTools.showlog - .. _new-user-ref: Creating a New Account diff --git a/docs/python_modules.rst b/docs/python_modules.rst index 11737d3..0da0e2b 100644 --- a/docs/python_modules.rst +++ b/docs/python_modules.rst @@ -55,12 +55,6 @@ Manage Actions .. automodule:: WorkflowWebTools.manageactions :members: -Show Log -~~~~~~~~ - -.. automodule:: WorkflowWebTools.showlog - :members: - Classifying Errors ~~~~~~~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index ddc2168..14f37b8 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'templates/*'] }, install_requires=[ - 'cmstoolbox==0.11.3', + 'cmstoolbox>=0.12.0', 'more-itertools<6.0.0', 'cherrypy<18.0.0', 'mako', diff --git a/test/test_report_action.py b/test/test_report_action.py index 7d5db6a..9a0d92c 100755 --- a/test/test_report_action.py +++ b/test/test_report_action.py @@ -28,7 +28,7 @@ def setUp(self): self.coll = manageactions.get_actions_collection() # Check that test database is empty if self.coll.count() != 0: - print 'Test database not empty, abort!!' + print('Test database not empty, abort!!') exit(123) now = time.time() diff --git a/test/test_workflowwebtools.py b/test/test_workflowwebtools.py index d834490..dcc96df 100755 --- a/test/test_workflowwebtools.py +++ b/test/test_workflowwebtools.py @@ -237,11 +237,11 @@ def setUp(self): if os.path.exists('reasons.db'): os.remove('reasons.db') - print 'before', rm.reasons_list() + print('before', rm.reasons_list()) rm.update_reasons(self.reasons) - print 'after', rm.reasons_list() + print('after', rm.reasons_list()) def tearDown(self): @@ -291,7 +291,7 @@ def setUp(self): rm.update_reasons(self.reasons1) if ma.get_actions_collection().count() != 0: - print 'Test database not empty, abort!!' + print('Test database not empty, abort!!') exit(123) file_name = WorkflowInfo(self.request_base['workflows']).cache_filename('workflow_params') diff --git a/workflowwebtools/__init__.py b/workflowwebtools/__init__.py index ff31842..9725434 100644 --- a/workflowwebtools/__init__.py +++ b/workflowwebtools/__init__.py @@ -4,6 +4,6 @@ :author: Daniel Abercrombie """ -__version__ = '0.9.6' +__version__ = '0.10.0' __all__ = [] diff --git a/workflowwebtools/clusterworkflows.py b/workflowwebtools/clusterworkflows.py index b960fcb..154fff4 100644 --- a/workflowwebtools/clusterworkflows.py +++ b/workflowwebtools/clusterworkflows.py @@ -208,7 +208,7 @@ def get_clustered_group(workflow, clusterer, session=None): group = predictions.get(workflow) if group is not None: - for wkf, cluster in predictions.iteritems(): + for wkf, cluster in predictions.items(): if cluster == group and wkf != workflow: output.append(wkf) diff --git a/workflowwebtools/errorutils.py b/workflowwebtools/errorutils.py index f7d7c18..740dff8 100644 --- a/workflowwebtools/errorutils.py +++ b/workflowwebtools/errorutils.py @@ -10,7 +10,10 @@ import os import json import re -import urlparse +try: + import urlparse +except ImportError: + import urllib.parse as urlparse # pylint: disable=import-error import validators import cherrypy @@ -82,12 +85,11 @@ def open_location(data_location): if raw is None: return raw - keys = raw.keys() - if not (keys and isinstance(raw[keys[0]], list)): + if not (raw and isinstance(raw[list(raw)[0]], list)): return raw return errors_from_list([ - workflow for workflow, statuses in raw.iteritems() + workflow for workflow, statuses in raw.items() if True in ['manual' in status for status in statuses] ]) diff --git a/workflowwebtools/globalerrors.py b/workflowwebtools/globalerrors.py index 63d8db4..13b5d7a 100644 --- a/workflowwebtools/globalerrors.py +++ b/workflowwebtools/globalerrors.py @@ -118,7 +118,7 @@ def setup(self): if not self.data_location: current_workflows = self.return_workflows() - prep_ids = set([self.get_workflow(wf).get_prep_id() for wf in current_workflows]) + prep_ids = {self.get_workflow(wf).get_prep_id() for wf in current_workflows} other_workflows = sum([self.get_prepid(prep_id).get_workflows() \ for prep_id in prep_ids], []) @@ -405,20 +405,20 @@ def group_errors(input_errors, grouping_function, **kwargs): output = default_errors_format() - for subgroup, values in input_errors.iteritems(): + for subgroup, values in input_errors.items(): group = grouping_function(subgroup) # We have three variables for everything, so we can write this by hand # Not ideal - for row, row_val in values['errors'].iteritems(): - for col, numerrors in row_val.iteritems(): + for row, row_val in values['errors'].items(): + for col, numerrors in row_val.items(): output[group]['errors'][row][col] += numerrors output[group]['sub'][subgroup] = values output[group]['total'] += values['total'] - for key, func in kwargs.iteritems(): + for key, func in kwargs.items(): output[group][key] = func(group) return output diff --git a/workflowwebtools/manageactions.py b/workflowwebtools/manageactions.py index 430d5a0..7e63e4e 100644 --- a/workflowwebtools/manageactions.py +++ b/workflowwebtools/manageactions.py @@ -5,6 +5,8 @@ :author: Daniel Abercrombie """ +from __future__ import print_function + import time import datetime import ssl @@ -34,7 +36,7 @@ def extract_reasons_params(action, **kwargs): if not isinstance(tasks_to_do, list): tasks_to_do = [tasks_to_do] - for key, item in kwargs.iteritems(): + for key, item in kwargs.items(): if 'shortreason' in key: short_re = item or reasonsmanip.DEFAULT_SHORT @@ -138,7 +140,7 @@ def submitaction(user, workflows, action, session=None, **kwargs): # Get any existing thing (most likely not there) step_params = wf_params.get(short_step_name, {}) - for key, val in all_steps.iteritems(): + for key, val in all_steps.items(): # This also includes if the key value is set but blank if not step_params.get(key): step_params[key] = val @@ -313,7 +315,7 @@ def fix_sites(**kwargs): coll = get_actions_collection() - for task, value in params.iteritems(): + for task, value in params.items(): split_task = task.split('/') workflow = split_task[1] subtask = '/'.join(split_task[2:]) @@ -324,4 +326,4 @@ def fix_sites(**kwargs): coll.update_one({'workflow': workflow}, {'$set': {'parameters': output}}) - print params + print(params) diff --git a/workflowwebtools/paramsregression.py b/workflowwebtools/paramsregression.py index b331ac5..d5f54d8 100644 --- a/workflowwebtools/paramsregression.py +++ b/workflowwebtools/paramsregression.py @@ -11,6 +11,8 @@ :author: Daniel Abercrombie """ +from __future__ import print_function + import sys import json @@ -40,7 +42,7 @@ def convert_to_dense(errors, keys=None, allerrors=None, allsites=None): allsites = set() for status in keys: - for error, sites in errors[status].iteritems(): + for error, sites in errors[status].items(): allerrors.add(int(error)) for site in sites: allsites.add(site) @@ -51,7 +53,7 @@ def convert_to_dense(errors, keys=None, allerrors=None, allsites=None): # Build the dense output output = {} for status in keys: - output[status] = [[0] * len(allsites) for _ in xrange(len(allerrors))] + output[status] = [[0] * len(allsites) for _ in allerrors] for i_error, error in enumerate(allerrors): for i_site, site in enumerate(allsites): output[status][i_error][i_site] += errors[status].get(str(error), {}).get(site, 0) @@ -75,7 +77,7 @@ def get_classifier(raw_data, parameter, **kwargs): :rtype: sklearn.neural_network.MLPClassifier """ - primary_ids = sorted(set([key.split('/')[1] for key in raw_data.keys()])) + primary_ids = sorted({key.split('/')[1] for key in raw_data.keys()}) # Only split samples when running interactive tests training_ids = primary_ids[0::2] if __name__ == '__main__' else primary_ids @@ -97,7 +99,7 @@ def get_classifier(raw_data, parameter, **kwargs): matrix = raw_data[key]['errors'][status] # Only do this for sparse matrices if not isinstance(matrix, list): - for error, sites in matrix.iteritems(): + for error, sites in matrix.items(): allerrors.add(int(error)) for site in sites: allsites.add(site) @@ -149,15 +151,15 @@ def print_results(data, target): right += 1 else: status = 'WRONG' - print '[%s] %i : %i -- %s : %s' % \ - (status, want, result, class_labels[want], class_labels[result]) + print('[%s] %i : %i -- %s : %s' % + (status, want, result, class_labels[want], class_labels[result])) - print '%f (%i/%i)' % (100.0 * right/len(target), right, len(target)) + print('%f (%i/%i)' % (100.0 * right/len(target), right, len(target))) - print '\nTraining:\n' + print('\nTraining:\n') print_results(training_data, training_target) - print '\nTesting:\n' + print('\nTesting:\n') print_results(testing_data, testing_target) return classifier diff --git a/workflowwebtools/predict/evaluate.py b/workflowwebtools/predict/evaluate.py index 21a8dc2..46d7f1a 100644 --- a/workflowwebtools/predict/evaluate.py +++ b/workflowwebtools/predict/evaluate.py @@ -4,6 +4,8 @@ A module that evaluates a model and returns the prediction """ +from __future__ import print_function + import os import random import itertools @@ -50,8 +52,8 @@ def build_table(df, template_table): if cond: chosen_site = site - print "Detected a site %s which was not present in the training dataset" % site - print "We would use a proxy site for this based on whether it is T1, T2 or T3" + print("Detected a site %s which was not present in the training dataset" % site) + print("We would use a proxy site for this based on whether it is T1, T2 or T3") tier = site.split("_")[0][1] if chosen_site is None: if tier == '1': @@ -60,15 +62,15 @@ def build_table(df, template_table): elif tier == '2': chosen_num = random.randint(0, n2-1) chosen_site = tier2_sites[chosen_num] - print 'The chosen site is ', chosen_site + print('The chosen site is ', chosen_site) elif tier == '3': chosen_num = random.randint(0, n3-1) chosen_site = tier3_sites[chosen_num] - print 'The chosen site is ', chosen_site + print('The chosen site is ', chosen_site) elif tier == '0': chosen_num = random.randint(0, n0-1) chosen_site = tier0_sites[chosen_num] - print 'The chosen site is ', chosen_site + print('The chosen site is ', chosen_site) else: continue if chosen_site is None: diff --git a/workflowwebtools/serverconfig.py b/workflowwebtools/serverconfig.py index 55b389a..7cdbee5 100644 --- a/workflowwebtools/serverconfig.py +++ b/workflowwebtools/serverconfig.py @@ -48,8 +48,8 @@ def config_dict(): '\n\n Copied a default configuration to %s.\n ' 'Please check it, and then run "workflowtool" again.\n' % LOCATION) - else: - LOCATION = os.path.join(default_loc) + + LOCATION = os.path.join(default_loc) output = {} diff --git a/workflowwebtools/showlog.py b/workflowwebtools/showlog.py deleted file mode 100644 index 46a56f9..0000000 --- a/workflowwebtools/showlog.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Generates content for the showlog webpage - -:author: Daniel Abercrombie -""" - - -from cmstoolbox.elasticsearch import search_logs - - -def give_logs(query, module='', limit=50): - """Takes output from :py:func:`cmstoolbox.elasticsearch.search_logs` - and organizes it to pass to a mako template - - :param str query: The query to pass to :py:func:`cmstoolbox.elasticsearch.search_logs` - :param str module: The module to show - :param int limit: The max number of logs to show on a single page - :returns: dictionary with ['logs'], which is a list of logs, and ['meta'], - which shows meta information of newest query result - :rtype: dict - """ - - formtext = ('
Submit query: ' - 'Module: ' - 'Logs Limit: ' - '
' % (module, limit)) - - if query == '': - # Get form page - return formtext - - else: - output = search_logs(query) - if not output: - return 'No logs were found!
' + formtext - - texts = set() - logs = list() - - for i in output: - if len(texts) > limit: - break - if i['_source']['text'] in texts: - continue - - if not module or i['_source']['subject'] == module: - - logs.append( - { - 'subject': i['_source']['subject'], - 'date': i['_source']['date'], - 'text': i['_source']['text'].split('\n') - } - ) - - texts.add(i['_source']['text']) - - return { - 'logs': logs, - 'meta': output[0]['_source']['meta'].split('\n') - } diff --git a/workflowwebtools/statuses.py b/workflowwebtools/statuses.py index 7638d4d..b2b377a 100644 --- a/workflowwebtools/statuses.py +++ b/workflowwebtools/statuses.py @@ -2,7 +2,10 @@ import os import json -import urlparse +try: + import urlparse +except ImportError: + import urllib.parse as urlparse # pylint: disable=import-error from cmstoolbox.webtools import get_json @@ -31,5 +34,5 @@ def get_manual_workflows(location): """ return [workflow for workflow, statuses - in open_statuses(location).iteritems() + in open_statuses(location).items() if True in ['manual' in status for status in statuses]] diff --git a/workflowwebtools/web/templates/showlog.html b/workflowwebtools/web/templates/showlog.html deleted file mode 100644 index 31e7952..0000000 --- a/workflowwebtools/web/templates/showlog.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - Unified Logs - - - - - -
- Submit another query: - - Module: - - Logs Limit: - - -
- - - % for log in logdata['logs']: - - - - - - - - - % endfor -
${log['subject']}${log['date']}
- % for line in log['text']: - ${line}
- % endfor -
-
- -

Meta Data:

- % for line in logdata['meta']: - ${line}
- % endfor - - - diff --git a/workflowwebtools/web/templates/welcome.html b/workflowwebtools/web/templates/welcome.html index e6486a4..5dceeb9 100644 --- a/workflowwebtools/web/templates/welcome.html +++ b/workflowwebtools/web/templates/welcome.html @@ -33,8 +33,3 @@

Welcome to the Workflow Team Web Tools

you can reset the password by email here. Your username will be in the email in the event you forgot that too.

- -

- Show Logs:
- This view shows logs from the elastic search server. -

diff --git a/workflowwebtools/workflowinfo.py b/workflowwebtools/workflowinfo.py index 8ac7bd4..12a9e83 100644 --- a/workflowwebtools/workflowinfo.py +++ b/workflowwebtools/workflowinfo.py @@ -6,6 +6,8 @@ :authors: Daniel Abercrombie """ +from __future__ import print_function + import os import re import json @@ -71,7 +73,7 @@ def function_wrapper(self, *args, **kwargs): with open(file_name, 'r') as cache_file: check_var = json.load(cache_file) except ValueError: - print 'JSON file no good. Deleting %s. Try again later.' % file_name + print('JSON file no good. Deleting %s. Try again later.' % file_name) os.remove(file_name) # If still None, call the wrapped function @@ -131,11 +133,11 @@ def errors_for_workflow(workflow, url='cmsweb.cern.ch'): if not result['result']: return output - for step, stepdata in result['result'][0].get(workflow, {}).iteritems(): + for step, stepdata in result['result'][0].get(workflow, {}).items(): errors = {} - for code, codedata in stepdata.get('jobfailed', {}).iteritems(): + for code, codedata in stepdata.get('jobfailed', {}).items(): sites = {} - for site, sitedata in codedata.iteritems(): + for site, sitedata in codedata.items(): if sitedata['errorCount']: sites[site] = sitedata['errorCount'] @@ -205,7 +207,7 @@ def reset(self): """ Reset the cache for this object and clear out the files. """ - print 'Reseting %s' % self + print('Reseting %s' % self) if not os.path.exists(self.bak_dir): os.mkdir(self.bak_dir) @@ -259,13 +261,13 @@ def get_workflow_parameters(self): use_https=True, use_cert=True) for params in result['result']: - for key, item in params.iteritems(): + for key, item in params.items(): if key == self.workflow: return item except Exception as error: - print 'Failed to get from reqmgr', self.workflow - print str(error) + print('Failed to get from reqmgr', self.workflow) + print(str(error)) return None @@ -349,13 +351,13 @@ def get_failure_rate(self): nsuccess = 0 nfailure = 0 - for agent, agentdata in wf_agents.iteritems(): + for agent, agentdata in wf_agents.items(): status = agentdata.get('status', {}) if not status: continue nsuccess += status.get('success', 0) - for ftype, num in status.get('failure', {}).iteritems(): + for ftype, num in status.get('failure', {}).items(): nfailure += num try: @@ -405,7 +407,7 @@ def get_recovery_info(self): task = doc['fileset_name'] # For each task, we have the following keys: # sites - a set of sites that the recovery docs say to run on. - for replica, info in doc['files'].iteritems(): + for replica, info in doc['files'].items(): # For fake files, just return the site whitelist if replica.startswith('MCFakeFile'): locations = site_white_list @@ -476,14 +478,14 @@ def get_explanation(self, errorcode, step=''): if self.explanations is None: self.explanations = defaultdict(lambda: defaultdict(lambda: [])) result = self._get_jobdetail() - for stepname, stepdata in result['result'][0].get(self.workflow, {}).iteritems(): + for stepname, stepdata in result['result'][0].get(self.workflow, {}).items(): # Get the errors from both 'jobfailed' and 'submitfailed' details for error, site in [(error, site) for status in ['jobfailed', 'submitfailed'] \ for error, site in stepdata.get(status, {}).items()]: if error == '0': continue - for sitename, samples in site.iteritems(): + for sitename, samples in site.items(): for detail in [values for sample in samples['samples'] for errs in sample['errors'].values() for values in errs]: @@ -564,7 +566,7 @@ def get_workflows_requesttime(self): request = self.get_requests() return [(workflow, time.mktime(datetime.datetime(*value['RequestDate']).timetuple())) \ - for workflow, value in request.iteritems()] + for workflow, value in request.items()] def get_workflows(self): """ diff --git a/workflowwebtools/workflowtools.py b/workflowwebtools/workflowtools.py index f6aed70..b922ef1 100644 --- a/workflowwebtools/workflowtools.py +++ b/workflowwebtools/workflowtools.py @@ -24,7 +24,6 @@ from workflowwebtools import serverconfig from workflowwebtools import manageactions from workflowwebtools import manageusers -from workflowwebtools import showlog from workflowwebtools import reasonsmanip from workflowwebtools import listpage from workflowwebtools import globalerrors @@ -136,30 +135,6 @@ def cluster(self): data['workflow_history'], data['all_errors']) return render('complete.html') - @cherrypy.expose - def showlog(self, search='', module='', limit=50): - """ - This page, located at ``https://localhost:8080/showlog``, - returns logs that are stored in an elastic search server. - If directed here from :ref:`workflow-view-ref`, then - the search will be for the relevant workflow. - - :param str search: The search string - :param str module: The module to look at, if only interested in one - :param int limit: The limit of number of logs to show on a single page - :returns: the logs from elastic search - :rtype: str - """ - logdata = showlog.give_logs(search, module, int(limit)) - if isinstance(logdata, dict): - return render('showlog.html', - logdata=logdata, - search=search, - module=module, - limit=limit) - - return logdata - @cherrypy.expose def globalerror2(self, reset=False): if reset: @@ -220,11 +195,11 @@ def getworkflows(self, prepid): "status": self.get_status(workflow), "errors": obj['obj'].sum_errors() } - for workflow, obj in workflow_objs.iteritems() + for workflow, obj in workflow_objs.items() ] self.lock.acquire() - for workflow, obj in workflow_objs.iteritems(): + for workflow, obj in workflow_objs.items(): if workflow not in self.workflows: self.workflows[workflow] = obj['obj'] self.lock.release() @@ -341,7 +316,7 @@ def getreasons(self): 'long': longreason } for shortreason, longreason in - sorted(reasonsmanip.reasons_list().iteritems()) + sorted(reasonsmanip.reasons_list().items()) ] @@ -354,17 +329,17 @@ def workflowerrors(self, workflow): output = [] # Need to track total sites, so we need to do nested loops here - for step, ecs in sorted(errors.iteritems()): + for step, ecs in sorted(errors.items()): allsites = set() codes = [] for code, sites in sorted([ (-1 if code == 'NotReported' else int(code), sites) - for code, sites in ecs.iteritems()]): + for code, sites in ecs.items()]): sites = { site: (num or int(code < 0)) - for site, num in sorted(sites.iteritems()) + for site, num in sorted(sites.items()) } allsites.update(sites.keys()) @@ -400,7 +375,7 @@ def seeworkflow(self, workflow='', issuggested=''): This should help operators understand what the error means. At the top of the page, there are links back for :ref:`global-view-ref`, - :ref:`show-logs-ref`, related JIRA tickets, + workflow logs, related JIRA tickets, and ReqMgr2 information about the workflow and prep ID. The main function of this page is to submit actions. @@ -692,7 +667,7 @@ def submitaction(self, workflows='', action='', **kwargs): for workflow in workflows: # Check sites of recovered workflows if check_actions[workflow]['Action'] in ['acdc', 'recovery']: - for subtask, params in check_actions[workflow]['Parameters'].iteritems(): + for subtask, params in check_actions[workflow]['Parameters'].items(): # Empty sites are noted if not params.get('sites'): blank_sites_subtask.append('/%s/%s' % (workflow, subtask))