Python 3

.travis.yml

@@ -1,6 +1,12 @@
 language: python
-python: 2.7
-dist: trusty
+matrix:
+  include:
+    - dist: trusty
+      python: 2.7
+    - dist: xenial
+      python: 3.7
+      env:
+        - nodoc=true
 services:
   - mongodb
 env: TMPDIR=$PWD/tmp

docs/api.rst

@@ -16,7 +16,6 @@ From the URL root index, users will be able to directly access the following:
 - :ref:`global-view-ref`
 - :ref:`new-user-ref`
 - :ref:`reset-pass-ref`
-- :ref:`show-logs-ref`
 
 .. _global-view-ref:
 
@@ -53,13 +52,6 @@ Reporting Completed Actions
 
 .. automethod:: workflowtools.WorkflowTools.reportaction
 
-.. _show-logs-ref:
-
-Viewing Workflow Logs
-~~~~~~~~~~~~~~~~~~~~~
-
-.. automethod:: workflowtools.WorkflowTools.showlog
-
 .. _new-user-ref:
 
 Creating a New Account

docs/python_modules.rst

@@ -55,12 +55,6 @@ Manage Actions
 .. automodule:: WorkflowWebTools.manageactions
    :members:
 
-Show Log
-~~~~~~~~
-
-.. automodule:: WorkflowWebTools.showlog
-   :members:
-
 Classifying Errors
 ~~~~~~~~~~~~~~~~~~
 

setup.py

@@ -18,7 +18,7 @@
                                  'templates/*']
         },
     install_requires=[
-        'cmstoolbox==0.11.3',
+        'cmstoolbox>=0.12.0',
         'more-itertools<6.0.0',
         'cherrypy<18.0.0',
         'mako',

test/test_report_action.py

@@ -28,7 +28,7 @@ def setUp(self):
         self.coll = manageactions.get_actions_collection()
         # Check that test database is empty
         if self.coll.count() != 0:
-            print 'Test database not empty, abort!!'
+            print('Test database not empty, abort!!')
             exit(123)
 
         now = time.time()

test/test_workflowwebtools.py

@@ -237,11 +237,11 @@ def setUp(self):
         if os.path.exists('reasons.db'):
             os.remove('reasons.db')
 
-        print 'before', rm.reasons_list()
+        print('before', rm.reasons_list())
 
         rm.update_reasons(self.reasons)
 
-        print 'after', rm.reasons_list()
+        print('after', rm.reasons_list())
 
 
     def tearDown(self):
@@ -291,7 +291,7 @@ def setUp(self):
         rm.update_reasons(self.reasons1)
 
         if ma.get_actions_collection().count() != 0:
-            print 'Test database not empty, abort!!'
+            print('Test database not empty, abort!!')
             exit(123)
 
         file_name = WorkflowInfo(self.request_base['workflows']).cache_filename('workflow_params')

workflowwebtools/__init__.py

@@ -4,6 +4,6 @@
 :author: Daniel Abercrombie <dabercro@mit.edu>
 """
 
-__version__ = '0.9.6'
+__version__ = '0.10.0'
 
 __all__ = []

workflowwebtools/clusterworkflows.py

@@ -208,7 +208,7 @@ def get_clustered_group(workflow, clusterer, session=None):
     group = predictions.get(workflow)
 
     if group is not None:
-        for wkf, cluster in predictions.iteritems():
+        for wkf, cluster in predictions.items():
             if cluster == group and wkf != workflow:
                 output.append(wkf)
 

workflowwebtools/errorutils.py

@@ -10,7 +10,10 @@
 import os
 import json
 import re
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse # pylint: disable=import-error
 
 import validators
 import cherrypy
@@ -82,12 +85,11 @@ def open_location(data_location):
     if raw is None:
         return raw
 
-    keys = raw.keys()
-    if not (keys and isinstance(raw[keys[0]], list)):
+    if not (raw and isinstance(raw[list(raw)[0]], list)):
         return raw
 
     return errors_from_list([
-        workflow for workflow, statuses in raw.iteritems()
+        workflow for workflow, statuses in raw.items()
         if True in ['manual' in status for status in statuses]
     ])
 

workflowwebtools/globalerrors.py

@@ -118,7 +118,7 @@ def setup(self):
         if not self.data_location:
             current_workflows = self.return_workflows()
 
-            prep_ids = set([self.get_workflow(wf).get_prep_id() for wf in current_workflows])
+            prep_ids = {self.get_workflow(wf).get_prep_id() for wf in current_workflows}
 
             other_workflows = sum([self.get_prepid(prep_id).get_workflows() \
                                        for prep_id in prep_ids], [])
@@ -405,20 +405,20 @@ def group_errors(input_errors, grouping_function, **kwargs):
 
     output = default_errors_format()
 
-    for subgroup, values in input_errors.iteritems():
+    for subgroup, values in input_errors.items():
 
         group = grouping_function(subgroup)
 
         # We have three variables for everything, so we can write this by hand
         # Not ideal
-        for row, row_val in values['errors'].iteritems():
-            for col, numerrors in row_val.iteritems():
+        for row, row_val in values['errors'].items():
+            for col, numerrors in row_val.items():
                 output[group]['errors'][row][col] += numerrors
 
         output[group]['sub'][subgroup] = values
         output[group]['total'] += values['total']
 
-        for key, func in kwargs.iteritems():
+        for key, func in kwargs.items():
             output[group][key] = func(group)
 
     return output

workflowwebtools/manageactions.py

@@ -5,6 +5,8 @@
 :author: Daniel Abercrombie <dabercro@mit.edu>
 """
 
+from __future__ import print_function
+
 import time
 import datetime
 import ssl
@@ -34,7 +36,7 @@ def extract_reasons_params(action, **kwargs):
     if not isinstance(tasks_to_do, list):
         tasks_to_do = [tasks_to_do]
 
-    for key, item in kwargs.iteritems():
+    for key, item in kwargs.items():
 
         if 'shortreason' in key:
             short_re = item or reasonsmanip.DEFAULT_SHORT
@@ -138,7 +140,7 @@ def submitaction(user, workflows, action, session=None, **kwargs):
                 # Get any existing thing (most likely not there)
                 step_params = wf_params.get(short_step_name, {})
 
-                for key, val in all_steps.iteritems():
+                for key, val in all_steps.items():
                     # This also includes if the key value is set but blank
                     if not step_params.get(key):
                         step_params[key] = val
@@ -313,7 +315,7 @@ def fix_sites(**kwargs):
 
     coll = get_actions_collection()
 
-    for task, value in params.iteritems():
+    for task, value in params.items():
         split_task = task.split('/')
         workflow = split_task[1]
         subtask = '/'.join(split_task[2:])
@@ -324,4 +326,4 @@ def fix_sites(**kwargs):
         coll.update_one({'workflow': workflow},
                         {'$set': {'parameters': output}})
 
-    print params
+    print(params)

workflowwebtools/paramsregression.py

@@ -11,6 +11,8 @@
 :author: Daniel Abercrombie <dabercro@mit.edu>
 """
 
+from __future__ import print_function
+
 import sys
 import json
 
@@ -40,7 +42,7 @@ def convert_to_dense(errors, keys=None, allerrors=None, allsites=None):
         allsites = set()
 
         for status in keys:
-            for error, sites in errors[status].iteritems():
+            for error, sites in errors[status].items():
                 allerrors.add(int(error))
                 for site in sites:
                     allsites.add(site)
@@ -51,7 +53,7 @@ def convert_to_dense(errors, keys=None, allerrors=None, allsites=None):
     # Build the dense output
     output = {}
     for status in keys:
-        output[status] = [[0] * len(allsites) for _ in xrange(len(allerrors))]
+        output[status] = [[0] * len(allsites) for _ in allerrors]
         for i_error, error in enumerate(allerrors):
             for i_site, site in enumerate(allsites):
                 output[status][i_error][i_site] += errors[status].get(str(error), {}).get(site, 0)
@@ -75,7 +77,7 @@ def get_classifier(raw_data, parameter, **kwargs):
     :rtype: sklearn.neural_network.MLPClassifier
     """
 
-    primary_ids = sorted(set([key.split('/')[1] for key in raw_data.keys()]))
+    primary_ids = sorted({key.split('/')[1] for key in raw_data.keys()})
 
     # Only split samples when running interactive tests
     training_ids = primary_ids[0::2] if __name__ == '__main__' else primary_ids
@@ -97,7 +99,7 @@ def get_classifier(raw_data, parameter, **kwargs):
             matrix = raw_data[key]['errors'][status]
             # Only do this for sparse matrices
             if not isinstance(matrix, list):
-                for error, sites in matrix.iteritems():
+                for error, sites in matrix.items():
                     allerrors.add(int(error))
                     for site in sites:
                         allsites.add(site)
@@ -149,15 +151,15 @@ def print_results(data, target):
                     right += 1
                 else:
                     status = 'WRONG'
-                print '[%s] %i : %i -- %s : %s' % \
-                    (status, want, result, class_labels[want], class_labels[result])
+                print('[%s] %i : %i -- %s : %s' %
+                      (status, want, result, class_labels[want], class_labels[result]))
 
-            print '%f (%i/%i)' % (100.0 * right/len(target), right, len(target))
+            print('%f (%i/%i)' % (100.0 * right/len(target), right, len(target)))
 
-        print '\nTraining:\n'
+        print('\nTraining:\n')
         print_results(training_data, training_target)
 
-        print '\nTesting:\n'
+        print('\nTesting:\n')
         print_results(testing_data, testing_target)
 
     return classifier

workflowwebtools/predict/evaluate.py

@@ -4,6 +4,8 @@
 A module that evaluates a model and returns the prediction
 """
 
+from __future__ import print_function
+
 import os
 import random
 import itertools
@@ -50,8 +52,8 @@ def build_table(df, template_table):
                 if cond:
                     chosen_site = site
 
-                print "Detected a site %s which was not present in the training dataset" % site
-                print "We would use a proxy site for this based on whether it is T1, T2 or T3"
+                print("Detected a site %s which was not present in the training dataset" % site)
+                print("We would use a proxy site for this based on whether it is T1, T2 or T3")
                 tier = site.split("_")[0][1]
                 if chosen_site is None:
                     if tier == '1':
@@ -60,15 +62,15 @@ def build_table(df, template_table):
                     elif tier == '2':
                         chosen_num = random.randint(0, n2-1)
                         chosen_site = tier2_sites[chosen_num]
-                        print 'The chosen site is ', chosen_site
+                        print('The chosen site is ', chosen_site)
                     elif tier == '3':
                         chosen_num = random.randint(0, n3-1)
                         chosen_site = tier3_sites[chosen_num]
-                        print 'The chosen site is ', chosen_site
+                        print('The chosen site is ', chosen_site)
                     elif tier == '0':
                         chosen_num = random.randint(0, n0-1)
                         chosen_site = tier0_sites[chosen_num]
-                        print 'The chosen site is ', chosen_site
+                        print('The chosen site is ', chosen_site)
                     else:
                         continue
                 if chosen_site is None:

workflowwebtools/serverconfig.py

@@ -48,8 +48,8 @@ def config_dict():
                 '\n\n  Copied a default configuration to %s.\n  '
                 'Please check it, and then run "workflowtool" again.\n'
                 % LOCATION)
-        else:
-            LOCATION = os.path.join(default_loc)
+
+        LOCATION = os.path.join(default_loc)
 
     output = {}