Enable flake8 tests.

.travis.yml

@@ -0,0 +1,9 @@
+sudo: false
+language: python
+matrix:
+  include:
+    - python: '3.6'
+      install:
+        - 'pip install flake8'
+      script:
+        - flake8

examples/demoPool.py

@@ -8,12 +8,11 @@
 #
 
 from __future__ import print_function
-from builtins import map
+import math
 from builtins import range
+from lsst.ctrl.pool.pool import startPool, Pool, Debugger
 NUM = 10  # Number of items in data list
 
-import math
-
 
 def test1(cache, data, *args, **kwargs):
     result = math.sqrt(data)
@@ -30,7 +29,6 @@ def test2(cache, data, *args, **kwargs):
                                                        args, cache.kwargs, kwargs, hasattr(cache, "p")))
     return None
 
-from lsst.ctrl.pool.pool import startPool, Pool, Debugger, Comm
 
 # Here's how to enable debugging messages from the pool
 Debugger().enabled = True
@@ -68,6 +66,7 @@ def context3(pool3):
     pool3.storeClear()
     pool3.storeList()
 
+
 pool1 = Pool(1)
 context1(pool1)
 pool2 = Pool(2)

python/lsst/ctrl/pool/log.py

@@ -1,10 +1,10 @@
 from future import standard_library
-standard_library.install_aliases()
 import os
 import copyreg
-
 import lsst.log as lsstLog
 from lsst.utils import getPackageDir
+standard_library.install_aliases()
+
 
 def pickleLog(log):
     """Pickle a log
@@ -13,14 +13,14 @@ def pickleLog(log):
     """
     return lsstLog.Log, tuple()
 
+
 copyreg.pickle(lsstLog.Log, pickleLog)
 
 
 def jobLog(job):
     """Add a job-specific log destination"""
     if job is None or job == "None":
         return
-    machine = os.uname()[1].split(".")[0]
     packageDir = getPackageDir("ctrl_pool")
     #   Set the environment variable which names the output file
     os.environ['JOBNAME'] = job

python/lsst/ctrl/pool/parallel.py

@@ -15,7 +15,6 @@
 import contextlib
 from lsst.pipe.base import CmdLineTask, TaskRunner
 from .pool import startPool, Pool, NODE, abortOnError, setBatchType
-from . import log  # register pickle functions for log
 
 __all__ = ["Batch", "PbsBatch", "SlurmBatch", "SmpBatch", "BATCH_TYPES", "BatchArgumentParser",
            "BatchCmdLineTask", "BatchPoolTask", ]
@@ -290,8 +289,8 @@ def submitCommand(self, scriptName):
         return "exec %s" % scriptName
 
 
-BATCH_TYPES = {'none' : None,
-               'None' : None,
+BATCH_TYPES = {'none': None,
+               'None': None,
                'pbs': PbsBatch,
                'slurm': SlurmBatch,
                'smp': SmpBatch,
@@ -445,7 +444,7 @@ def parseAndSubmit(cls, args=None, **kwargs):
         setBatchType(batchArgs.batch)
 
         if batchArgs.batch is None:     # don't use a batch system
-            sys.argv = [sys.argv[0]] + batchArgs.leftover # Remove all batch arguments
+            sys.argv = [sys.argv[0]] + batchArgs.leftover  # Remove all batch arguments
 
             return cls.parseAndRun()
         else:
@@ -510,7 +509,7 @@ def logOperation(self, operation, catch=False, trace=True):
         self.log.info("%s: Start %s" % (NODE, operation))
         try:
             yield
-        except:
+        except Exception:
             if catch:
                 cls, e, _ = sys.exc_info()
                 self.log.warn("%s: Caught %s while %s: %s" % (NODE, cls.__name__, operation, e))
@@ -560,7 +559,6 @@ def run(self, parsedCmd):
         """
         resultList = None
 
-        import multiprocessing
         self.prepareForMultiProcessing()
         pool = Pool()
 
@@ -573,7 +571,7 @@ def run(self, parsedCmd):
                 resultList = pool.map(self, targetList)
             else:
                 parsedCmd.log.warn("Not running the task because there is no data to process; "
-                    "you may preview data using \"--show data\"")
+                                   "you may preview data using \"--show data\"")
                 resultList = []
 
         return resultList

python/lsst/ctrl/pool/pool.py

@@ -1,8 +1,6 @@
 from future import standard_library
-standard_library.install_aliases()
 from builtins import zip
 from builtins import range
-from past.builtins import basestring
 from builtins import object
 # MPI process pool
 # Copyright 2013 Paul A. Price
@@ -35,6 +33,8 @@
 from lsst.pipe.base import Struct
 from future.utils import with_metaclass
 
+standard_library.install_aliases()
+
 __all__ = ["Comm", "Pool", "startPool", "setBatchType", "getBatchType", "abortOnError", "NODE", ]
 
 NODE = "%s:%d" % (os.uname()[1], os.getpid())  # Name of node
@@ -59,6 +59,7 @@ def pickleInstanceMethod(method):
     name = method.__name__
     return unpickleInstanceMethod, (obj, name)
 
+
 copyreg.pickle(types.MethodType, pickleInstanceMethod)
 
 
@@ -89,22 +90,26 @@ def pickleFunction(function):
     funcName = function.__name__
     return unpickleFunction, (moduleName, funcName)
 
+
 copyreg.pickle(types.FunctionType, pickleFunction)
 
 try:
     _batchType
 except NameError:
     _batchType = "unknown"
 
+
 def getBatchType():
     """Return a string giving the type of batch system in use"""
     return _batchType
 
+
 def setBatchType(batchType):
     """Return a string giving the type of batch system in use"""
     global _batchType
     _batchType = batchType
 
+
 def abortOnError(func):
     """Function decorator to throw an MPI abort on an unhandled exception"""
     @wraps(func)
@@ -180,7 +185,7 @@ def guessPickleObj():
         # This should work if it's coming from the regular pickle module, and they
         # haven't changed the variable names since python 2.7.3.
         return stack[-2].tb_frame.f_locals["obj"]
-    except:
+    except Exception:
         return None
 
 
@@ -231,6 +236,7 @@ class is.  However, all SWIG objects are reported as "SwigPyObject".
             else:
                 sys.exit(1)
 
+
 def catchPicklingError(func):
     """Function decorator to catch errors in pickling and print something useful"""
     @wraps(func)
@@ -508,9 +514,9 @@ def _getCache(self, context, index):
 
         The cache is updated with the contents of the store.
         """
-        if not context in self._cache:
+        if context not in self._cache:
             self._cache[context] = {}
-        if not context in self._store:
+        if context not in self._store:
             self._store[context] = {}
         cache = self._cache[context]
         store = self._store[context]
@@ -584,30 +590,30 @@ def _reduceQueue(self, context, reducer, func, queue, *args, **kwargs):
     def storeSet(self, context, **kwargs):
         """Set values in store for a particular context"""
         self.log("storing", context, kwargs)
-        if not context in self._store:
+        if context not in self._store:
             self._store[context] = {}
         for name, value in kwargs.items():
             self._store[context][name] = value
 
     def storeDel(self, context, *nameList):
         """Delete value in store for a particular context"""
         self.log("deleting from store", context, nameList)
-        if not context in self._store:
+        if context not in self._store:
             raise KeyError("No such context: %s" % context)
         for name in nameList:
             del self._store[context][name]
 
     def storeClear(self, context):
         """Clear stored data for a particular context"""
         self.log("clearing store", context)
-        if not context in self._store:
+        if context not in self._store:
             raise KeyError("No such context: %s" % context)
         self._store[context] = {}
 
     def cacheClear(self, context):
         """Reset cache for a particular context"""
         self.log("clearing cache", context)
-        if not context in self._cache:
+        if context not in self._cache:
             return
         self._cache[context] = {}
 
@@ -618,7 +624,7 @@ def cacheList(self, context):
 
     def storeList(self, context):
         """List contents of store for a particular context"""
-        if not context in self._store:
+        if context not in self._store:
             raise KeyError("No such context: %s" % context)
         sys.stderr.write("Store on %s (%s): %s\n" % (self.node, context, self._store[context]))
 

setup.cfg

@@ -0,0 +1,13 @@
+[flake8]
+max-line-length = 110
+ignore = E133, E226, E228, N802, N803, N806
+exclude =
+  bin,
+  doc,
+  **/*/__init__.py,
+  **/*/version.py,
+  tests/.tests
+
+[tool:pytest]
+addopts = --flake8
+flake8-ignore = E133 E226 E228 E251 N802 N803 N806

tests/.tests/pytest-ctrl_pool.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuite errors="0" failures="0" name="pytest" skips="0" tests="1" time="6.006"><testcase classname="ctrl_pool" file="python/lsst/ctrl/pool/pool.py" line="-1" name="python.lsst.ctrl.pool.pool" time="5.808513879776001"></testcase></testsuite>

tests/.tests/pytest-ctrl_pool.xml.failed

@@ -0,0 +1,21 @@
+=========================================================== test session starts ===========================================================
+platform linux -- Python 3.6.2, pytest-3.2.0, py-1.5.4, pluggy-0.4.0
+rootdir: /home/jcarlin/repos/ctrl_pool, inifile: setup.cfg
+plugins: cov-2.5.1, session2file-0.1.9, forked-0.2, xdist-1.20.1, flake8-0.9.1, nbval-0.9.1
+
+collecting 12 items
+
+collected 12 items 
+run-last-failure: rerun previous 1 failure
+
+python/lsst/ctrl/pool/pool.py .
+
+--------------------------- generated xml file: /home/jcarlin/repos/ctrl_pool/tests/.tests/pytest-ctrl_pool.xml ---------------------------
+============================================================ warnings summary =============================================================
+python/lsst/ctrl/pool/pool.py
+  /opt/lsst/software/stack/stack/miniconda3-4.5.4-10a4fa6/Linux64/pycodestyle/2.3.1+3/lib/python/pycodestyle-2.3.1-py3.6.egg/pycodestyle.py:127: DeprecationWarning: invalid escape sequence \s
+    r'^\s*({0})'.format('|'.join(s.replace(' ', '\s+') for s in (
+
+-- Docs: http://doc.pytest.org/en/latest/warnings.html
+=========================================================== 11 tests deselected ===========================================================
+=========================================== 1 passed, 11 deselected, 1 warnings in 6.00 seconds ===========================================

tests/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.tests(pyList=[])