Merge pull request #50 from tillahoffmann/master

Update dependencies and use GitHub Actions for tests.

.github/workflows/main.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    name: Continuous Integration
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@v2"
+      - uses: "actions/setup-python@v2"
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.pythonLocation }}-${{ hashFiles('test_requirements.txt') }}
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Build the project
+        run: make

Makefile

@@ -5,7 +5,7 @@ all : tests docs
 tests : code_tests lint_tests
 
 lint_tests :
-	pylint pythonflow
+	flake8
 
 code_tests :
 	py.test --cov pythonflow --cov-fail-under=100 --cov-report=term-missing --cov-report=html --verbose --durations=5 -s
@@ -19,5 +19,3 @@ clean :
 
 requirements.txt : requirements.in setup.py
 	pip-compile -v requirements.in
-	./make_paths_relative.py < requirements.txt > requirements.tmp
-	mv requirements.tmp $@

docs/conf.py

@@ -93,7 +93,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

docs/examples/consumer.py

@@ -1,5 +1,4 @@
 import argparse
-import itertools as it
 import multiprocessing
 import re
 import textwrap
@@ -97,5 +96,6 @@ def entrypoint(args=None):
         ))
     print(summary)
 
+
 if __name__ == '__main__':
     entrypoint()

docs/examples/processor.py

@@ -17,6 +17,7 @@ def create_benchmark_graph():
     'benchmark': create_benchmark_graph,
 }
 
+
 def start_processor(args):
     graph = GRAPHS[args.graph]()
     with pf.Processor.from_graph(args.push_address, args.pull_address, graph) as processor:
@@ -55,7 +56,7 @@ def entrypoint(args=None):
             for processor in processors:
                 if processor.join(1) is not None:
                     break
-    except:  # pylint: disable=broad-except
+    except:  # noqa: E722
         print("Processors interrupted.")
 
     for processor in processors:

pythonflow/__init__.py

@@ -1,5 +1,5 @@
-# pylint: disable=missing-docstring
-# pylint: enable=missing-docstring
+
+
 # Copyright 2017 Spotify AB
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=wildcard-import
-from .core import *
-from .operations import *
-from .util import *
+from .core import *  # noqa: F401, F403
+from .operations import *  # noqa: F401, F403
+from .util import *  # noqa: F401, F403

pythonflow/core.py

@@ -1,5 +1,5 @@
-# pylint: disable=missing-docstring
-# pylint: enable=missing-docstring
+
+
 # Copyright 2017 Spotify AB
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -38,7 +38,7 @@ def __init__(self):
     _globals = threading.local()
 
     def __enter__(self):
-        assert getattr(self._globals, 'default_graph', None) is  None, \
+        assert getattr(self._globals, 'default_graph', None) is None, \
             "cannot have more than one default graph"
         Graph._globals.default_graph = self
         return self
@@ -47,7 +47,7 @@ def __exit__(self, *args):
         assert self._globals.default_graph is self
         Graph._globals.default_graph = None
 
-    def normalize_operation(self, operation):  # pylint:disable=W0621
+    def normalize_operation(self, operation):
         """
         Normalize an operation by resolving its name if necessary.
 
@@ -111,7 +111,7 @@ def normalize_context(self, context, **kwargs):
             raise ValueError("`context` must be a mapping.")
 
         operations = list(context)
-        for operation in operations:  # pylint:disable=W0621
+        for operation in operations:
             value = context.pop(operation)
             operation = self.normalize_operation(operation)
             if operation in context:
@@ -205,7 +205,7 @@ class EvaluationError(RuntimeError):
     """
 
 
-class Operation:  # pylint:disable=too-few-public-methods,too-many-instance-attributes
+class Operation:
     """
     Base class for operations.
 
@@ -357,7 +357,7 @@ def evaluate_operation(cls, operation, context, **kwargs):
         except Exception as ex:  # pragma: no cover
             stack = []
             interactive = False
-            for frame in reversed(operation._stack):  # pylint: disable=protected-access
+            for frame in reversed(operation._stack):
                 # Do not capture any internal stack traces
                 if 'pythonflow' in frame.filename:
                     continue
@@ -387,7 +387,6 @@ def __iter__(self):
         for i in range(num):
             yield self[i]
 
-    # pylint: disable=
     def __getattr__(self, name):
         return getattr_(self, name, graph=self.graph)
 
@@ -507,10 +506,9 @@ def __reversed__(self):
 
     def __call__(self, *args, **kwargs):
         return call(self, *args, **kwargs)
-    # pylint: enable=
 
 
-class func_op(Operation):  # pylint: disable=C0103,R0903
+class func_op(Operation):
     """
     Operation wrapper for stateless functions.
 
@@ -586,7 +584,6 @@ def control_dependencies(dependencies, graph=None):
     del graph.dependencies[-len(dependencies):]
 
 
-# pylint: disable=C0103
 abs_ = opmethod(builtins.abs)
 dict_ = opmethod(builtins.dict)
 help_ = opmethod(builtins.help)
@@ -698,4 +695,3 @@ def control_dependencies(dependencies, graph=None):
 xor = opmethod(operator.xor)
 
 import_ = opmethod(importlib.import_module)
-# pylint: enable=C0103

pythonflow/operations.py

@@ -1,5 +1,5 @@
-# pylint: disable=missing-docstring
-# pylint: enable=missing-docstring
+
+
 # Copyright 2017 Spotify AB
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,29 +23,29 @@
 from .util import _noop_callback, deprecated
 
 
-class placeholder(Operation):  # pylint: disable=C0103,R0903
+class placeholder(Operation):
     """
     Placeholder that needs to be given in the context to be evaluated.
     """
     def __init__(self, name=None, **kwargs):
         super(placeholder, self).__init__(name=name, **kwargs)
 
-    def _evaluate(self):  # pylint: disable=W0221
+    def _evaluate(self):
         raise ValueError("missing value for placeholder '%s'" % self.name)
 
     def __repr__(self):
         return "<pf.placeholder '%s'>" % self.name
 
 
-class conditional(Operation):  # pylint: disable=C0103,W0223
+class conditional(Operation):
     """
     Return `x` if `predicate` is `True` and `y` otherwise.
 
     .. note::
         The conditional operation will only execute one branch of the computation graph depending on
         `predicate`.
     """
-    def __init__(self, predicate, x, y=None, *, length=None, name=None, dependencies=None):  # pylint: disable=W0235
+    def __init__(self, predicate, x, y=None, *, length=None, name=None, dependencies=None):
         super(conditional, self).__init__(predicate, x, y,
                                           length=length, name=name, dependencies=dependencies)
 
@@ -54,7 +54,7 @@ def evaluate(self, context, callback=None):
         callback = callback or _noop_callback
         self.evaluate_dependencies(context, callback)
 
-        predicate, x, y = self.args  # pylint: disable=E0632,C0103
+        predicate, x, y = self.args
         # Evaluate the predicate and pick the right operation
         predicate = self.evaluate_operation(predicate, context, callback=callback)
         with callback(self, context):
@@ -63,7 +63,7 @@ def evaluate(self, context, callback=None):
         return value
 
 
-class try_(Operation):  # pylint: disable=C0103,W0223
+class try_(Operation):
     """
     Try to evaluate `operation`, fall back to alternative operations in `except_`, and ensure that
     `finally_` is evaluated.
@@ -89,13 +89,13 @@ def evaluate(self, context, callback=None):
         callback = callback or _noop_callback
         self.evaluate_dependencies(context, callback=callback)
 
-        operation, except_, finally_ = self.args # pylint: disable=E0632,C0103
+        operation, except_, finally_ = self.args
         with callback(self, context):
             try:
                 value = self.evaluate_operation(operation, context, callback=callback)
                 context[self] = value
                 return value
-            except:
+            except:  # noqa: E722
                 # Check the exceptions
                 _, ex, _ = sys.exc_info()
                 for type_, alternative in except_:
@@ -139,18 +139,18 @@ def cache(operation, get, put, key=None):
     return try_(
         func_op(get, key), [
             ((KeyError, FileNotFoundError),
-             identity(operation, dependencies=[func_op(put, key, operation)]))  # pylint: disable=unexpected-keyword-arg
+             identity(operation, dependencies=[func_op(put, key, operation)]))
         ]
     )
 
 
 def _pickle_load(filename):
-    with open(filename, 'rb') as fp:  # pylint: disable=invalid-name
+    with open(filename, 'rb') as fp:
         return pickle.load(fp)
 
 
 def _pickle_dump(value, filename):
-    with open(filename, 'wb') as fp:  # pylint: disable=invalid-name
+    with open(filename, 'wb') as fp:
         pickle.dump(value, fp)
 
 
@@ -196,11 +196,11 @@ def identity(value):
 
 
 # Short hand for the identity
-constant = identity  # pylint: disable=invalid-name
+constant = identity
 
 
 @opmethod
-def assert_(condition, message=None, *args, value=None):  # pylint: disable=keyword-arg-before-vararg
+def assert_(condition, message=None, *args, value=None):
     """
     Return `value` if the `condition` is satisfied and raise an `AssertionError` with the specified
     `message` and `args` if not.
@@ -244,33 +244,33 @@ def __init__(self, logger_name=None):
         self.logger = logging.getLogger(logger_name)
 
     @functools.wraps(logging.Logger.log)
-    def log(self, level, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def log(self, level, message, *args, **kwargs):
         if isinstance(level, str):
             level = getattr(logging, level.upper())
         return func_op(self.logger.log, level, message, *args, **kwargs)
 
     @functools.wraps(logging.Logger.debug)
-    def debug(self, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def debug(self, message, *args, **kwargs):
         return func_op(self.logger.debug, message, *args, **kwargs)
 
     @functools.wraps(logging.Logger.info)
-    def info(self, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def info(self, message, *args, **kwargs):
         return func_op(self.logger.info, message, *args, **kwargs)
 
     @functools.wraps(logging.Logger.warning)
-    def warning(self, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def warning(self, message, *args, **kwargs):
         return func_op(self.logger.warning, message, *args, **kwargs)
 
     @functools.wraps(logging.Logger.error)
-    def error(self, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def error(self, message, *args, **kwargs):
         return func_op(self.logger.error, message, *args, **kwargs)
 
     @functools.wraps(logging.Logger.critical)
-    def critical(self, message, *args, **kwargs):  # pylint: disable=missing-docstring
+    def critical(self, message, *args, **kwargs):
         return func_op(self.logger.critical, message, *args, **kwargs)
 
 
-class lazy_constant(Operation):  # pylint: disable=invalid-name
+class lazy_constant(Operation):
     """
     Operation that returns the output of `target` lazily.
 
@@ -288,7 +288,7 @@ def __init__(self, target, **kwargs):
             raise ValueError("`target` must be callable")
         self.value = None
 
-    def _evaluate(self):  # pylint: disable=W0221
+    def _evaluate(self):
         if self.value is None:
             self.value = self.target()
         return self.value

pythonflow/pfmq/__init__.py

@@ -1,5 +1,5 @@
-# pylint: disable=missing-docstring
-# pylint: enable=missing-docstring
+
+
 # Copyright 2018 Spotify AB
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .task import Task, SerializationError, apply
-from .broker import Broker
-from .worker import Worker
+from .task import Task, SerializationError, apply  # noqa: F401
+from .broker import Broker  # noqa: F401
+from .worker import Worker  # noqa: F401