setup: pip-compile based pinned dependencies

Pins all dependencies via pip-compile and amends installation procedure
to used pinned packages. The package versions will be updated from time
to time via dedicated upgrade campaigns. Addresses reanahub/reana#322.

Signed-off-by: Tibor Šimko <tibor.simko@cern.ch>

.travis.yml

@@ -1,5 +1,5 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -21,7 +21,7 @@ services:
   - docker
 
 before_install:
-  - pip install -r requirements-dev.txt
+  - pip install -r requirements.txt
   - travis_retry pip install --upgrade pip setuptools py
   - travis_retry pip install twine wheel coveralls
 

CHANGES.rst

@@ -4,6 +4,7 @@ Changes
 Version master (UNRELEASED)
 ---------------------------
 
+- Pins all Python dependencies allowing to easily rebuild component images at later times.
 - Upgrades to Yadage 0.20.1.
 - Creates workflow visualisation graph by default.
 - Allows specifying unpacked Docker images as workflow step requirement.

Dockerfile

@@ -1,12 +1,11 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
+# Install base image and its dependencies
 FROM python:3.6-slim
-
-ENV TERM=xterm
 RUN apt-get update && \
     apt-get install -y \
       autoconf \
@@ -24,24 +23,25 @@ RUN apt-get update && \
       vim-tiny && \
     pip install --upgrade pip
 
-COPY CHANGES.rst README.rst setup.py /code/
-COPY reana_workflow_engine_yadage/version.py /code/reana_workflow_engine_yadage/
-WORKDIR /code
-RUN pip install requirements-builder && \
-    requirements-builder -l pypi setup.py | pip install -r /dev/stdin && \
-    pip uninstall -y requirements-builder
+# Install dependencies
+COPY requirements.txt /code/
+RUN pip install -r /code/requirements.txt
 
+# Copy cluster component source code
+WORKDIR /code
 COPY . /code
 
-# Debug off by default
+# Are we debugging?
 ARG DEBUG=0
-RUN if [ "${DEBUG}" -gt 0 ]; then pip install -r requirements-dev.txt; pip install -e .; else pip install .; fi;
+RUN if [ "${DEBUG}" -gt 0 ]; then pip install pip install -e ".[debug]"; else pip install .; fi;
 
-# Building with locally-checked-out shared modules?
-RUN if test -e modules/reana-commons; then pip install -e modules/reana-commons --upgrade; fi
+# Are we building with locally-checked-out shared modules?
+RUN if test -e modules/reana-commons; then pip install -e modules/reana-commons[kubernetes] --upgrade; fi
 
 # Check if there are broken requirements
 RUN pip check
 
-ENV PYTHONPATH=/workdir
-ENV PACKTIVITY_ASYNCBACKEND reana_workflow_engine_yadage.externalbackend:ExternalBackend:ExternalProxy
+# Set useful environment variables
+ENV PACKTIVITY_ASYNCBACKEND=reana_workflow_engine_yadage.externalbackend:ExternalBackend:ExternalProxy \
+    PYTHONPATH=/workdir \
+    TERM=xterm

MANIFEST.in

@@ -1,15 +1,15 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018 CERN.
+# Copyright (C) 2017, 2018, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
 include Dockerfile
 include .dockerignore
-include requirements-dev.txt
 include LICENSE
 include *.rst
 include *.sh
+include *.txt
 include pytest.ini
 recursive-include docs *.py
 recursive-include docs *.png

requirements.txt

@@ -0,0 +1,67 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile
+#
+adage==0.10.1             # via reana-workflow-engine-yadage (setup.py), yadage
+amqp==2.6.0               # via kombu
+attrs==19.3.0             # via jsonschema
+bravado-core==5.17.0      # via bravado
+bravado==10.3.2           # via reana-commons
+certifi==2020.6.20        # via requests
+cffi==1.14.0              # via cryptography
+chardet==3.0.4            # via requests
+checksumdir==1.1.9        # via packtivity, reana-commons, yadage
+click==7.1.2              # via packtivity, reana-commons, reana-workflow-engine-yadage (setup.py), yadage, yadage-schemas
+cryptography==2.9.2       # via pyopenssl, requests
+decorator==4.4.2          # via jsonpath-rw, networkx
+enum34==1.1.10            # via reana-workflow-engine-yadage (setup.py)
+glob2==0.7                # via packtivity, yadage
+graphviz==0.14            # via reana-workflow-engine-yadage (setup.py)
+idna==2.8                 # via jsonschema, requests
+importlib-metadata==1.7.0  # via jsonschema, kombu
+jq==0.1.6                 # via packtivity, reana-workflow-engine-yadage (setup.py), yadage
+jsonpath-rw==1.4.0        # via packtivity, yadage
+jsonpointer==2.0          # via jsonschema, packtivity, yadage
+jsonref==0.2              # via bravado-core, packtivity, yadage, yadage-schemas
+jsonschema[format]==3.2.0  # via bravado-core, packtivity, reana-commons, yadage, yadage-schemas
+kombu==4.6.11             # via reana-commons
+mock==4.0.2               # via packtivity, reana-commons
+monotonic==1.5            # via bravado
+msgpack-python==0.5.6     # via bravado
+msgpack==1.0.0            # via bravado-core
+networkx==1.11            # via adage, reana-workflow-engine-yadage (setup.py)
+packtivity==0.14.21       # via reana-workflow-engine-yadage (setup.py), yadage
+ply==3.11                 # via jsonpath-rw
+psutil==5.7.0             # via packtivity, yadage
+pycparser==2.20           # via cffi
+pydot2==1.0.33            # via reana-workflow-engine-yadage (setup.py)
+pydotplus==2.0.2          # via reana-workflow-engine-yadage (setup.py)
+pygraphviz==1.5           # via reana-workflow-engine-yadage (setup.py)
+pyopenssl==19.0.0         # via reana-workflow-engine-yadage (setup.py), requests
+pyparsing==2.4.7          # via pydot2, pydotplus
+pyrsistent==0.16.0        # via jsonschema
+python-dateutil==2.8.1    # via bravado, bravado-core
+pytz==2020.1              # via bravado-core
+pyyaml==5.3.1             # via bravado, bravado-core, packtivity, reana-commons, yadage, yadage-schemas
+reana-commons==0.7.0.dev20200625  # via reana-workflow-engine-yadage (setup.py)
+requests[security]==2.22.0  # via bravado, packtivity, reana-workflow-engine-yadage (setup.py), yadage, yadage-schemas
+rfc3987==1.3.8            # via jsonschema, reana-workflow-engine-yadage (setup.py)
+simplejson==3.17.0        # via bravado, bravado-core
+six==1.15.0               # via bravado, bravado-core, cryptography, jsonpath-rw, jsonschema, pyopenssl, pyrsistent, python-dateutil, sqlalchemy-utils, webcolors
+sqlalchemy-utils==0.36.8  # via reana-workflow-engine-yadage (setup.py)
+sqlalchemy==1.3.18        # via reana-workflow-engine-yadage (setup.py), sqlalchemy-utils
+strict-rfc3339==0.7       # via jsonschema, reana-workflow-engine-yadage (setup.py)
+swagger-spec-validator==2.7.3  # via bravado-core
+typing-extensions==3.7.4.2  # via bravado
+urllib3==1.25.9           # via requests
+vine==1.3.0               # via amqp
+webcolors==1.9.1          # via jsonschema, reana-workflow-engine-yadage (setup.py)
+werkzeug==1.0.1           # via reana-commons
+yadage-schemas==0.10.6    # via packtivity, reana-workflow-engine-yadage (setup.py), yadage
+yadage==0.20.1            # via reana-workflow-engine-yadage (setup.py)
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

setup.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -23,6 +23,7 @@
 ]
 
 extras_require = {
+    "debug": ["wdb", "ipdb", "Flask-DebugToolbar",],
     "docs": ["Sphinx>=1.4.4", "sphinx-rtd-theme>=0.1.9",],
     "tests": tests_require,
 }