setup: pip-compile based pinned dependencies

Pins all dependencies via pip-compile and amends installation procedure to used pinned packages. The package versions will be updated from time to time via dedicated upgrade campaigns. Addresses reanahub/reana#322. Signed-off-by: Tibor Šimko <tibor.simko@cern.ch>
reanahub · Jul 9, 2020 · 181e8ec · 181e8ec
1 parent 04042bf
commit 181e8ec
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 24 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,5 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018 CERN.
+# Copyright (C) 2017, 2018, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -24,12 +24,11 @@ services:
   - docker
 
 before_install:
-  - pip install -r requirements-dev.txt
+  - pip install -r requirements.txt
   - travis_retry pip install --upgrade pip setuptools py
   - travis_retry pip install twine wheel coveralls
 
 install:
-  - pip install git+git://github.com/reanahub/reana-commons.git@master#egg=reana-commons
   - travis_retry pip install -e .[all]
 
 script:

diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,16 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
+# Install base image and its dependencies
 FROM python:3.6-slim
-
-ENV TERM=xterm
 RUN apt-get update && \
     apt-get install -y vim-tiny && \
     pip install --upgrade pip
 
+# Install Kerberos dependencies
 RUN export DEBIAN_FRONTEND=noninteractive ;\
     apt-get -yq install krb5-user \
                         krb5-config \
@@ -19,8 +19,9 @@ RUN export DEBIAN_FRONTEND=noninteractive ;\
                         gcc;
 ADD etc/krb5.conf /etc/krb5.conf
 
-
+# Default compute backend is Kubernetes
 ARG COMPUTE_BACKENDS=kubernetes
+
 # CERN HTCondor part taken from https://gitlab.cern.ch/batch-team/condorsubmit
 RUN if echo "$COMPUTE_BACKENDS" | grep -q "htcondorcern"; then \
       export DEBIAN_FRONTEND=noninteractive ;\
@@ -36,15 +37,16 @@ RUN if echo "$COMPUTE_BACKENDS" | grep -q "htcondorcern"; then \
       apt-get -y remove gnupg2 wget alien; \
     fi
 
+# CERN Slurm backend requires SSH to headnode
 RUN if echo "$COMPUTE_BACKENDS" | grep -q "slurmcern"; then \
       export DEBIAN_FRONTEND=noninteractive ;\
       apt-get -yq install openssh-client \
                           --no-install-recommends; \
     fi
 
+# Add HTCondor related files
 ADD etc/cernsubmit.yaml /etc/condor/
 ADD etc/10_cernsubmit.config /etc/condor/config.d/
-
 ADD etc/ngbauth-submit /etc/sysconfig/
 ADD etc/ngauth_batch_crypt_pub.pem /etc/
 ADD etc/cerngridca.crt /usr/local/share/ca-certificates/cerngridca.crt
@@ -53,29 +55,32 @@ ADD etc/job_wrapper.sh etc/job_wrapper.sh
 RUN chmod +x /etc/job_wrapper.sh
 RUN update-ca-certificates
 
-COPY CHANGES.rst README.rst setup.py /code/
-COPY reana_job_controller/version.py /code/reana_job_controller/
-WORKDIR /code
-RUN pip install requirements-builder && \
-    requirements-builder -l pypi setup.py | pip install -r /dev/stdin && \
-    pip uninstall -y requirements-builder
+# Install dependencies
+COPY requirements.txt /code/
+RUN pip install -r /code/requirements.txt
 
+# Copy cluster component source code
+WORKDIR /code
 COPY . /code
 
-# Debug off by default
+# Are we debugging?
 ARG DEBUG=0
-RUN if [ "${DEBUG}" -gt 0 ]; then pip install -r requirements-dev.txt; pip install -e .; else pip install .; fi;
+RUN if [ "${DEBUG}" -gt 0 ]; then pip install pip install -e ".[debug]"; else pip install .; fi;
 
-# Building with locally-checked-out shared modules?
+# Are we building with locally-checked-out shared modules?
 RUN if test -e modules/reana-commons; then pip install -e modules/reana-commons[kubernetes] --upgrade; fi
 RUN if test -e modules/reana-db; then pip install -e modules/reana-db --upgrade; fi
 
 # Check if there are broken requirements
 RUN pip check
 
-EXPOSE 5000
+# Set useful environment variables
+ENV COMPUTE_BACKENDS=$COMPUTE_BACKENDS \
+    FLASK_APP=reana_job_controller/app.py \
+    TERM=xterm
 
-ENV COMPUTE_BACKENDS $COMPUTE_BACKENDS
-ENV FLASK_APP reana_job_controller/app.py
+# Expose ports to clients
+EXPOSE 5000
 
+# Run server
 CMD ["flask", "run", "-h", "0.0.0.0"]
diff --git a/requirements-dev.txt b/requirements-dev.txt
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,75 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile
+#
+amqp==2.6.0               # via kombu
+apispec==0.39.0           # via reana-job-controller (setup.py)
+appdirs==1.4.4            # via fs
+attrs==19.3.0             # via jsonschema
+bcrypt==3.1.7             # via paramiko
+bravado-core==5.17.0      # via bravado
+bravado==10.3.2           # via reana-commons
+cachetools==4.1.1         # via google-auth
+certifi==2020.6.20        # via kubernetes, requests
+cffi==1.14.0              # via bcrypt, cryptography, pynacl
+chardet==3.0.4            # via requests
+checksumdir==1.1.9        # via reana-commons
+click==7.1.2              # via flask, reana-commons
+cryptography==2.9.2       # via paramiko, reana-db
+decorator==4.4.2          # via gssapi
+flask==1.1.2              # via reana-job-controller (setup.py)
+fs==2.4.11                # via reana-job-controller (setup.py)
+google-auth==1.18.0       # via kubernetes
+gssapi==1.6.9             # via paramiko
+htcondor==8.9.2           # via reana-job-controller (setup.py)
+idna==2.10                # via jsonschema, requests
+importlib-metadata==1.7.0  # via jsonschema, kombu
+itsdangerous==1.1.0       # via flask
+jinja2==2.11.2            # via flask
+jsonpointer==2.0          # via jsonschema
+jsonref==0.2              # via bravado-core
+jsonschema[format]==3.2.0  # via bravado-core, reana-commons, swagger-spec-validator
+kombu==4.6.11             # via reana-commons
+kubernetes==11.0.0        # via reana-commons
+markupsafe==1.1.1         # via jinja2
+marshmallow==2.20.1       # via reana-job-controller (setup.py)
+mock==4.0.2               # via reana-commons
+monotonic==1.5            # via bravado
+msgpack-python==0.5.6     # via bravado
+msgpack==1.0.0            # via bravado-core
+oauthlib==3.1.0           # via requests-oauthlib
+paramiko[gssapi]==2.7.1   # via reana-job-controller (setup.py)
+psycopg2-binary==2.8.5    # via reana-db
+pyasn1-modules==0.2.8     # via google-auth
+pyasn1==0.4.8             # via paramiko, pyasn1-modules, rsa
+pycparser==2.20           # via cffi
+pynacl==1.4.0             # via paramiko
+pyrsistent==0.16.0        # via jsonschema
+python-dateutil==2.8.1    # via bravado, bravado-core, kubernetes
+pytz==2020.1              # via bravado-core, fs
+pyyaml==5.3.1             # via apispec, bravado, bravado-core, kubernetes, reana-commons, swagger-spec-validator
+reana-commons[kubernetes]==0.7.0.dev20200625  # via reana-db, reana-job-controller (setup.py)
+reana-db==0.7.0.dev20200619  # via reana-job-controller (setup.py)
+requests-oauthlib==1.3.0  # via kubernetes
+requests==2.24.0          # via bravado, kubernetes, requests-oauthlib
+retrying==1.3.3           # via reana-job-controller (setup.py)
+rfc3987==1.3.8            # via jsonschema
+rsa==4.6                  # via google-auth
+simplejson==3.17.0        # via bravado, bravado-core
+six==1.15.0               # via bcrypt, bravado, bravado-core, cryptography, fs, google-auth, gssapi, jsonschema, kubernetes, pynacl, pyrsistent, python-dateutil, retrying, sqlalchemy-utils, swagger-spec-validator, websocket-client
+sqlalchemy-utils==0.36.8  # via reana-db
+sqlalchemy==1.3.18        # via reana-db, sqlalchemy-utils
+strict-rfc3339==0.7       # via jsonschema
+swagger-spec-validator==2.7.3  # via bravado-core
+typing-extensions==3.7.4.2  # via bravado
+urllib3==1.25.9           # via kubernetes, requests
+vine==1.3.0               # via amqp
+webcolors==1.11.1         # via jsonschema
+websocket-client==0.57.0  # via kubernetes
+werkzeug==0.16.1          # via flask, reana-commons, reana-job-controller (setup.py)
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/setup.py b/setup.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -23,6 +23,7 @@
 ]
 
 extras_require = {
+    "debug": ["wdb", "ipdb", "Flask-DebugToolbar",],
     "docs": [
         "Sphinx>=1.4.4,<1.6",
         "sphinx-rtd-theme>=0.1.9",