Install dependencies from PyPi

Now dependencies are installed not as RPM packages but as python modules
thought pip.

JIRA:CWFHEALTH-286

Signed-off-by: Andrei Paplauski <apaplaus@redhat.com>

Makefile

@@ -0,0 +1,11 @@
+.PHONY: test dependencies
+
+pin_dependencies:
+		tox -e pin-dependencies
+
+test:
+		tox -r
+
+dependencies:
+		python -m pip install --upgrade pip
+		pip install tox

README.md

@@ -7,6 +7,23 @@
 Estuary Updater is a micro-service that updates the Neo4j graph database used by Estuary in real-time
 by reading and processing messages from the UMB.
 
+## Dependency Management
+
+To manage dependencies, this project uses [pip-tools](https://github.com/jazzband/pip-tools)
+so that the production dependencies are pinned and the hashes of the dependencies 
+are verified during installation.
+
+The unpinned dependencies are recorded in **setup.py**, and to generate 
+the **requirements.txt** file, run `make pin_dependencies`. This is only 
+necessary when modifying one of the __*requirements.in__ files. To upgrade a package, 
+use the `-P` argument of the `pip-compile` command.
+
+When installing the dependencies in a production environment, run 
+`pip install --require-hashes -r requirements.txt`. Alternatively, you may use 
+`pip-sync requirements.txt`, which will make sure your virtualenv only has the 
+packages listed in **requirements.txt**.
+
+
 ## Run the Unit Tests
 
 Since the unit tests require a running Neo4j instance, the tests are run in Docker containers using

build-requirements.in

@@ -0,0 +1,2 @@
+# required by cryptography
+setuptools-rust

build-requirements.txt

@@ -0,0 +1,22 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=build-requirements.txt build-requirements.in
+#
+semantic-version==2.8.5 \
+    --hash=sha256:45e4b32ee9d6d70ba5f440ec8cc5221074c7f4b0e8918bdab748cc37912440a9 \
+    --hash=sha256:d2cb2de0558762934679b9a104e82eca7af448c9f4974d1f3eeccff651df8a54
+    # via setuptools-rust
+setuptools-rust==0.12.1 \
+    --hash=sha256:60c9bf1423a725e472c4a2a6274598251f959f3ed5ffe7698526e78bb431b9b7 \
+    --hash=sha256:647009e924f0ae439c7f3e0141a184a69ad247ecb9044c511dabde232d3d570e
+    # via -r build-requirements.in
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via setuptools-rust
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools

docker/Dockerfile

@@ -1,39 +1,26 @@
-FROM fedora:30
+FROM registry.access.redhat.com/ubi8/ubi-minimal
 LABEL maintainer="Factory 2.0"
 
 ENV KRB5_CLIENT_KTNAME /etc/estuary-updater/estuary-updater.keytab
 WORKDIR /src
-RUN dnf -y install \
-  --setopt=deltarpm=0 \
-  --setopt=install_weak_deps=false \
-  --setopt=tsflags=nodocs \
+RUN microdnf -y install \
   bash \
-  krb5-workstation \
+  krb5-devel \
+  gcc \
+  openssl-devel \
   python3-devel \
-  # Install the RPM dependencies for both estuary-api and estuary-updater
-  python3-fedmsg \
-  python3-flask \
-  # python3-stomper requires this but doesn't install it for some reason
-  python3-future \
-  python3-koji \
-  python3-neomodel \
-  python3-pyOpenSSL \
-  python3-requests \
-  python3-requests-kerberos \
-  python3-stomper \
-  && dnf clean all
-# Install estuary-api directly from GitHub since there aren't any releases yet
-RUN pip3 install https://github.com/release-engineering/estuary-api/tarball/master#egg=estuary  --no-deps
+  && microdnf clean all
 # This will allow a non-root user to install a custom root CA at run-time
 RUN chmod 777 /etc/pki/tls/certs/ca-bundle.crt
 # This resolves the "Invalid UID in persistent keyring name while getting default ccache" error
 RUN sed -i '/default_ccache_name = KEYRING:persistent:%{uid}/d' /etc/krb5.conf
+# Set this variable to build cryptography package without Rust
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 COPY . .
-# We install the python3-koji RPM but it doesn't register as installed according to setuptools.
-# This hack keeps the application from raising a pkg_resources.DistributionNotFound exception.
-RUN sed -i '/koji/d' requirements.txt
-# Install the application itself
-RUN pip3 install . --no-deps
+# Install dependencies and the application itself
+RUN pip3 install -r build-requirements.txt --require-hashes --no-deps \
+    && pip3 install -r requirements.txt --require-hashes --no-deps \
+    && pip3 install . --no-deps
 # Remove the default fedmsg config files
 RUN rm -f /etc/fedmsg.d/* && rm -rf ./fedmsg.d
 

docker/Dockerfile-tests

@@ -1,29 +1,22 @@
-FROM fedora:30
+FROM registry.access.redhat.com/ubi8/ubi-minimal
 
-RUN dnf -y install \
-  --setopt=deltarpm=0 \
-  --setopt=install_weak_deps=false \
-  --setopt=tsflags=nodocs \
+RUN microdnf -y install \
   bash \
   nmap-ncat \
+  krb5-devel \
+  gcc \
+  openssl-devel \
   python3-devel \
-  # Install RPM dependencies from estuary-api otherwise they'll be installed via pip which won't map
-  # to production
-  python3-fedmsg \
-  python3-flake8 \
-  python3-flask \
-  python3-koji \
-  python3-neomodel \
-  python3-pyOpenSSL \
-  python3-pytest \
-  python3-pytest-cov \
-  python3-pytz \
-  python3-requests \
-  python3-requests-kerberos \
-  python3-tox \
-  python3-flake8-docstrings \
-  && dnf clean all
+  && microdnf clean all
 
 VOLUME /src
 WORKDIR /src
+
+COPY build-requirements.txt /tmp/build-requirements.txt
+COPY test-requirements.txt /tmp/test-requirements.txt
+# Set this variable to build cryptography package without Rust
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+# Installing the Python dependencies for Estuary Updater
+RUN pip3 install -r /tmp/build-requirements.txt --require-hashes --no-deps \
+    && pip3 install -r /tmp/test-requirements.txt --require-hashes --no-deps --prefix /usr
 CMD ["bash", "docker/test.sh"]

docker/docker-compose-tests.yml

@@ -3,8 +3,8 @@ version: '3'
 services:
   estuary_updater_tests:
     build:
-      context: .
-      dockerfile: Dockerfile-tests
+      context: ../
+      dockerfile: docker/Dockerfile-tests
     volumes:
       - ../:/src:Z
     links:

docker/run.sh

@@ -8,4 +8,4 @@ if [ -n "${CA_URL}" ] && [ ! -f "/tmp/.imported" ]; then
     touch /tmp/.imported
 fi
 
-exec fedmsg-hub-3
+exec fedmsg-hub

docker/test.sh

@@ -1,9 +1,7 @@
 #!/bin/bash
 
-echo "Installing the Python dependencies for Estuary Updater..."
-python3 setup.py develop --prefix /usr > /dev/null
 # Wait until Neo4j is up
 while ! nc -z -w 2 neo4j 7687; do sleep 1; done;
 # Run the tests
-    pytest-3 -vvv --cov-report term-missing --cov=estuary_updater tests/ && \
+    pytest -vvv --cov-report term-missing --cov=estuary_updater tests/ && \
     flake8

docs-requirements.in

@@ -0,0 +1,3 @@
+recommonmark
+sphinx
+sphinx_rtd_theme