Use PyPi dependencies for running the scrapers and tests

This also pins the version of neomodel to avoid pulling in pytest
and its dependencies in production. This should be fixed by:
neo4j-contrib/neomodel#553

Makefile

@@ -18,6 +18,9 @@ infra: up
 functional: up
 	tox -e py36, py39
 
+pin_dependencies:
+	tox -e pin-dependencies
+
 static_analysis:
 	tox -e flake8, isort
 

README.md

@@ -35,9 +35,9 @@ Or
 
 To manage dependencies, this project uses [pip-tools](https://github.com/jazzband/pip-tools) so that the production dependencies are pinned and the hashes of the dependencies are verified during installation.
 
-The unpinned dependencies are recorded in **setup.py**, and to generate the **requirements.txt** file, run `pip-compile --generate-hashes --output-file=requirements.txt --allow-unsafe`. This is only necessary when adding a new package. To upgrade a package, use the `-P` argument of the `pip-compile` command.
-
-To update **docs-requirements.txt** or **scraper-requirements.txt**, run `pip-compile --generate-hashes docs-requirements.in -o docs-requirements.txt --allow-unsafe` or `pip-compile --generate-hashes scraper-requirements.in -o scraper-requirements.txt --allow-unsafe` respectively. Also, there are test dependencies specified in **tests/requirements.txt** to regenerate this file run `pip-compile --generate-hashes tests/requirements.in -o tests/requirements.txt --allow-unsafe`. All those commands are meant to be run from the root directory of the project.
+The unpinned dependencies are recorded in **setup.py**, and to generate the **requirements.txt**
+file, run `make pin_dependencies`. This is only necessary when modifying the `requirements.in`
+files. To upgrade a package, use the `-P` argument of the `pip-compile` command.
 
 When installing the dependencies in a production environment, run `pip install --require-hashes -r requirements.txt`. Alternatively, you may use `pip-sync requirements.txt`, which will make sure your virtualenv only has the packages listed in **requirements.txt**.
 

docker/Dockerfile-scrapers

@@ -15,16 +15,16 @@ RUN dnf -y install \
     --setopt=deltarpm=0 \
     --setopt=install_weak_deps=false \
     --setopt=tsflags=nodocs \
-    python3-beautifulsoup4 \
-    python3-flask \
-    python3-ldap3 \
-    python3-neomodel \
-    python3-psycopg2 \
-    python3-requests \
-    python3-six \
+    bash \
+    gcc \
+    libpq-devel \
+    python3-devel \
+    python3-pip \
     && dnf clean all
 # This will allow a non-root user to install a custom root CA at run-time
 RUN chmod 777 /etc/pki/tls/certs/ca-bundle.crt
 COPY . .
+RUN pip3 install -r scraper-requirements.txt --no-deps --require-hashes --prefix /usr \
+    && pip3 install . --no-deps --prefix /usr
 USER 1001
 CMD ["/usr/bin/bash", "-c", "docker/scrape.sh"]

docker/Dockerfile-tests

@@ -2,24 +2,17 @@ FROM fedora:30
 
 # TODO: Replace the Koji URL once the following is stable https://bodhi.fedoraproject.org/updates/FEDORA-2018-c929a49b1b
 RUN dnf -y install \
-  --setopt=deltarpm=0 \
-  --setopt=install_weak_deps=false \
-  --setopt=tsflags=nodocs \
-  nmap-ncat \
-  python3-devel \
-  python3-flask \
-  python3-flask-oidc \
-  python3-ldap3 \
-  python3-mock \
-  python3-mypy \
-  python3-neomodel \
-  python3-prometheus_client \
-  python3-pytest \
-  python3-pytest-cov \
-  python3-flake8-docstrings \
-  && dnf clean all
+    --setopt=deltarpm=0 \
+    --setopt=install_weak_deps=false \
+    --setopt=tsflags=nodocs \
+    bash \
+    nmap-ncat \
+    python3-pip \
+    && dnf clean all
 
+COPY tests/requirements.txt /tmp/requirements.txt
 VOLUME /src
 WORKDIR /src
+RUN pip3 install -r /tmp/requirements.txt --no-deps --require-hashes --prefix /usr
 # Inspired from https://github.com/neo4j-contrib/neomodel/blob/master/tests-with-docker-compose.sh
-CMD ["sh", "-c", "while ! nc -z -w 2 neo4j 7687; do sleep 1; done; pytest-3 -vvv --cov-report term-missing --cov=estuary --ignore=tests/functional --ignore=tests/infra tests && mypy --ignore-missing-imports ."]
+CMD ["sh", "-c", "while ! nc -z -w 2 neo4j 7687; do sleep 1; done; pytest -vvv --cov-report term-missing --cov=estuary --ignore=tests/functional --ignore=tests/infra tests && mypy --ignore-missing-imports ."]

docker/docker-compose-tests.yml

@@ -3,8 +3,8 @@ version: '3'
 services:
   estuary_api_tests:
     build:
-      context: .
-      dockerfile: Dockerfile-tests
+      context: ../
+      dockerfile: docker/Dockerfile-tests
     volumes:
       - ../:/src:Z
     links:

docs-requirements.in

@@ -1,5 +1,3 @@
--r requirements.txt
--r scraper-requirements.txt
 recommonmark
 sphinx
 sphinx_rtd_theme