Add vendor support to generate wheels from dependencies

.gitignore

@@ -99,3 +99,5 @@ ENV/
 .mypy_cache/
 
 .DS_Store
+
+*.whl

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "deps/django-prometheus"]
+	path = deps/django-prometheus
+	url = https://github.com/korfuri/django-prometheus.git

Dockerfile

@@ -2,19 +2,25 @@ FROM alpine:3.5
 
 # Add requirements for python and pip
 RUN apk add --update python3 pytest
-RUN apk add --update postgresql-dev 
+RUN apk add --update postgresql-libs
 RUN apk add --update curl
 
 RUN mkdir -p /opt/code
 WORKDIR /opt/code
 
 ADD requirements.txt /opt/code
 
+# Try to use local wheels. Even if not present, it will proceed
+ADD ./vendor /opt/vendor
+ADD ./deps /opt/deps
+# Only install them if there's any
+RUN if ls /opt/vendor/*.whl 1> /dev/null 2>&1; then pip3 install /opt/vendor/*.whl; fi
+
 # Some Docker-fu. In one step install the compile packages, install the
 # dependencies and then remove them. That skims the image size quite
 # sensibly.
 RUN apk add --no-cache --virtual .build-deps \
-  python3-dev build-base linux-headers gcc \
+  python3-dev build-base linux-headers gcc postgresql-dev \
     # Installing python requirements
     && pip3 install -r requirements.txt \
     && find /usr/local \

README.md

@@ -26,6 +26,8 @@ Tree structure
 ├── docker-compose.yaml (general dockerfile description, aimed at development)
 ├── Dockerfile (General Dockerfile of the main server)
 ├── requirements.txt (python requirements)
+├── vendor (cache with generated wheels from dependencies)
+├── deps (git submodules with embedded dependencies)
 ├── docker (Files related to build and operation of containers)
 │   └── (docker subdirs, like db or server)
 │      └── Scripts related to docker creation and operation of that service
@@ -88,6 +90,13 @@ in the DB do, like new fixtures or a new migration. Build all services with
     docker-compose build
 ```
 
+- *build-deps*: Precompile all dependencies into wheels and copy them in ./vendor. This is not
+required, but can save time when rebuilding the containers often, to avoid compile the same code
+over and over. Check the more detailed documentation in the ./vendor/README.md file.
+  Note that dependencies embedded in ./deps won't be compiled (though their dependencies will be).
+Check more details in the ./deps/README.md file. 
+
+
 Docker services oriented to production
 =========
 

deps/README.md

@@ -0,0 +1,57 @@
+Embedded dependencies
+=====================
+This directory is created to set direct dependencies that are not under pypi control or have more
+manual installation. This is mainly aimed to modules that live in private git repos and are not
+downloadable from PyPI, avoiding problems like requiring to pull from the repo with a ssh key
+from inside the container.
+
+The recommended way of dealing with them is to add them into this subdir git submodules and use
+the Python [setuptool module](https://docs.python.org/3.6/distributing/index.html) (setup.py).
+
+Note that dependencies here won't be installed from wheel, though their dependencies will be (if
+done in the proper format, through a setup.py). That's to avoid problems with setup and cache the 
+wrong version, with often happens while developing.
+Remember to include the dependency in the requirements.txt file
+
+An example of a module has been included (django-prometheus)
+
+How to add a new submodule
+==========================
+
+    cd deps
+    git submodule add https://github.com/foo
+
+this creates the subdir foo with the submodule. The file .gitmodules will be updated and needs
+to be tracked and commited.
+
+How to update a submodule
+==========================
+
+Log into the submodule and set git to the desired commit/tag/branch
+
+    cd deps/foo
+    git checkout v7.5.0
+    # or, for latest commit in current branch
+    git pull
+
+Then add the commit to the main repo, like a regular file
+
+    cd ..
+    git add foo
+    git commit
+
+
+If the submodule is updated
+============================
+
+and yours is not the proper version, it will appear as 
+
+    git status
+        modified:   foo (new commits)
+
+Get the new commits with the command
+
+    git submodule update --remote
+
+
+NOTE: Working with git submodules is a little tricky. Feel free to add and modify this document.

docker-compose.yaml

@@ -6,6 +6,13 @@ services:
         build:
             context: .
             dockerfile: ./docker/db/Dockerfile-postgres
+    build-deps:
+        build:
+            context: .
+            dockerfile: ./docker/deps/Dockerfile
+        volumes:
+            - ./vendor:/opt/ext_vendor
+        command: /opt/copy_deps.sh
     dev-server:
         build: .
         command: ./start_dev_server.sh
@@ -22,6 +29,8 @@ services:
             - ./src:/opt/code
         depends_on:
             - db
+            - build-deps
+
     # Producion related
     server:
         build: .

docker/deps/Dockerfile

@@ -0,0 +1,21 @@
+FROM alpine:3.5
+RUN mkdir -p /opt/vendor
+WORKDIR /opt/vendor
+RUN apk update
+# Basic python usage
+RUN apk add python3
+RUN apk add py3-pip 
+
+# Required for compiling
+RUN apk add python3-dev build-base linux-headers gcc postgresql-dev
+RUN pip3 install cython wheel
+
+ADD ./deps /opt/deps
+RUN mkdir -p /opt/vendor
+ADD requirements.txt /opt/deps
+ADD ./docker/deps/build_deps.sh /opt/
+ADD ./docker/deps/copy_deps.sh /opt/
+ADD ./docker/deps/search_wheels.py /opt/
+
+WORKDIR /opt/
+RUN ./build_deps.sh

docker/deps/build_deps.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+echo 'Building dependencies'
+mkdir -p /opt/wheels/
+cd /opt/vendor
+
+echo 'Building wheels from requirements'
+pip3 wheel -r /opt/deps/requirements.txt --process-dependency-links
+
+echo 'Done'
+
+# Clean the direct dependencies to avoid issues with caches
+for D in /opt/deps/*; do
+    if [ -d "${D}" ]; then
+        echo "Removing dependency in ${D}"   # your processing here
+        PKG_NAME=`python3 ${D}/setup.py --name`
+        WHEEL=`python3 /opt/search_wheels.py $PKG_NAME -d /opt/vendor`
+        echo "Deleting file $WHEEL"
+        rm $WHEEL
+    fi
+done
+
+echo "Wheels available: `ls /opt/vendor/*.whl`"

docker/deps/copy_deps.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+echo 'Deleting wheels and copy new ones'
+rm /opt/ext_vendor/*.whl
+echo "Dependencies are created at build. Run build --no-cache to recreate"
+cp /opt/vendor/* /opt/ext_vendor
+

docker/deps/search_wheels.py

@@ -0,0 +1,37 @@
+import zipfile
+import re
+import os
+import argparse
+
+
+def main(dir, name_to_search):
+    # Check all files in the directory and print the name of the package
+    for root, dirs, files in os.walk(dir):
+        wheels = (fname for fname in files if fname.endswith('whl'))
+        for fname in wheels:
+            filename = os.path.join(dir, fname)
+            zfile = zipfile.ZipFile(filename)
+            metadata = [file for file in zfile.infolist()
+                        if file.filename.endswith('METADATA')][0]
+            data = zfile.open(metadata.filename)
+            name = [line.rstrip().decode('ascii')
+                    for line in data.readlines() if b'Name' in line][0]
+            # Extract the name
+            name = re.match('Name: (?P<name>\S+)$', name).groupdict()['name']
+            if name == name_to_search:
+                print(filename)
+                exit(0)
+
+    print('Package {} not found'.format(name_to_search))
+    exit(1)
+
+
+if __name__ == '__main__':
+    desc = 'Return the wheel that that contains the package name'
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument('-d', dest='dir',
+                         help='directory to search')
+    parser.add_argument('name', help='Name of the package to search')
+    args = parser.parse_args()
+
+    main(args.dir, args.name)

requirements.txt

@@ -1,4 +1,9 @@
+uwsgi==2.0.15
 Django==1.11
-pytest-django
-psycopg2
-djangorestframework
+pytest-django==3.1.2
+psycopg2==2.7.3
+djangorestframework==3.6.3
+
+
+# dependencies
+/opt/deps/django-prometheus

vendor/README.md

@@ -0,0 +1,30 @@
+Pregenerated wheels
+===================
+The dependencies can be precompiled in wheels to avoid time compiling while building the service.
+
+The service `build-deps` is doing that. Collect all dependencies from requirements.txt file, downloading them,
+and compiling them as python wheel files. The wheel files are there shared with the host in the ./vendor
+directory
+
+**This step is optional. The container should build with an empty ./vendor directory**
+
+Some extra dependencies (like compilers, dev packages, etc) may be required in `docker/deps/Dockerfile` 
+to allow the creation of the wheel.
+
+To generate the dependencies, run
+
+    docker-compose up --build build-deps
+
+Remember to run it again if you change the recipe (like adding a new dependency), which will rebuild all dependencies. 
+The generation of wheels will be performed on deployment using the cache.
+
+If you want to force the rebuild, run
+
+    docker-compose build --no-cache build-deps
+    docker-compose up build-deps
+
+Note that direct dependencies in ./deps won't generate a wheel*, but their dependencies will. This is
+done to ensure they are always installed fresh, as they are likely to be changed often. See more info
+in ./deps/README.md
+
+* (The wheel will be generated internally to ensure compilation of dependencies, but it will be deleted) *