From 44948c58081931f42a81a0f215337a6f00c27d3a Mon Sep 17 00:00:00 2001 From: Jaime Buelta Date: Thu, 27 Jul 2017 00:05:44 +0100 Subject: [PATCH] Add vendor support to generate wheels from dependencies --- .gitignore | 2 ++ .gitmodules | 3 ++ Dockerfile | 10 +++++-- README.md | 9 ++++++ deps/README.md | 57 ++++++++++++++++++++++++++++++++++++ deps/django-prometheus | 1 + docker-compose.yaml | 9 ++++++ docker/deps/Dockerfile | 21 +++++++++++++ docker/deps/build_deps.sh | 23 +++++++++++++++ docker/deps/copy_deps.sh | 6 ++++ docker/deps/search_wheels.py | 37 +++++++++++++++++++++++ requirements.txt | 11 +++++-- vendor/README.md | 30 +++++++++++++++++++ 13 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 100644 deps/README.md create mode 160000 deps/django-prometheus create mode 100644 docker/deps/Dockerfile create mode 100755 docker/deps/build_deps.sh create mode 100755 docker/deps/copy_deps.sh create mode 100644 docker/deps/search_wheels.py create mode 100644 vendor/README.md diff --git a/.gitignore b/.gitignore index 816f75d..67caede 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,5 @@ ENV/ .mypy_cache/ .DS_Store + +*.whl diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7921423 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/django-prometheus"] + path = deps/django-prometheus + url = https://github.com/korfuri/django-prometheus.git diff --git a/Dockerfile b/Dockerfile index e42406b..a423e0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 # Add requirements for python and pip RUN apk add --update python3 pytest -RUN apk add --update postgresql-dev +RUN apk add --update postgresql-libs RUN apk add --update curl RUN mkdir -p /opt/code @@ -10,11 +10,17 @@ WORKDIR /opt/code ADD requirements.txt /opt/code +# Try to use local wheels. Even if not present, it will proceed +ADD ./vendor /opt/vendor +ADD ./deps /opt/deps +# Only install them if there's any +RUN if ls /opt/vendor/*.whl 1> /dev/null 2>&1; then pip3 install /opt/vendor/*.whl; fi + # Some Docker-fu. In one step install the compile packages, install the # dependencies and then remove them. That skims the image size quite # sensibly. RUN apk add --no-cache --virtual .build-deps \ - python3-dev build-base linux-headers gcc \ + python3-dev build-base linux-headers gcc postgresql-dev \ # Installing python requirements && pip3 install -r requirements.txt \ && find /usr/local \ diff --git a/README.md b/README.md index d2f6ede..01ba052 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Tree structure ├── docker-compose.yaml (general dockerfile description, aimed at development) ├── Dockerfile (General Dockerfile of the main server) ├── requirements.txt (python requirements) +├── vendor (cache with generated wheels from dependencies) +├── deps (git submodules with embedded dependencies) ├── docker (Files related to build and operation of containers) │   └── (docker subdirs, like db or server) │      └── Scripts related to docker creation and operation of that service @@ -88,6 +90,13 @@ in the DB do, like new fixtures or a new migration. Build all services with docker-compose build ``` +- *build-deps*: Precompile all dependencies into wheels and copy them in ./vendor. This is not +required, but can save time when rebuilding the containers often, to avoid compile the same code +over and over. Check the more detailed documentation in the ./vendor/README.md file. + Note that dependencies embedded in ./deps won't be compiled (though their dependencies will be). +Check more details in the ./deps/README.md file. + + Docker services oriented to production ========= diff --git a/deps/README.md b/deps/README.md new file mode 100644 index 0000000..6682829 --- /dev/null +++ b/deps/README.md @@ -0,0 +1,57 @@ +Embedded dependencies +===================== +This directory is created to set direct dependencies that are not under pypi control or have more +manual installation. This is mainly aimed to modules that live in private git repos and are not +downloadable from PyPI, avoiding problems like requiring to pull from the repo with a ssh key +from inside the container. + +The recommended way of dealing with them is to add them into this subdir git submodules and use +the Python [setuptool module](https://docs.python.org/3.6/distributing/index.html) (setup.py). + +Note that dependencies here won't be installed from wheel, though their dependencies will be (if +done in the proper format, through a setup.py). That's to avoid problems with setup and cache the +wrong version, with often happens while developing. +Remember to include the dependency in the requirements.txt file + +An example of a module has been included (django-prometheus) + +How to add a new submodule +========================== + + cd deps + git submodule add https://github.com/foo + +this creates the subdir foo with the submodule. The file .gitmodules will be updated and needs +to be tracked and commited. + +How to update a submodule +========================== + +Log into the submodule and set git to the desired commit/tag/branch + + cd deps/foo + git checkout v7.5.0 + # or, for latest commit in current branch + git pull + +Then add the commit to the main repo, like a regular file + + cd .. + git add foo + git commit + + +If the submodule is updated +============================ + +and yours is not the proper version, it will appear as + + git status + modified: foo (new commits) + +Get the new commits with the command + + git submodule update --remote + + +NOTE: Working with git submodules is a little tricky. Feel free to add and modify this document. diff --git a/deps/django-prometheus b/deps/django-prometheus new file mode 160000 index 0000000..b1245aa --- /dev/null +++ b/deps/django-prometheus @@ -0,0 +1 @@ +Subproject commit b1245aaec22a568389d053d26f386db0b0bf4ce0 diff --git a/docker-compose.yaml b/docker-compose.yaml index 747b06c..443f5a4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,6 +6,13 @@ services: build: context: . dockerfile: ./docker/db/Dockerfile-postgres + build-deps: + build: + context: . + dockerfile: ./docker/deps/Dockerfile + volumes: + - ./vendor:/opt/ext_vendor + command: /opt/copy_deps.sh dev-server: build: . command: ./start_dev_server.sh @@ -22,6 +29,8 @@ services: - ./src:/opt/code depends_on: - db + - build-deps + # Producion related server: build: . diff --git a/docker/deps/Dockerfile b/docker/deps/Dockerfile new file mode 100644 index 0000000..963dea0 --- /dev/null +++ b/docker/deps/Dockerfile @@ -0,0 +1,21 @@ +FROM alpine:3.5 +RUN mkdir -p /opt/vendor +WORKDIR /opt/vendor +RUN apk update +# Basic python usage +RUN apk add python3 +RUN apk add py3-pip + +# Required for compiling +RUN apk add python3-dev build-base linux-headers gcc postgresql-dev +RUN pip3 install cython wheel + +ADD ./deps /opt/deps +RUN mkdir -p /opt/vendor +ADD requirements.txt /opt/deps +ADD ./docker/deps/build_deps.sh /opt/ +ADD ./docker/deps/copy_deps.sh /opt/ +ADD ./docker/deps/search_wheels.py /opt/ + +WORKDIR /opt/ +RUN ./build_deps.sh diff --git a/docker/deps/build_deps.sh b/docker/deps/build_deps.sh new file mode 100755 index 0000000..f8f9ebb --- /dev/null +++ b/docker/deps/build_deps.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +echo 'Building dependencies' +mkdir -p /opt/wheels/ +cd /opt/vendor + +echo 'Building wheels from requirements' +pip3 wheel -r /opt/deps/requirements.txt --process-dependency-links + +echo 'Done' + +# Clean the direct dependencies to avoid issues with caches +for D in /opt/deps/*; do + if [ -d "${D}" ]; then + echo "Removing dependency in ${D}" # your processing here + PKG_NAME=`python3 ${D}/setup.py --name` + WHEEL=`python3 /opt/search_wheels.py $PKG_NAME -d /opt/vendor` + echo "Deleting file $WHEEL" + rm $WHEEL + fi +done + +echo "Wheels available: `ls /opt/vendor/*.whl`" diff --git a/docker/deps/copy_deps.sh b/docker/deps/copy_deps.sh new file mode 100755 index 0000000..bacf29b --- /dev/null +++ b/docker/deps/copy_deps.sh @@ -0,0 +1,6 @@ +#!/bin/sh +echo 'Deleting wheels and copy new ones' +rm /opt/ext_vendor/*.whl +echo "Dependencies are created at build. Run build --no-cache to recreate" +cp /opt/vendor/* /opt/ext_vendor + diff --git a/docker/deps/search_wheels.py b/docker/deps/search_wheels.py new file mode 100644 index 0000000..07c0fbd --- /dev/null +++ b/docker/deps/search_wheels.py @@ -0,0 +1,37 @@ +import zipfile +import re +import os +import argparse + + +def main(dir, name_to_search): + # Check all files in the directory and print the name of the package + for root, dirs, files in os.walk(dir): + wheels = (fname for fname in files if fname.endswith('whl')) + for fname in wheels: + filename = os.path.join(dir, fname) + zfile = zipfile.ZipFile(filename) + metadata = [file for file in zfile.infolist() + if file.filename.endswith('METADATA')][0] + data = zfile.open(metadata.filename) + name = [line.rstrip().decode('ascii') + for line in data.readlines() if b'Name' in line][0] + # Extract the name + name = re.match('Name: (?P\S+)$', name).groupdict()['name'] + if name == name_to_search: + print(filename) + exit(0) + + print('Package {} not found'.format(name_to_search)) + exit(1) + + +if __name__ == '__main__': + desc = 'Return the wheel that that contains the package name' + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('-d', dest='dir', + help='directory to search') + parser.add_argument('name', help='Name of the package to search') + args = parser.parse_args() + + main(args.dir, args.name) diff --git a/requirements.txt b/requirements.txt index 0ea2f7d..609c71a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,9 @@ +uwsgi==2.0.15 Django==1.11 -pytest-django -psycopg2 -djangorestframework +pytest-django==3.1.2 +psycopg2==2.7.3 +djangorestframework==3.6.3 + + +# dependencies +/opt/deps/django-prometheus diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 0000000..abe6ad8 --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,30 @@ +Pregenerated wheels +=================== +The dependencies can be precompiled in wheels to avoid time compiling while building the service. + +The service `build-deps` is doing that. Collect all dependencies from requirements.txt file, downloading them, +and compiling them as python wheel files. The wheel files are there shared with the host in the ./vendor +directory + +**This step is optional. The container should build with an empty ./vendor directory** + +Some extra dependencies (like compilers, dev packages, etc) may be required in `docker/deps/Dockerfile` +to allow the creation of the wheel. + +To generate the dependencies, run + + docker-compose up --build build-deps + +Remember to run it again if you change the recipe (like adding a new dependency), which will rebuild all dependencies. +The generation of wheels will be performed on deployment using the cache. + +If you want to force the rebuild, run + + docker-compose build --no-cache build-deps + docker-compose up build-deps + +Note that direct dependencies in ./deps won't generate a wheel*, but their dependencies will. This is +done to ensure they are always installed fresh, as they are likely to be changed often. See more info +in ./deps/README.md + +* (The wheel will be generated internally to ensure compilation of dependencies, but it will be deleted) *