Merge pull request #552 from pi-hole/deploy-images-from-ci

CircleCI, CD, Python3, and PipEnv

.circleci/config.yml

@@ -6,45 +6,31 @@ version: 2
   steps:
     - checkout
     - run:
-        command: |
-          # setup qemu/variables
-          docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-          HUB_NAMESPACE=$CIRCLE_PROJECT_USERNAME
-          [[ $CIRCLE_PROJECT_USERNAME == "pi-hole" ]] && HUB_NAMESPACE="pihole"
-          [[ $IMAGE != *"/"* ]] && IMAGE="${HUB_NAMESPACE}/${IMAGE}"
-          [[ $IMAGE != *":"* ]] && IMAGE="${IMAGE}:$CIRCLE_JOB"
-          if [[ -n "$CIRCLE_TAG" ]]; then
-              # remove latest tag if used
-              IMAGE="${IMAGE/:latest/:}"
-              # and tack the github tag (version) on the front of the tag. image:arch = image:v1.0-arch
-              IMAGE="${IMAGE/:/:${CIRCLE_TAG}-}"
-              # latest gets a trailing slash, remove it
-              IMAGE="${IMAGE/%-/}"
-          fi
-
-          # generate and build dockerfile
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          ./Dockerfile.py --arch=${CIRCLE_JOB} -v
-          docker images
-          # run docker build & tests
-          # 2 parallel max b/c race condition with docker fixture (I think?)
-          py.test -vv -n 2 -k "${CIRCLE_JOB}" ./test/
-
-          # push image
-          # [[ "$CIRCLE_PR_NUMBER" == "" ]] && sudo docker push $IMAGE
-
+        command: ./circle-test.sh
+    - persist_to_workspace:
+        root: .
+        paths: [ 'ci-workspace' ]
 
 jobs:
   amd64:
     <<: *job_template
-  aarch64:
+  arm64:
     <<: *job_template
   armhf:
     <<: *job_template
   armel:
     <<: *job_template
+  deploy:
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - setup_remote_docker:
+          version: 18.06.0-ce
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          command: ./circle-deploy.sh
 
 
 
@@ -56,7 +42,7 @@ workflows:
           filters:
             tags:
               only: /^v.*/
-      - aarch64:
+      - arm64:
           filters:
             tags:
               only: /^v.*/
@@ -68,3 +54,12 @@ workflows:
           filters:
             tags:
               only: /^v.*/
+      - deploy:
+          requires:
+            - amd64
+            - arm64
+            - armhf
+            - armel
+          filters:
+            tags:
+              only: /^v.*/

.dockerignore

@@ -1 +1,5 @@
 **/*.sw*
+.tox
+.git
+**/__pycache__
+.pipenv

.gitignore

@@ -3,9 +3,11 @@
 .cache
 __pycache__
 .tox
+.pipenv
 .eggs
 UNKNOWN.egg-info
 .env
+ci-workspace
 
 # WIP/test stuff
 doco.yml

Dockerfile.py

@@ -1,21 +1,21 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """ Dockerfile.py - generates and build dockerfiles
 
 Usage:
-  Dockerfile.py [--arch=<arch> ...] [--skip=<arch> ...] [-v] [-t] [--no-build | --no-generate] [--no-cache]
+  Dockerfile.py [--hub_tag=<tag>] [--arch=<arch> ...] [-v] [-t] [--no-build | --no-generate] [--no-cache]
 
 Options:
     --no-build      Skip building the docker images
     --no-cache      Build without using any cache data
     --no-generate   Skip generating Dockerfiles from template
-    --arch=<arch>   What Architecture(s) to build   [default: amd64 armel armhf aarch64]
-    --skip=<arch>   What Architectures(s) to skip   [default: None]
+    --hub_tag=<tag> What the Docker Hub Image should be tagged as [default: None]
+    --arch=<arch>   What Architecture(s) to build   [default: amd64 armel armhf arm64]
     -v              Print docker's command output   [default: False]
     -t              Print docker's build time       [default: False]
 
 Examples:
 """
-from __future__ import print_function
+
 
 from docopt import docopt
 from jinja2 import Environment, FileSystemLoader
@@ -29,7 +29,7 @@
 base_vars = {
     'name': 'pihole/pihole',
     'maintainer' : 'adam@diginc.us',
-    's6_version' : 'v1.21.7.0',
+    's6_version' : 'v1.22.1.0',
 }
 
 os_base_vars = {
@@ -47,19 +47,23 @@
     __version__: [
         {
             'base': 'pihole/debian-base:latest',
-            'arch': 'amd64'
+            'arch': 'amd64',
+            's6arch': 'amd64',
         },
         {
             'base': 'multiarch/debian-debootstrap:armel-stretch-slim',
-            'arch': 'armel'
+            'arch': 'armel',
+            's6arch': 'arm',
         },
         {
             'base': 'multiarch/debian-debootstrap:armhf-stretch-slim',
-            'arch': 'armhf'
+            'arch': 'arm',
+            's6arch' : 'arm',
         },
         {
             'base': 'multiarch/debian-debootstrap:arm64-stretch-slim',
-            'arch': 'aarch64'
+            'arch': 'arm64',
+            's6arch' : 'aarch64',
         }
     ]
 }
@@ -69,19 +73,17 @@ def generate_dockerfiles(args):
         print(" ::: Skipping Dockerfile generation")
         return
 
-    for version, archs in images.iteritems():
+    for version, archs in images.items():
         for image in archs:
-            if image['arch'] not in args['--arch'] or image['arch'] in args['--skip']:
-                    return
-            s6arch = image['arch']
-            if image['arch'] == 'armel':
-                s6arch = 'arm'
+            if image['arch'] not in args['--arch']:
+                continue
+            s6arch = image['s6arch'] if image['s6arch'] else image['arch']
             merged_data = dict(
-                { 'version': version }.items() +
-                base_vars.items() +
-                os_base_vars.items() +
-                image.items() +
-                { 's6arch': s6arch }.items()
+                list({ 'version': version }.items()) +
+                list(base_vars.items()) +
+                list(os_base_vars.items()) +
+                list(image.items()) +
+                list({ 's6arch': s6arch }.items())
             )
             j2_env = Environment(loader=FileSystemLoader(THIS_DIR),
                                  trim_blocks=True)
@@ -98,17 +100,28 @@ def build_dockerfiles(args):
         return
 
     for arch in args['--arch']:
-        # TODO: include from external .py that can be shared with Dockerfile.py / Tests / deploy scripts '''
-        #if arch == 'armel':
-        #    print("Skipping armel, incompatible upstream binaries/broken")
-        #    continue
         build('pihole', arch, args)
 
 
+def run_and_stream_command_output(command, args):
+    print("Running", command)
+    build_result = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                    bufsize=1, universal_newlines=True)
+    if args['-v']:
+        while build_result.poll() is None:
+            for line in build_result.stdout:
+                print(line, end='')
+    build_result.wait()
+    if build_result.returncode != 0:
+        print("     ::: Error running".format(command))
+        print(build_result.stderr)
+
+
 def build(docker_repo, arch, args):
     dockerfile = 'Dockerfile_{}'.format(arch)
     repo_tag = '{}:{}_{}'.format(docker_repo, __version__, arch)
     cached_image = '{}/{}'.format('pihole', repo_tag)
+    print(" ::: Building {}".format(repo_tag))
     time=''
     if args['-t']:
         time='time '
@@ -118,22 +131,20 @@ def build(docker_repo, arch, args):
     build_command = '{time}docker build {no_cache} --pull --cache-from="{cache},{create_tag}" -f {dockerfile} -t {create_tag} .'\
         .format(time=time, no_cache=no_cache, cache=cached_image, dockerfile=dockerfile, create_tag=repo_tag)
     print(" ::: Building {} into {}".format(dockerfile, repo_tag))
+    run_and_stream_command_output(build_command, args)
     if args['-v']:
         print(build_command, '\n')
-    build_result = subprocess.Popen(build_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if args['-v']:
-        for c in iter(lambda: build_result.stdout.read(1), b''):
-            sys.stdout.write(c)
-    build_result.wait()
-    if build_result.returncode != 0:
-        print("     ::: Building {} encountered an error".format(dockerfile))
-        print(build_result.stderr)
-    assert build_result.returncode == 0
+    if args['--hub_tag']:
+        hub_tag_command = "{time}docker tag {create_tag} {hub_tag}"\
+            .format(time=time, create_tag=repo_tag, hub_tag=args['--hub_tag'])
+        print(" ::: Tagging {} into {}".format(repo_tag, args['--hub_tag']))
+        run_and_stream_command_output(hub_tag_command, args)
 
 
 if __name__ == '__main__':
-    args = docopt(__doc__, version='Dockerfile 1.0')
-    # print args
+    args = docopt(__doc__, version='Dockerfile 1.1')
+    if args['-v']:
+        print(args)
 
     generate_dockerfiles(args)
     build_dockerfiles(args)

Dockerfile.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+# alpine sh only
+
+set -eux
+./Dockerfile.py -v --arch="${ARCH}" --hub_tag="${ARCH_IMAGE}"
+# TODO: Add junitxml output and have circleci consume it
+# 2 parallel max b/c race condition with docker fixture (I think?)
+py.test -vv -n 2 -k "${ARCH}" ./test/

Dockerfile_amd64

@@ -1,7 +1,7 @@
 FROM pihole/debian-base:latest
 
 ENV ARCH amd64
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-amd64.tar.gz
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-amd64.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version

Dockerfile_aarch64 → Dockerfile_arm64

@@ -1,7 +1,7 @@
 FROM multiarch/debian-debootstrap:arm64-stretch-slim
 
-ENV ARCH aarch64
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-aarch64.tar.gz
+ENV ARCH arm64
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-aarch64.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version
@@ -40,7 +40,7 @@ ENV DNSMASQ_USER root
 ENV VERSION v4.3.2
 ENV PATH /opt/pihole:${PATH}
 
-LABEL image="pihole/pihole:v4.3.2_aarch64"
+LABEL image="pihole/pihole:v4.3.2_arm64"
 LABEL maintainer="adam@diginc.us"
 LABEL url="https://www.github.com/pi-hole/docker-pi-hole"
 

Dockerfile_armel

@@ -1,7 +1,7 @@
 FROM multiarch/debian-debootstrap:armel-stretch-slim
 
 ENV ARCH armel
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-arm.tar.gz
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-arm.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version

Dockerfile_build

@@ -0,0 +1,23 @@
+FROM docker:latest
+
+# Based on https://github.com/Ilhicas/alpine-pipenv
+ARG packages
+RUN apk --update add python3 python3-dev curl gcc make \
+        musl-dev libffi-dev openssl-dev ${packages} \
+    && rm -rf /var/cache/apk/* \
+    && pip3 install -U pip pipenv
+
+
+# -v "$(pwd):/$(pwd)" -w "$(pwd)" to prevent nested docker path confusion
+COPY ./Dockerfile.sh /usr/local/bin/
+COPY Pipfile* /root/
+WORKDIR /root
+
+RUN pipenv install --system \
+    && sed -i 's|/bin/sh|/bin/bash|g' /usr/lib/python3.8/site-packages/testinfra/backend/docker.py
+
+
+RUN echo "set -ex && Dockerfile.sh && \$@" > /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT entrypoint.sh
+CMD Dockerfile.sh

Pipfile

@@ -0,0 +1,63 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+apipkg = "==1.5"
+atomicwrites = "==1.3.0"
+attrs = "==19.3.0"
+bcrypt = "==3.1.7"
+cached-property = "==1.5.1"
+certifi = "==2019.11.28"
+cffi = "==1.13.2"
+chardet = "==3.0.4"
+configparser = "==4.0.2"
+contextlib2 = "==0.6.0.post1"
+coverage = "==5.0.1"
+cryptography = "==2.8"
+docker = "==4.1.0"
+dockerpty = "==0.4.1"
+docopt = "==0.6.2"
+enum34 = "==1.1.6"
+execnet = "==1.7.1"
+filelock = "==3.0.12"
+funcsigs = "==1.0.2"
+idna = "==2.8"
+importlib-metadata = "==1.3.0"
+ipaddress = "==1.0.23"
+jsonschema = "==3.2.0"
+more-itertools = "==5.0.0"
+pathlib2 = "==2.3.5"
+pluggy = "==0.13.1"
+py = "==1.8.1"
+pycparser = "==2.19"
+pyparsing = "==2.4.6"
+pyrsistent = "==0.15.6"
+pytest = "==4.6.8"
+pytest-cov = "==2.8.1"
+pytest-forked = "==1.1.3"
+pytest-xdist = "==1.31.0"
+requests = "==2.22.0"
+scandir = "==1.10.0"
+six = "==1.13.0"
+subprocess32 = "==3.5.4"
+testinfra = "==3.3.0"
+texttable = "==1.6.2"
+toml = "==0.10.0"
+tox = "==3.14.3"
+urllib3 = "==1.25.7"
+virtualenv = "==16.7.9"
+wcwidth = "==0.1.7"
+zipp = "==0.6.0"
+"backports.shutil_get_terminal_size" = "==1.0.0"
+"backports.ssl_match_hostname" = "==3.7.0.1"
+Jinja2 = "==2.10.3"
+MarkupSafe = "==1.1.1"
+PyYAML = "==5.2"
+websocket_client = "==0.57.0"
+
+[requires]
+python_version = "3.8"