Update way too much

- Tox py3.7 + pipenv - Python3 Dockerfile.py - Dockerfile.py tags remote instead of just local image names now - Circle.sh instead of in-line circle.yml script breakout - probably other stuff I forgot - Docker images build during the tests should hopefullly now be available at the deploy job workflow thanks to shared docker layers. - Rename aarch64 to arm64 to reduce custom map
pi-hole · Jan 31, 2020 · 3765d41 · 3765d41
1 parent 939d299
commit 3765d41
Show file tree

Hide file tree

Showing 22 changed files with 921 additions and 145 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,56 +5,34 @@ version: 2
     enabled: true
   steps:
     - checkout
+    #- attach_workspace:
+    #    at: ./ci-workspace/
     - run:
-        command: |
-          # setup qemu/variables
-          docker run --rm --privileged multiarch/qemu-user-static:register --reset > /dev/null
-
-          IMAGE="${IMAGE:-$CIRCLE_PROJECT_REPONAME}"
-          HUB_NAMESPACE="${HUB_NAMESPACE:-$CIRCLE_PROJECT_USERNAME}"
-          [[ $CIRCLE_PROJECT_USERNAME == "pi-hole" ]] && HUB_NAMESPACE="pihole"
-          [[ $IMAGE != *"/"* ]] && IMAGE="${HUB_NAMESPACE}/${IMAGE}"
-          [[ $IMAGE != *":"* ]] && IMAGE="${IMAGE}:$CIRCLE_JOB"
-          if [[ -n "$CIRCLE_TAG" ]]; then
-              # remove latest tag if used
-              IMAGE="${IMAGE/:latest/:}"
-              # and tack the github tag (version) on the front of the tag. image:arch = image:v1.0-arch
-              IMAGE="${IMAGE/:/:${CIRCLE_TAG}-}"
-              # latest gets a trailing slash, remove it
-              IMAGE="${IMAGE/%-/}"
-          fi
-
-          # generate and build dockerfile
-          pip install -q --upgrade pip
-          pip install -q -r requirements.txt
-          ./Dockerfile.py --arch=${CIRCLE_JOB}
-          docker images
-          # run docker build & tests
-          # 2 parallel max b/c race condition with docker fixture (I think?)
-          py.test -vv -n 2 -k "${CIRCLE_JOB}" ./test/
-
-          # push image
-          if [[ "$CIRCLE_PR_NUMBER" == "" ]]; then
-              if [[ -n "$CIRCLE_TAG" ]] ; then
-                  echo "Building a tag: $CIRCLE_TAG"
-                  version="$CIRCLE_TAG"
-                  # TODO: write script to get all releases, see if our tag is newest and tell deploy script to update the latest tag (for manfiest push stage only)
-                  # python3 latest_version_check.py && latest="true"
-              fi
-              echo $DOCKERHUB_PASS | docker login --username=$DOCKERHUB_USER --password-stdin
-              ./deploy_docker.sh
-          fi
-
+        command: ./circle-test.sh
+    - persist_to_workspace:
+        root: .
+        paths: [ 'ci-workspace' ]
 
 jobs:
   amd64:
     <<: *job_template
-  aarch64:
+  arm64:
     <<: *job_template
   armhf:
     <<: *job_template
   armel:
     <<: *job_template
+  deploy:
+    docker:
+      - image: circleci/python:latest
+    steps:
+      - setup_remote_docker:
+          version: 18.06.0-ce
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          command: ./circle-deploy.sh
 
 
 
@@ -66,7 +44,7 @@ workflows:
           filters:
             tags:
               only: /^v.*/
-      - aarch64:
+      - arm64:
           filters:
             tags:
               only: /^v.*/
@@ -78,9 +56,12 @@ workflows:
           filters:
             tags:
               only: /^v.*/
-            #branches:
-            #  only:
-            #    - dev
-            #    - master
-            #    - release
-            #    - prerelease
+      - deploy:
+          requires:
+            - amd64
+            - arm64
+            - armhf
+            - armel
+          filters:
+            tags:
+              only: /^v.*/
diff --git a/.dockerignore b/.dockerignore
@@ -1 +1,5 @@
 **/*.sw*
+.tox
+.git
+**/__pycache__
+.pipenv
diff --git a/.gitignore b/.gitignore
@@ -3,9 +3,11 @@
 .cache
 __pycache__
 .tox
+.pipenv
 .eggs
 UNKNOWN.egg-info
 .env
+ci-workspace
 
 # WIP/test stuff
 doco.yml
diff --git a/Dockerfile.py b/Dockerfile.py
@@ -1,21 +1,21 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """ Dockerfile.py - generates and build dockerfiles
 
 Usage:
-  Dockerfile.py [--arch=<arch> ...] [--skip=<arch> ...] [-v] [-t] [--no-build | --no-generate] [--no-cache]
+  Dockerfile.py [--hub_tag=<tag>] [--arch=<arch> ...] [-v] [-t] [--no-build | --no-generate] [--no-cache]
 
 Options:
     --no-build      Skip building the docker images
     --no-cache      Build without using any cache data
     --no-generate   Skip generating Dockerfiles from template
-    --arch=<arch>   What Architecture(s) to build   [default: amd64 armel armhf aarch64]
-    --skip=<arch>   What Architectures(s) to skip   [default: None]
+    --hub_tag=<tag> What the Docker Hub Image should be tagged as [default: None]
+    --arch=<arch>   What Architecture(s) to build   [default: amd64 armel armhf arm64]
     -v              Print docker's command output   [default: False]
     -t              Print docker's build time       [default: False]
 
 Examples:
 """
-from __future__ import print_function
+
 
 from docopt import docopt
 from jinja2 import Environment, FileSystemLoader
@@ -29,7 +29,7 @@
 base_vars = {
     'name': 'pihole/pihole',
     'maintainer' : 'adam@diginc.us',
-    's6_version' : 'v1.21.7.0',
+    's6_version' : 'v1.22.1.0',
 }
 
 os_base_vars = {
@@ -47,19 +47,23 @@
     __version__: [
         {
             'base': 'pihole/debian-base:latest',
-            'arch': 'amd64'
+            'arch': 'amd64',
+            's6arch': 'amd64',
         },
         {
             'base': 'multiarch/debian-debootstrap:armel-stretch-slim',
-            'arch': 'armel'
+            'arch': 'armel',
+            's6arch': 'arm',
         },
         {
             'base': 'multiarch/debian-debootstrap:armhf-stretch-slim',
-            'arch': 'armhf'
+            'arch': 'arm',
+            's6arch' : 'arm',
         },
         {
             'base': 'multiarch/debian-debootstrap:arm64-stretch-slim',
-            'arch': 'aarch64'
+            'arch': 'arm64',
+            's6arch' : 'aarch64',
         }
     ]
 }
@@ -69,19 +73,17 @@ def generate_dockerfiles(args):
         print(" ::: Skipping Dockerfile generation")
         return
 
-    for version, archs in images.iteritems():
+    for version, archs in images.items():
         for image in archs:
-            if image['arch'] not in args['--arch'] or image['arch'] in args['--skip']:
-                    return
-            s6arch = image['arch']
-            if image['arch'] == 'armel':
-                s6arch = 'arm'
+            if image['arch'] not in args['--arch']:
+                continue
+            s6arch = image['s6arch'] if image['s6arch'] else image['arch']
             merged_data = dict(
-                { 'version': version }.items() +
-                base_vars.items() +
-                os_base_vars.items() +
-                image.items() +
-                { 's6arch': s6arch }.items()
+                list({ 'version': version }.items()) +
+                list(base_vars.items()) +
+                list(os_base_vars.items()) +
+                list(image.items()) +
+                list({ 's6arch': s6arch }.items())
             )
             j2_env = Environment(loader=FileSystemLoader(THIS_DIR),
                                  trim_blocks=True)
@@ -98,17 +100,28 @@ def build_dockerfiles(args):
         return
 
     for arch in args['--arch']:
-        # TODO: include from external .py that can be shared with Dockerfile.py / Tests / deploy scripts '''
-        #if arch == 'armel':
-        #    print("Skipping armel, incompatible upstream binaries/broken")
-        #    continue
         build('pihole', arch, args)
 
 
+def run_and_stream_command_output(command, args):
+    print("Running", command)
+    build_result = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                    bufsize=1, universal_newlines=True)
+    if args['-v']:
+        while build_result.poll() is None:
+            for line in build_result.stdout:
+                print(line, end='')
+    build_result.wait()
+    if build_result.returncode != 0:
+        print("     ::: Error running".format(command))
+        print(build_result.stderr)
+
+
 def build(docker_repo, arch, args):
     dockerfile = 'Dockerfile_{}'.format(arch)
     repo_tag = '{}:{}_{}'.format(docker_repo, __version__, arch)
     cached_image = '{}/{}'.format('pihole', repo_tag)
+    print(" ::: Building {}".format(repo_tag))
     time=''
     if args['-t']:
         time='time '
@@ -118,22 +131,20 @@ def build(docker_repo, arch, args):
     build_command = '{time}docker build {no_cache} --pull --cache-from="{cache},{create_tag}" -f {dockerfile} -t {create_tag} .'\
         .format(time=time, no_cache=no_cache, cache=cached_image, dockerfile=dockerfile, create_tag=repo_tag)
     print(" ::: Building {} into {}".format(dockerfile, repo_tag))
+    run_and_stream_command_output(build_command, args)
     if args['-v']:
         print(build_command, '\n')
-    build_result = subprocess.Popen(build_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if args['-v']:
-        for c in iter(lambda: build_result.stdout.read(1), b''):
-            sys.stdout.write(c)
-    build_result.wait()
-    if build_result.returncode != 0:
-        print("     ::: Building {} encountered an error".format(dockerfile))
-        print(build_result.stderr)
-    assert build_result.returncode == 0
+    if args['--hub_tag']:
+        hub_tag_command = "{time}docker tag {create_tag} {hub_tag}"\
+            .format(time=time, create_tag=repo_tag, hub_tag=args['--hub_tag'])
+        print(" ::: Tagging {} into {}".format(repo_tag, args['--hub_tag']))
+        run_and_stream_command_output(hub_tag_command, args)
 
 
 if __name__ == '__main__':
-    args = docopt(__doc__, version='Dockerfile 1.0')
-    # print args
+    args = docopt(__doc__, version='Dockerfile 1.1')
+    if args['-v']:
+        print(args)
 
     generate_dockerfiles(args)
     build_dockerfiles(args)
diff --git a/Dockerfile.sh b/Dockerfile.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+# alpine sh only
+
+set -eux
+./Dockerfile.py -v --arch="${ARCH}" --hub_tag="${ARCH_IMAGE}"
+# TODO: Add junitxml output and have circleci consume it
+# 2 parallel max b/c race condition with docker fixture (I think?)
+py.test -vv -n 2 -k "${ARCH}" ./test/
diff --git a/Dockerfile_amd64 b/Dockerfile_amd64
@@ -1,7 +1,7 @@
 FROM pihole/debian-base:latest
 
 ENV ARCH amd64
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-amd64.tar.gz
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-amd64.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version

diff --git a/Dockerfile_aarch64 → Dockerfile_arm64 b/Dockerfile_aarch64 → Dockerfile_arm64
@@ -1,7 +1,7 @@
 FROM multiarch/debian-debootstrap:arm64-stretch-slim
 
-ENV ARCH aarch64
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-aarch64.tar.gz
+ENV ARCH arm64
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-aarch64.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version
@@ -40,7 +40,7 @@ ENV DNSMASQ_USER root
 ENV VERSION v4.3.2
 ENV PATH /opt/pihole:${PATH}
 
-LABEL image="pihole/pihole:v4.3.2_aarch64"
+LABEL image="pihole/pihole:v4.3.2_arm64"
 LABEL maintainer="adam@diginc.us"
 LABEL url="https://www.github.com/pi-hole/docker-pi-hole"
 

diff --git a/Dockerfile_armel b/Dockerfile_armel
@@ -1,7 +1,7 @@
 FROM multiarch/debian-debootstrap:armel-stretch-slim
 
 ENV ARCH armel
-ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.21.7.0/s6-overlay-arm.tar.gz
+ENV S6OVERLAY_RELEASE https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-arm.tar.gz
 
 COPY install.sh /usr/local/bin/install.sh
 COPY VERSION /etc/docker-pi-hole-version

diff --git a/Dockerfile_build b/Dockerfile_build
@@ -0,0 +1,23 @@
+FROM docker:latest
+
+# Based on https://github.com/Ilhicas/alpine-pipenv
+ARG packages
+RUN apk --update add python3 python3-dev curl gcc make \
+        musl-dev libffi-dev openssl-dev ${packages} \
+    && rm -rf /var/cache/apk/* \
+    && pip3 install -U pip pipenv
+
+
+# -v "$(pwd):/$(pwd)" -w "$(pwd)" to prevent nested docker path confusion
+COPY ./Dockerfile.sh /usr/local/bin/
+COPY Pipfile* /root/
+WORKDIR /root
+
+RUN pipenv install --system \
+    && sed -i 's|/bin/sh|/bin/bash|g' /usr/lib/python3.8/site-packages/testinfra/backend/docker.py
+
+
+RUN echo "set -ex && Dockerfile.sh && \$@" > /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT entrypoint.sh
+CMD Dockerfile.sh