Some backports from work on #46 to greatly speed up Circle builds

This backports from topic-docker the base image, which we can now use in Circle to MASSIVELY speed up our CI situation. The base image can currently only be pushed by @klardotsh as nobody else has access to the Docker Hub org, but that can change in the future.
KMKfw · Oct 12, 2018 · 7d2d04f · 7d2d04f
1 parent 3c5c95c
commit 7d2d04f
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 85 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,155 +1,79 @@
 version: 2
 jobs:
-  lint:
-    docker:
-      - image: 'ubuntu:bionic'
-
-    environment:
-        KMK_TEST: 1
-        PIPENV_VENV_IN_PROJECT: 1
-
-    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh wget
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
-      - run: make lint
-
-      - save_cache:
-          key: v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-          paths:
-            - .venv
-
   test:
     docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'
 
     environment:
         KMK_TEST: 1
         PIPENV_VENV_IN_PROJECT: 1
 
     steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
       - checkout
       - restore_cache:
           keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}
 
-      - run: python3.7 -m pip install pipenv==2018.7.1
       - run: make test
 
   build_feather_m4_express:
     docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'
 
     environment:
         KMK_TEST: 1
         PIPENV_VENV_IN_PROJECT: 1
 
     steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
       - checkout
       - restore_cache:
           keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}
 
       - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-feather-m4-express
 
   build_itsybitsy_m4_express:
     docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'
 
     environment:
         KMK_TEST: 1
         PIPENV_VENV_IN_PROJECT: 1
 
     steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
       - checkout
       - restore_cache:
           keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}
 
       - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-itsybitsy-m4-express
 
   build_pyboard:
     docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'
 
     environment:
         KMK_TEST: 1
         PIPENV_VENV_IN_PROJECT: 1
 
     steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
       - checkout
       - restore_cache:
           keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}
 
       - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-pyboard
 
 workflows:
   version: 2
   build-deploy:
     jobs:
-      - lint:
-          filters:
-            branches:
-              only: /.*/
-            tags:
-              only: /.*/
       - test:
           filters:
             branches:
               only: /.*/
             tags:
               only: /.*/
-          requires:
-            - lint
       - build_feather_m4_express:
           filters:
             branches:

diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,10 @@
+build
+vendor
+.devdeps
+.submodules
+.idea
+.micropython-deps
+.circuitpython-deps
+.ampy
+.venv
+venv
diff --git a/.gitignore b/.gitignore
@@ -108,6 +108,7 @@ venv.bak/
 .circuitpy-deps
 .micropython-deps
 .devdeps
+.docker_base
 
 # Pycharms cruft
 .idea
diff --git a/Dockerfile_base b/Dockerfile_base
@@ -0,0 +1,21 @@
+# vim: ft=dockerfile
+
+# Not using python:3.7 here because team-gcc-arm-embedded/ppa does not support
+# Ubuntu Cosmic or Debian Stretch, and Alpine, bizarrely, does not seem to
+# package GCC cross compilers
+FROM ubuntu:bionic
+
+# Set up PPAs we'll need for Python and for GCC ARM
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN add-apt-repository ppa:team-gcc-arm-embedded/ppa
+
+# Install Python
+RUN apt-get update && apt-get install -y python3.7 python3.7-dev build-essential pkg-config libffi-dev curl
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+# Downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
+RUN python3.7 -m pip install pip==18.0
+RUN python3.7 -m pip install pipenv==2018.7.1
+
+# Install KMK CI and/or build-time dependencies
+RUN apt-get install -y gcc-arm-embedded gettext ssh wget unzip rsync git locales libusb-dev
diff --git a/Makefile b/Makefile
@@ -8,12 +8,26 @@
 	freeze-nrf-build-deps \
 	lint
 
+DOCKER_BASE_TAG ?= latest
+DOCKER_TAG ?= latest
+
 AMPY_PORT ?= /dev/ttyUSB0
 AMPY_BAUD ?= 115200
 AMPY_DELAY ?= 1.5
 ARDUINO ?= /usr/share/arduino
 PIPENV ?= $(shell which pipenv)
 
+.docker_base: Dockerfile_base
+	@echo "===> Building Docker base image kmkfw/base:${DOCKER_BASE_TAG}"
+	@docker build -f Dockerfile_base -t kmkfw/base:${DOCKER_BASE_TAG} .
+	@touch .docker_base
+
+docker-base: .docker_base
+
+docker-base-deploy: docker-base
+	@echo "===> Pushing Docker base image kmkfw/base:${DOCKER_BASE_TAG} to Docker Hub"
+	@docker push kmkfw/base:${DOCKER_BASE_TAG}
+
 .devdeps: Pipfile.lock
 	@echo "===> Installing dependencies with pipenv"
 	@$(PIPENV) install --dev --ignore-pipfile
@@ -41,7 +55,7 @@ powerwash: clean
 	@echo "===> Removing pipenv-managed virtual environment"
 	@$(PIPENV) --rm || true
 
-test: micropython-build-unix
+test: lint micropython-build-unix
 	@echo "===> Testing keymap_sanity_check.py script"
 	@echo "    --> Known good layout should pass..."
 	@MICROPYPATH=tests/test_data:./ ./bin/micropython.sh bin/keymap_sanity_check.py keymaps/known_good.py