Enables multi-arch Docker build on CircleCI (#207)

* CircleCI: build multiarch docker image

* CircleCI: build multiarch docker image #2

* CircleCI: build multiarch docker image #3

* CircleCI: build multiarch docker image #4

* CircleCI: build multiarch docker image #5

* CircleCI: build multiarch docker image #6

* CircleCI: build multiarch docker image #7

* CircleCI: build multiarch docker image #8

* CircleCI: build multiarch docker image #9

* CircleCI: build multiarch docker image #10

* CircleCI: build multiarch docker image #11

* CircleCI: build multiarch docker image #12

* CircleCI: build multiarch docker image #13

* CircleCI: build multiarch docker image #14

* CircleCI: build multiarch docker image #15

* CircleCI: build multiarch docker image #16

* CircleCI: build multiarch docker image #17

* CircleCI: build multiarch docker image #18

* CircleCI: build multiarch docker image #19

* CircleCI: build multiarch docker image #20

* CircleCI: build multiarch docker image #21

* CircleCI: build multiarch docker image #22

* CircleCI: build multiarch docker image #23

* CircleCI: build multiarch docker image #24

* CircleCI: build multiarch docker image #25

* CircleCI: build multiarch docker image #26

* CircleCI: build multiarch docker image #27

* CircleCI: build multiarch docker image #28

* CircleCI: build multiarch docker image

* CircleCI: build multiarch docker image #2

* CircleCI: build multiarch docker image #3

* CircleCI: build multiarch docker image #4

* CircleCI: build multiarch docker image #5

* CircleCI: build multiarch docker image #6

* CircleCI: build multiarch docker image #7

* CircleCI: build multiarch docker image #8

* CircleCI: build multiarch docker image #9

* CircleCI: build multiarch docker image #10

* CircleCI: build multiarch docker image #11

* CircleCI: build multiarch docker image #12

* CircleCI: build multiarch docker image #13

* CircleCI: build multiarch docker image #14

* CircleCI: build multiarch docker image #15

* CircleCI: build multiarch docker image #16

* CircleCI: build multiarch docker image #17

* CircleCI: build multiarch docker image #18

* CircleCI: build multiarch docker image #19

* CircleCI: build multiarch docker image #20

* CircleCI: build multiarch docker image #21

* CircleCI: build multiarch docker image #22

* CircleCI: build multiarch docker image #23

* CircleCI: build multiarch docker image #24

* CircleCI: build multiarch docker image #25

* CircleCI: build multiarch docker image #26

* CircleCI: build multiarch docker image #27

* CircleCI: build multiarch docker image #28

* Readies sync

.circleci/config.yml

@@ -54,12 +54,29 @@ jobs:
     steps:
       - run:
           name: Set up workspace
-          command: >-
-            sudo mkdir -p /workspace;
+          command: |
+            sudo mkdir -p /workspace
             sudo chown $USER /workspace
       - ci_steps:
           platform: macosx
-
+  build-multiarch-docker:
+    machine:
+      enabled: true
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Setup Docker client experimental features
+          command: |
+            sudo ./deps/readies/bin/getdocker --just-enable-exp
+            docker version
+      - run:
+          name: Build
+          command: |
+            sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+            make -C build/docker build
+            sudo make -C build/docker publish
+
   deploy_package:
     parameters:
       package:
@@ -112,6 +129,12 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - build-multiarch-docker:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
       - deploy_package:
           name: deploy_branch
           package: branch

build/docker/Makefile

@@ -87,7 +87,7 @@ publish:
 endif
 	$(call create_manifest,$(VERSION))
 	$(call create_manifest,latest)
-	
+
 show:
 	@echo "$(STEM)-$(OSNICK):"
 ifeq ($(INSPECT),1)

deps/readies/bin/getdocker

@@ -0,0 +1,61 @@
+#!/usr/bin/env python2
+
+import sys
+import os
+import argparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+import paella
+
+#----------------------------------------------------------------------------------------------
+
+class DockerSetup(paella.Setup):
+    def __init__(self, nop=False, no_install=False, no_exp=False):
+        self.no_install = no_install
+        self.no_exp = no_exp
+        paella.Setup.__init__(self, nop)
+
+    def debian_compat(self):
+        if self.no_install:
+            return
+        self.install("apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release")
+        self.run("curl -fsSL https://download.docker.com/linux/{}/gpg | apt-key add -".format(self.dist))
+        self.add_repo("'deb [arch=amd64] https://download.docker.com/linux/debian {} stable'".format(sh("lsb_release -cs")))
+        self.run("apt-get -qq update")
+        self.install("docker-ce docker-ce-cli containerd.io")
+
+    def redhat_compat(self):
+        if self.no_install:
+            return   
+        self.install("yum-utils device-mapper-persistent-data lvm2")
+        self.add_repo("https://download.docker.com/linux/centos/docker-ce.repo")
+        self.install("docker-ce docker-ce-cli containerd.io")
+
+    def fedora(self):
+        if self.no_install:
+            return
+        self.add_repo("https://download.docker.com/linux/fedora/docker-ce.repo")
+        self.install("docker-ce docker-ce-cli containerd.io")
+
+    def common_last(self):
+        if self.no_exp:
+            return
+        self.install("jq moreutils")
+        self.run("mkdir -p ~/.docker")
+        self.run("if [ ! -f ~/.docker/config.json ]; then echo '{}' >  ~/.docker/config.json; fi")
+        self.run("jq '.experimental = \"enabled\"' ~/.docker/config.json | sponge ~/.docker/config.json")
+        if self.has_command("systemctl"):
+            self.run("systemctl restart docker")
+        else:
+            self.run("service docker restart")
+        pass
+
+#----------------------------------------------------------------------------------------------
+
+parser = argparse.ArgumentParser(description='Install Docker CE')
+parser.add_argument('-n', '--nop', action="store_true", help='no operation')
+parser.add_argument('--just-enable-exp', action="store_true", help='no install, just enable experimental features')
+parser.add_argument('--no-exp', action="store_true", help="don't enable experimental features")
+args = parser.parse_args()
+
+DockerSetup(nop = args.nop, no_install=args.just_enable_exp, no_exp=args.no_exp).setup()

deps/readies/bin/getpy

@@ -1,21 +1,19 @@
 #!/bin/bash
 
-[ "$VERBOSE" = "1" ] && set -x
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+. $HERE/../shibumi/functions
 
-runn() {
-	[[ $NOP == 1 ]] && { echo "${@:1}"; return; }
-	__runn_log=$(mktemp /tmp/run.XXXXX)
-	{ "${@:1}"; } > $__runn_log  2>&1
-	[ $? != 0 ] && cat $__runn_log
-	rm -f $__runn_log
-}
+[ "$VERBOSE" = "1" ] && set -x
 
 if [ ! -z $(command -v python) ]; then
 	[ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "3" ] && exit 0
 fi
 
 [ ! -z $(command -v python3) ] && exit 0
 
+[[ $CHECK == 1 ]] && exit 1
+
 if [ ! -z $(command -v apt-get) ]; then
 	runn apt-get -qq update
 	runn apt-get -qq install -y python3

deps/readies/bin/getpy2

@@ -1,31 +1,32 @@
 #!/bin/bash
 
-[ "$VERBOSE" = "1" ] && set -x
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+. $HERE/../shibumi/functions
 
-show_if_error() {
-	{ "${@:1}"; } > /tmp/py2.log  2>&1
-	[ $? != 0 ] && cat /tmp/py2.log
-	rm -f /tmp/py2.log
-}
+[ "$VERBOSE" = "1" ] && set -x
 
 if [ ! -z $(command -v python) ]; then
 	[ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "2" ] && exit 0
 fi
 
 [ ! -z $(command -v python2) ] && exit 0
+[ ! -z $(command -v python2.7) ] && exit 0
+
+[[ $CHECK == 1 ]] && exit 1
 
 if [ ! -z $(command -v apt-get) ]; then
-	show_if_error apt-get -qq update
-	show_if_error apt-get -qq install -y python
+	runn apt-get -qq update
+	runn apt-get -qq install -y python
 elif [ ! -z $(command -v dnf) ]; then
-	show_if_error dnf install -y python2
+	runn dnf install -y python2
 elif [ ! -z $(command -v yum) ]; then
-	show_if_error yum install -y python2
+	runn yum install -y python2
 elif [ ! -z $(command -v apk) ]; then
-	show_if_error apk update
-	show_if_error apk add python2
+	runn apk update
+	runn apk add python2
 elif [ ! -z $(command -v brew) ]; then
-	show_if_error brew install python2
+	runn brew install python2
 fi
 
 if [ -z $(command -v python) ]; then

deps/readies/bin/getredis5

@@ -20,8 +20,7 @@ class Redis5Setup(paella.Setup):
 
     def debian_compat(self):
         # https://chilts.org/installing-redis-from-chris-leas-ppa/
-        self.install("software-properties-common")
-        self.run("add-apt-repository -y ppa:chris-lea/redis-server")
+        self.add_repo("ppa:chris-lea/redis-server")
         self.install("redis-server")
         # if not removed, might break apt-get update
         self.run("add-apt-repository -r -y ppa:chris-lea/redis-server")

deps/readies/mk/bindirs.defs

@@ -0,0 +1,9 @@
+
+BINROOT=$(ROOT)/bin/$(FULL_VARIANT)
+BINROOT.release=$(ROOT)/bin/$(FULL_VARIANT.release)
+BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS))) $(BINDIRS))
+
+define mkdir_rule
+$(1):
+	$$(SHOW)mkdir -p $(1)
+endef

deps/readies/mk/bindirs.rules

@@ -0,0 +1,10 @@
+
+ifeq ($(BINDIR),)
+$(error BINDIR is undefined)
+endif
+
+.PHONY: bindirs
+
+bindirs: $(BIN_DIRS)
+
+$(foreach DIR,$(BIN_DIRS),$(eval $(call mkdir_rule,$(DIR))))

deps/readies/mk/build.rules

@@ -0,0 +1,11 @@
+
+.PHONY: build clean
+
+build: $(MK_MAKEFILES) $(TARGET)
+
+#clean:
+#ifeq ($(ALL),1) 
+#	$(SHOW)rm -rf $(BINDIR)
+#else
+#	$(SHOW)$(MAKE) clean -C $(BUILD_DIR)
+#endif 

deps/readies/mk/cmake.defs

@@ -0,0 +1,14 @@
+
+ifneq ($(MK_CMAKE),)
+
+MK_MAKEFILES += $(BINDIR)/Makefile
+
+ifeq ($(DEBUG),1)
+CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
+endif
+
+ifeq ($(WHY),1)
+CMAKE_WHY=--trace-expand
+endif
+
+endif # MK_CMAKE

deps/readies/mk/cmake.rules

@@ -0,0 +1,12 @@
+
+ifneq ($(MK_CMAKE),)
+
+$(BINDIR)/Makefile : bindirs $(SRCDIR)/CMakeLists.txt
+	$(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi
+	$(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR))
+
+#	$(SHOW)cd $(BINDIR); \
+#	CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \
+#	cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT
+
+endif # MK_CMAKE

deps/readies/mk/common.defs

@@ -0,0 +1,17 @@
+
+_SHOW:=$(SHOW)
+ifeq ($(_SHOW),)
+_SHOW:=0
+endif
+ifeq ($(SHOW),1)
+override SHOW:=
+else
+override SHOW:=@
+endif
+
+MAKEFLAGS += --no-builtin-rules  --no-print-directory
+# --no-builtin-variables
+
+define __SEP
+import os; rows, cols = os.popen('stty size', 'r').read().split(); print('-' * (int(cols) - 1) + \"\n\")
+endef

deps/readies/mk/common.rules

@@ -0,0 +1,6 @@
+
+.PHONY: __sep
+
+__sep: ;
+#	@python -c "$(__SEP)"
+

deps/readies/mk/configure.defs

@@ -0,0 +1,2 @@
+
+CONFIGURE_FLAGS=

deps/readies/mk/configure.rules

@@ -0,0 +1,15 @@
+
+.PHONY: build clean
+
+build: $(MK_MAKEFILES) $(TARGET)
+ifeq (,$(wildcard $(BUILD_DIR)/Makefile))
+	$(SHOW)cd $(BUILD_DIR); $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS)
+endif
+	@make -C $(BUILD_DIR)
+
+clean:
+ifeq ($(ALL),1) 
+	$(SHOW)rm -rf $(BINDIR)
+else
+	$(SHOW)$(MAKE) clean -C $(BUILD_DIR)
+endif

deps/readies/mk/defs

@@ -0,0 +1,4 @@
+
+include $(MK)/help.defs
+include $(MK)/build.defs
+include $(MK)/cmake.defs

deps/readies/mk/functions

@@ -0,0 +1,5 @@
+
+reverse=$(if $(wordlist 2,2,$(1)),$(call reverse,$(wordlist 2,$(words $(1)),$(1))) $(firstword $(1)),$(1))
+
+__EMPTY:=
+__SPACE:=$(__EMPTY) $(__EMPTY)

deps/readies/mk/help.defs

@@ -0,0 +1,4 @@
+
+ifneq ($(HELP),) 
+HELPFILE:=$(shell mktemp /tmp/make.help.XXXX)
+endif

deps/readies/mk/help.rules

@@ -0,0 +1,13 @@
+
+ifneq ($(HELP),) 
+
+.PHONY: help
+
+help:
+	$(file >$(HELPFILE),$(HELP))
+	@echo
+	@cat $(HELPFILE)
+	@echo
+	@-rm -f $(HELPFILE)
+
+endif

deps/readies/mk/main

@@ -0,0 +1,29 @@
+
+SHELL=/bin/bash
+
+ifeq ($(ROOT),)
+$(error ROOT is undefined)
+endif
+
+ifneq ($(wildcard $(ROOT)/deps/readies),)
+READIES:=$(ROOT)/deps/readies
+else
+ifneq ($(wildcard $(ROOT)/opt/readies),)
+READIES:=$(ROOT)/opt/readies
+else
+$(error Cannot find readies root)
+endif
+endif
+
+MK:=$(READIES)/mk
+
+ifneq ($(shell { CHECK=1 $(READIES)/bin/getpy; echo $$?; }),0)
+$(error It seems prerequisites have not been installed: please run 'make setup'.)
+endif
+
+MK_ALL_TARGETS:=bindirs build
+
+include $(MK)/functions
+include $(MK)/common.defs
+include $(MK)/variant.defs
+include $(MK)/bindirs.defs

deps/readies/mk/rules

@@ -0,0 +1,11 @@
+
+.PHONY: all
+
+all: $(MK_ALL_TARGETS)
+
+include $(MK)/common.rules
+include $(MK)/variant.rules
+include $(MK)/bindirs.rules
+include $(MK)/help.rules
+include $(MK)/build.rules
+include $(MK)/cmake.rules