Build docker image using Taskcluster dind

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party = taskboot
-known_third_party = boto3,botocore,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml
+known_third_party = boto3,botocore,docker,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

.taskcluster.yml

@@ -91,6 +91,61 @@ tasks:
         owner: bastien@mozilla.com
         source: https://github.com/mozilla/task-boot
 
+    - taskId: {$eval: as_slugid("docker_build_dind")}
+      dependencies:
+        - {$eval: as_slugid("code_checks")}
+      provisionerId: proj-relman
+      workerType: ci
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        features:
+          dind: true
+        maxRunTime: 3600
+        image: python:3.7-alpine
+        env:
+          IMAGE: mozilla/taskboot
+          REGISTRY: registry.hub.docker.com
+          VERSION: "${tag}"
+        command:
+          - sh
+          - -lxce
+          - "apk add --no-cache git --quiet &&
+            git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot &&
+            pip install --no-cache-dir --quiet . &&
+            taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar tests/dockerfile.empty"
+        artifacts:
+          public/taskboot/test-dind.tar:
+            expires: {$fromNow: '2 weeks'}
+            path: /image.tar
+            type: file
+      metadata:
+        name: TaskBoot docker build using Docker in Docker
+        description: Taskcluster boot utilities - build a test image
+        owner: bastien@mozilla.com
+        source: https://github.com/mozilla/task-boot
+
+    - taskId: {$eval: as_slugid("docker_run_dind")}
+      dependencies:
+        - {$eval: as_slugid("docker_build_dind")}
+      provisionerId: proj-relman
+      workerType: ci
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        features:
+          dind: true
+        maxRunTime: 3600
+        image:
+          type: task-image
+          path: public/taskboot/test-dind.tar
+          taskId: {$eval: as_slugid("docker_build_dind")}
+      metadata:
+        name: TaskBoot docker run DinD image
+        description: Taskcluster boot utilities - run a test image
+        owner: bastien@mozilla.com
+        source: https://github.com/mozilla/task-boot
+
     - $if: 'tasks_for == "github-push" && (head_branch == "refs/heads/master" || head_branch[:10] == "refs/tags/")'
       then:
         taskId: {$eval: as_slugid("docker_push")}

requirements.txt

@@ -1,5 +1,6 @@
 boto3>=1.9
 dockerfile-parse==0.0.15
+https://github.com/docker/docker-py/archive/1.10.6.tar.gz#egg=docker-py
 multidict<4.6.0
 PyGithub==1.45
 pyyaml==5.3

setup.py

@@ -7,8 +7,13 @@
 
 
 def requirements(path):
+    lines = []
     with open(path) as f:
-        return f.read().splitlines()
+        for line in f.read().splitlines():
+            if line.startswith("https://"):
+                line = line.split("#")[1].split("egg=")[1]
+            lines.append(line)
+    return sorted(lines)
 
 
 setup(

taskboot/build.py

@@ -13,6 +13,7 @@
 import yaml
 
 from taskboot.config import Configuration
+from taskboot.docker import DinD
 from taskboot.docker import Docker
 from taskboot.docker import Img
 from taskboot.docker import patch_dockerfile
@@ -53,6 +54,8 @@ def build_image(target, args):
         build_tool = Img(cache=args.cache)
     elif args.build_tool == "docker":
         build_tool = Docker()
+    elif args.build_tool == "dind":
+        build_tool = DinD()
     else:
         raise ValueError("Unsupported build tool: {}".format(args.build_tool))
 

taskboot/cli.py

@@ -94,7 +94,7 @@ def main():
     build.add_argument(
         "--build-tool",
         dest="build_tool",
-        choices=["img", "docker"],
+        choices=["img", "docker", "dind"],
         default=os.environ.get("BUILD_TOOL") or "img",
         help="Tool to build docker images.",
     )

taskboot/docker.py

@@ -15,6 +15,7 @@
 import tarfile
 import tempfile
 
+import docker as really_old_docker
 from dockerfile_parse import DockerfileParser
 
 logger = logging.getLogger(__name__)
@@ -26,6 +27,10 @@
 
 IMG_NAME_REGEX = re.compile(r"(?P<name>[\/\w\-\._]+):?(?P<tag>\S*)")
 
+# Taskcluster uses a really outdated version of Docker daemon API
+# so we need to use a *really* outdated client too
+TASKCLUSTER_DIND_API_VERSION = "1.18"
+
 
 def read_archive_tags(path):
     tar = tarfile.open(path)
@@ -281,6 +286,88 @@ def push(self, tag):
         self.run(["push", "--state", self.state, tag])
 
 
+class DinD(Tool):
+    """
+    Interface to the Docker In Docker Taskcluster feature
+    """
+
+    def __init__(self, cache=None):
+        # Check version of remote daemon
+        self.client = really_old_docker.from_env(version=TASKCLUSTER_DIND_API_VERSION)
+        version = self.client.version()
+        assert (
+            version["ApiVersion"] == TASKCLUSTER_DIND_API_VERSION
+        ), f"DinD version mismatch: {version}"
+
+    def list_images(self):
+        """
+        List images stored on remote daemon
+        """
+
+        def _list_images():
+            for image in self.client.images(all=True):
+                for repo_tag in image["RepoTags"]:
+                    repo, tag = parse_image_name(repo_tag)
+                    image.update({"tag": tag, "repository": repo})
+                    yield image
+
+        return [
+            {
+                "repository": image["repository"],
+                "tag": image["tag"],
+                "size": image["VirtualSize"],
+                "created": image["Created"],
+                "digest": image["Id"],
+            }
+            for image in _list_images()
+        ]
+
+    def build(self, context_dir, dockerfile, tags, build_args=[]):
+        logger.info(f"Building docker image with DinD {dockerfile}")
+        build_output = self.client.build(
+            path=context_dir, dockerfile=dockerfile, buildargs=build_args, tag=tags
+        )
+
+        # The build is not processed if the generator is not used
+        for line in build_output:
+            try:
+                state = json.loads(line)
+                if "stream" in state:
+                    out = state["stream"].rstrip()
+                elif "status" in state:
+                    if "id" in state:
+                        out = f"[{state['id']}] {state['status']}"
+                    else:
+                        out = state["status"]
+                    progress = state.get("progressDetail")
+                    if progress and "current" in progress and "total" in progress:
+                        percent = round(100.0 * progress["current"] / progress["total"])
+                        out += f" {percent}%"
+                logger.info(f"DinD build: {out}")
+            except (KeyError, json.decoder.JSONDecodeError):
+                logger.info(f"DinD build: {line}")
+
+        logger.info("Built image {}".format(", ".join(tags)))
+
+    def save(self, tags, path):
+        assert isinstance(tags, list)
+        assert len(tags) > 0, "Missing tags to save"
+
+        # save the image using only one tag
+        main_tag = tags[0]
+        logger.info("Saving image {} to {}".format(main_tag, path))
+
+        image = self.client.get_image(main_tag)
+        with open(path, "wb") as dest:
+            dest.write(image.data)
+
+    def login(self, *args, **kwargs):
+        raise NotImplementedError("Cannot login using dind")
+
+    def push(self, *args, **kwargs):
+        raise NotImplementedError("Cannot push using dind")
+
+
 class Skopeo(Tool):
     """
     Interface to the skopeo tool, used to copy local images to remote repositories

tests/dockerfile.empty

@@ -1,6 +1,6 @@
 FROM alpine
 
 RUN mkdir -p /etc
-RUN echo 'Hello from taskboot' /etc/boot
+RUN echo 'Hello from taskboot' > /etc/boot
 
-CMD ['cat', '/etc/boot']
+CMD cat /etc/boot