From 387937237e534e7dc25e9c5c6dd05423f43b21e2 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 11:41:56 +0100 Subject: [PATCH 01/14] WIP build tool --- .isort.cfg | 2 +- .taskcluster.yml | 34 +++++++++++++++++++++++++ requirements.txt | 1 + taskboot/build.py | 3 +++ taskboot/cli.py | 2 +- taskboot/docker.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 79e07c6..4a3218d 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = taskboot -known_third_party = boto3,botocore,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml +known_third_party = boto3,botocore,docker,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/.taskcluster.yml b/.taskcluster.yml index 371eb2d..bb4910d 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -91,6 +91,40 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/task-boot + - taskId: {$eval: as_slugid("docker_build_dind")} + #dependencies: + # - {$eval: as_slugid("code_checks")} + provisionerId: proj-relman + workerType: ci + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + features: + dind: true + maxRunTime: 3600 + image: python:3.7-alpine + env: + IMAGE: mozilla/taskboot + REGISTRY: registry.hub.docker.com + VERSION: "${tag}" + command: + - sh + - -lxce + - "apk add --no-cache git --quiet && + git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && + pip install --no-cache-dir --quiet . && + taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile" + artifacts: + public/taskboot/image.tar: + expires: {$fromNow: '2 weeks'} + path: /image.tar + type: file + metadata: + name: TaskBoot docker build using Docker in Docker + description: Taskcluster boot utilities - build latest docker image + owner: bastien@mozilla.com + source: https://github.com/mozilla/task-boot + - $if: 'tasks_for == "github-push" && (head_branch == "refs/heads/master" || head_branch[:10] == "refs/tags/")' then: taskId: {$eval: as_slugid("docker_push")} diff --git a/requirements.txt b/requirements.txt index 24b9ee1..0295179 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ boto3>=1.9 dockerfile-parse==0.0.15 +https://github.com/docker/docker-py/archive/1.10.6.tar.gz multidict<4.6.0 PyGithub==1.45 pyyaml==5.3 diff --git a/taskboot/build.py b/taskboot/build.py index d34b973..02d1ae2 100644 --- a/taskboot/build.py +++ b/taskboot/build.py @@ -13,6 +13,7 @@ import yaml from taskboot.config import Configuration +from taskboot.docker import DinD from taskboot.docker import Docker from taskboot.docker import Img from taskboot.docker import patch_dockerfile @@ -53,6 +54,8 @@ def build_image(target, args): build_tool = Img(cache=args.cache) elif args.build_tool == "docker": build_tool = Docker() + elif args.build_tool == "dind": + build_tool = DinD() else: raise ValueError("Unsupported build tool: {}".format(args.build_tool)) diff --git a/taskboot/cli.py b/taskboot/cli.py index b4047f9..fa6be3d 100644 --- a/taskboot/cli.py +++ b/taskboot/cli.py @@ -94,7 +94,7 @@ def main(): build.add_argument( "--build-tool", dest="build_tool", - choices=["img", "docker"], + choices=["img", "docker", "dind"], default=os.environ.get("BUILD_TOOL") or "img", help="Tool to build docker images.", ) diff --git a/taskboot/docker.py b/taskboot/docker.py index c4cfb2a..390a3b6 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -15,6 +15,7 @@ import tarfile import tempfile +import docker as really_old_docker from dockerfile_parse import DockerfileParser logger = logging.getLogger(__name__) @@ -26,6 +27,10 @@ IMG_NAME_REGEX = re.compile(r"(?P[\/\w\-\._]+):?(?P\S*)") +# Taskcluster uses a really outdated version of Docker daemon API +# so we need to use a *really* outdated client too +TASKCLUSTER_DIND_API_VERSION = "1.18" + def read_archive_tags(path): tar = tarfile.open(path) @@ -281,6 +286,63 @@ def push(self, tag): self.run(["push", "--state", self.state, tag]) +class DinD(Tool): + """ + Interface to the Docker In Docker Taskcluster feature + """ + + def __init__(self, cache=None): + # Check version of remote daemon + self.client = really_old_docker.from_env(version=TASKCLUSTER_DIND_API_VERSION) + version = self.client.version() + assert ( + version["ApiVersion"] == TASKCLUSTER_DIND_API_VERSION + ), f"DinD version mismatch: {version}" + + def list_images(self): + """ + List images stored on remote daemon + """ + return [ + { + "registry": image.group(1), + "repository": image.group(2), + "tag": image.group(5), + "size": image.group(6), + "created": image.group(7), + "updated": image.group(8), + "digest": image.group(9), + } + for image in self.client.images(all=True) + ] + + def build(self, context_dir, dockerfile, tags, build_args=[]): + logger.info("Building docker image with DinD {}".format(dockerfile)) + out = self.client.build( + path=context_dir, dockerfile=dockerfile, buildargs=build_args, tag=tags + ) + print("BUILD OUTPUT", out) + logger.info("Built image {}".format(", ".join(tags))) + + def save(self, tags, path): + assert isinstance(tags, list) + assert len(tags) > 0, "Missing tags to save" + + # First save the image using only one tag + # dind does not support (yet) writing multiple tags + main_tag = tags[0] + logger.info("Saving image {} to {}".format(main_tag, path)) + + with open(path, "wb") as dest: + dest.write(self.client.get_image(main_tag)) + + def login(self, *args, **kwargs): + raise NotImplementedError("Cannot login using dind") + + def push(self, *args, **kwargs): + raise NotImplementedError("Cannot push using dind") + + class Skopeo(Tool): """ Interface to the skopeo tool, used to copy local images to remote repositories From 721a4a2772b0dfa0586b246b237f07d2fb8b7caf Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 11:52:22 +0100 Subject: [PATCH 02/14] Support https:// reqs --- requirements.txt | 2 +- setup.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0295179..6ba62c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ boto3>=1.9 dockerfile-parse==0.0.15 -https://github.com/docker/docker-py/archive/1.10.6.tar.gz +https://github.com/docker/docker-py/archive/1.10.6.tar.gz#egg=docker-py multidict<4.6.0 PyGithub==1.45 pyyaml==5.3 diff --git a/setup.py b/setup.py index ea2d5ae..0dce1cc 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,13 @@ def requirements(path): + lines = [] with open(path) as f: - return f.read().splitlines() + for line in f.read().splitlines(): + if line.startswith("https://"): + line = line.split("#")[1].split("egg=")[1] + lines.append(line) + return sorted(lines) setup( From b9def1d5f8b85e01257e3a336772e982f5d5a9e5 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 11:58:16 +0100 Subject: [PATCH 03/14] Read build output --- taskboot/docker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index 390a3b6..66a5012 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -318,10 +318,17 @@ def list_images(self): def build(self, context_dir, dockerfile, tags, build_args=[]): logger.info("Building docker image with DinD {}".format(dockerfile)) - out = self.client.build( + build_output = self.client.build( path=context_dir, dockerfile=dockerfile, buildargs=build_args, tag=tags ) - print("BUILD OUTPUT", out) + + # The build is not processed if the generator is not used + for line in build_output: + try: + stream = json.loads(line)["stream"].rstrip() + logger.info(f"DinD build: {stream}") + except (KeyError, json.decoder.JSONDecodeError): + logger.info("DinD build", line=line) logger.info("Built image {}".format(", ".join(tags))) def save(self, tags, path): From 3b5a5c94331b5497d9a260f4d6f186f1e2a506d2 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:01:10 +0100 Subject: [PATCH 04/14] Fix inner exception --- taskboot/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index 66a5012..d3b3ee5 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -328,7 +328,7 @@ def build(self, context_dir, dockerfile, tags, build_args=[]): stream = json.loads(line)["stream"].rstrip() logger.info(f"DinD build: {stream}") except (KeyError, json.decoder.JSONDecodeError): - logger.info("DinD build", line=line) + logger.info(f"DinD build: {line}") logger.info("Built image {}".format(", ".join(tags))) def save(self, tags, path): From 788fd148bab5587e0b18cb1f0cc993677145e642 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:11:38 +0100 Subject: [PATCH 05/14] Support progress --- taskboot/docker.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index d3b3ee5..74c6644 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -325,10 +325,19 @@ def build(self, context_dir, dockerfile, tags, build_args=[]): # The build is not processed if the generator is not used for line in build_output: try: - stream = json.loads(line)["stream"].rstrip() - logger.info(f"DinD build: {stream}") + state = json.loads(line) + if "stream" in state: + out = state["stream"].rstrip() + elif "status" in state: + out = f"[{state['id']}] {state['status']}" + progress = state.get("progressDetail") + if progress and "current" in progress and "total" in progress: + percent = round(100.0 * progress["current"] / progress["total"]) + out += f"{percent}%" + logger.info(f"DinD build: {out}") except (KeyError, json.decoder.JSONDecodeError): logger.info(f"DinD build: {line}") + logger.info("Built image {}".format(", ".join(tags))) def save(self, tags, path): From d35d209f8159c472c9e61898ba8a03874bc37ff1 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:11:49 +0100 Subject: [PATCH 06/14] Old school dockerfile --- .taskcluster.yml | 2 +- Dockerfile.dind | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.dind diff --git a/.taskcluster.yml b/.taskcluster.yml index bb4910d..503f7f1 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -77,7 +77,7 @@ tasks: - "apk add --no-cache git img --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing --quiet && git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && pip install --no-cache-dir --quiet . && - taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile" + taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile.dind" artifacts: public/taskboot/image.tar: expires: {$fromNow: '2 weeks'} diff --git a/Dockerfile.dind b/Dockerfile.dind new file mode 100644 index 0000000..0267aed --- /dev/null +++ b/Dockerfile.dind @@ -0,0 +1,13 @@ +FROM python:3.7-alpine + +# Add img +RUN apk add --no-cache img --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing + +# Setup other deps +RUN apk add --no-cache git skopeo docker + +# Install taskboot from copied source code +COPY . /src/taskboot +pip cd /src/taskboot && install --no-cache-dir . + +CMD ["taskboot", "--help"] From df220c03858d13cd11bbc686b2fb558533a9b20d Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:16:44 +0100 Subject: [PATCH 07/14] Fixes --- .taskcluster.yml | 4 ++-- taskboot/docker.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 503f7f1..b03cd41 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -77,7 +77,7 @@ tasks: - "apk add --no-cache git img --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing --quiet && git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && pip install --no-cache-dir --quiet . && - taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile.dind" + taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile" artifacts: public/taskboot/image.tar: expires: {$fromNow: '2 weeks'} @@ -113,7 +113,7 @@ tasks: - "apk add --no-cache git --quiet && git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && pip install --no-cache-dir --quiet . && - taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile" + taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile.dind" artifacts: public/taskboot/image.tar: expires: {$fromNow: '2 weeks'} diff --git a/taskboot/docker.py b/taskboot/docker.py index 74c6644..62a1739 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -329,11 +329,14 @@ def build(self, context_dir, dockerfile, tags, build_args=[]): if "stream" in state: out = state["stream"].rstrip() elif "status" in state: - out = f"[{state['id']}] {state['status']}" + if "id" in state: + out = f"[{state['id']}] {state['status']}" + else: + out = state["status"] progress = state.get("progressDetail") if progress and "current" in progress and "total" in progress: percent = round(100.0 * progress["current"] / progress["total"]) - out += f"{percent}%" + out += f" {percent}%" logger.info(f"DinD build: {out}") except (KeyError, json.decoder.JSONDecodeError): logger.info(f"DinD build: {line}") From a012f21575286591469e15eaa3942848290861b6 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:18:21 +0100 Subject: [PATCH 08/14] fix dind dockerfile --- Dockerfile.dind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dind b/Dockerfile.dind index 0267aed..6db2161 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -8,6 +8,6 @@ RUN apk add --no-cache git skopeo docker # Install taskboot from copied source code COPY . /src/taskboot -pip cd /src/taskboot && install --no-cache-dir . +RUN pip cd /src/taskboot && install --no-cache-dir . CMD ["taskboot", "--help"] From 4b0ffd6ed09938d84898de3f37215289ab0304b8 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:19:55 +0100 Subject: [PATCH 09/14] woops --- Dockerfile.dind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dind b/Dockerfile.dind index 6db2161..5ce15a4 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -8,6 +8,6 @@ RUN apk add --no-cache git skopeo docker # Install taskboot from copied source code COPY . /src/taskboot -RUN pip cd /src/taskboot && install --no-cache-dir . +RUN cd /src/taskboot && pip install --no-cache-dir . CMD ["taskboot", "--help"] From d2d6b88afa02ade6e7df5f32f62807bf46330a4c Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:23:23 +0100 Subject: [PATCH 10/14] Fix writing --- taskboot/docker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index 62a1739..f06195a 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -347,13 +347,13 @@ def save(self, tags, path): assert isinstance(tags, list) assert len(tags) > 0, "Missing tags to save" - # First save the image using only one tag - # dind does not support (yet) writing multiple tags + # save the image using only one tag main_tag = tags[0] logger.info("Saving image {} to {}".format(main_tag, path)) + image = self.client.get_image(main_tag) with open(path, "wb") as dest: - dest.write(self.client.get_image(main_tag)) + dest.write(image.data) def login(self, *args, **kwargs): raise NotImplementedError("Cannot login using dind") From 067dad27d00009baedd1c2337080b668ead85164 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:38:19 +0100 Subject: [PATCH 11/14] Build demo image & run it --- .taskcluster.yml | 31 ++++++++++++++++++++++++++----- Dockerfile.dind | 13 ------------- taskboot/docker.py | 1 + 3 files changed, 27 insertions(+), 18 deletions(-) delete mode 100644 Dockerfile.dind diff --git a/.taskcluster.yml b/.taskcluster.yml index b03cd41..e3a9097 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -92,8 +92,8 @@ tasks: source: https://github.com/mozilla/task-boot - taskId: {$eval: as_slugid("docker_build_dind")} - #dependencies: - # - {$eval: as_slugid("code_checks")} + dependencies: + - {$eval: as_slugid("code_checks")} provisionerId: proj-relman workerType: ci created: {$fromNow: ''} @@ -113,15 +113,36 @@ tasks: - "apk add --no-cache git --quiet && git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && pip install --no-cache-dir --quiet . && - taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile.dind" + taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar tests/dockerfile.empty" artifacts: - public/taskboot/image.tar: + public/taskboot/test-dind.tar: expires: {$fromNow: '2 weeks'} path: /image.tar type: file metadata: name: TaskBoot docker build using Docker in Docker - description: Taskcluster boot utilities - build latest docker image + description: Taskcluster boot utilities - build a test image + owner: bastien@mozilla.com + source: https://github.com/mozilla/task-boot + + - taskId: {$eval: as_slugid("docker_run_dind")} + dependencies: + - {$eval: as_slugid("docker_build_dind")} + provisionerId: proj-relman + workerType: ci + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + features: + dind: true + maxRunTime: 3600 + image: + type: task-image + path: public/taskboot/test-dind.tar + taskId: {$eval: as_slugid("docker_build_dind")} + metadata: + name: TaskBoot docker run DinD image + description: Taskcluster boot utilities - run a test image owner: bastien@mozilla.com source: https://github.com/mozilla/task-boot diff --git a/Dockerfile.dind b/Dockerfile.dind deleted file mode 100644 index 5ce15a4..0000000 --- a/Dockerfile.dind +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.7-alpine - -# Add img -RUN apk add --no-cache img --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing - -# Setup other deps -RUN apk add --no-cache git skopeo docker - -# Install taskboot from copied source code -COPY . /src/taskboot -RUN cd /src/taskboot && pip install --no-cache-dir . - -CMD ["taskboot", "--help"] diff --git a/taskboot/docker.py b/taskboot/docker.py index f06195a..ce0b1a4 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -303,6 +303,7 @@ def list_images(self): """ List images stored on remote daemon """ + # TODO return [ { "registry": image.group(1), From 6ad4aea3fdc77ca3f9e9850637d3f67932996f56 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 12:43:49 +0100 Subject: [PATCH 12/14] Fix dockerfile.empty --- tests/dockerfile.empty | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dockerfile.empty b/tests/dockerfile.empty index fdb3f90..d707bde 100644 --- a/tests/dockerfile.empty +++ b/tests/dockerfile.empty @@ -1,6 +1,6 @@ FROM alpine RUN mkdir -p /etc -RUN echo 'Hello from taskboot' /etc/boot +RUN echo 'Hello from taskboot' > /etc/boot -CMD ['cat', '/etc/boot'] +CMD cat /etc/boot From c8dd1e5179ae08aa787760ee161130adc76437a0 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 14:35:00 +0100 Subject: [PATCH 13/14] Support listing images --- taskboot/docker.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index ce0b1a4..e8ae1fe 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -303,18 +303,23 @@ def list_images(self): """ List images stored on remote daemon """ - # TODO + + def _list_images(): + for image in self.client.images(all=True): + for repo_tag in image["RepoTags"]: + repo, tag = parse_image_name(repo_tag) + image.update({"tag": tag, "repository": repo}) + yield image + return [ { - "registry": image.group(1), - "repository": image.group(2), - "tag": image.group(5), - "size": image.group(6), - "created": image.group(7), - "updated": image.group(8), - "digest": image.group(9), + "repository": image["repository"], + "tag": image["tag"], + "size": image["VirtualSize"], + "created": image["Created"], + "digest": image["Id"], } - for image in self.client.images(all=True) + for image in _list_images() ] def build(self, context_dir, dockerfile, tags, build_args=[]): From abb03cd892d3094baf69b174637a3f808c8b8943 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 22 Jan 2020 16:04:04 +0100 Subject: [PATCH 14/14] Fix nit --- taskboot/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskboot/docker.py b/taskboot/docker.py index e8ae1fe..847ac8d 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -323,7 +323,7 @@ def _list_images(): ] def build(self, context_dir, dockerfile, tags, build_args=[]): - logger.info("Building docker image with DinD {}".format(dockerfile)) + logger.info(f"Building docker image with DinD {dockerfile}") build_output = self.client.build( path=context_dir, dockerfile=dockerfile, buildargs=build_args, tag=tags )