diff --git a/.isort.cfg b/.isort.cfg index 79e07c6..4a3218d 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = taskboot -known_third_party = boto3,botocore,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml +known_third_party = boto3,botocore,docker,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/.taskcluster.yml b/.taskcluster.yml index 371eb2d..e3a9097 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -91,6 +91,61 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/task-boot + - taskId: {$eval: as_slugid("docker_build_dind")} + dependencies: + - {$eval: as_slugid("code_checks")} + provisionerId: proj-relman + workerType: ci + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + features: + dind: true + maxRunTime: 3600 + image: python:3.7-alpine + env: + IMAGE: mozilla/taskboot + REGISTRY: registry.hub.docker.com + VERSION: "${tag}" + command: + - sh + - -lxce + - "apk add --no-cache git --quiet && + git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot && + pip install --no-cache-dir --quiet . && + taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar tests/dockerfile.empty" + artifacts: + public/taskboot/test-dind.tar: + expires: {$fromNow: '2 weeks'} + path: /image.tar + type: file + metadata: + name: TaskBoot docker build using Docker in Docker + description: Taskcluster boot utilities - build a test image + owner: bastien@mozilla.com + source: https://github.com/mozilla/task-boot + + - taskId: {$eval: as_slugid("docker_run_dind")} + dependencies: + - {$eval: as_slugid("docker_build_dind")} + provisionerId: proj-relman + workerType: ci + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + features: + dind: true + maxRunTime: 3600 + image: + type: task-image + path: public/taskboot/test-dind.tar + taskId: {$eval: as_slugid("docker_build_dind")} + metadata: + name: TaskBoot docker run DinD image + description: Taskcluster boot utilities - run a test image + owner: bastien@mozilla.com + source: https://github.com/mozilla/task-boot + - $if: 'tasks_for == "github-push" && (head_branch == "refs/heads/master" || head_branch[:10] == "refs/tags/")' then: taskId: {$eval: as_slugid("docker_push")} diff --git a/requirements.txt b/requirements.txt index 24b9ee1..6ba62c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ boto3>=1.9 dockerfile-parse==0.0.15 +https://github.com/docker/docker-py/archive/1.10.6.tar.gz#egg=docker-py multidict<4.6.0 PyGithub==1.45 pyyaml==5.3 diff --git a/setup.py b/setup.py index ea2d5ae..0dce1cc 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,13 @@ def requirements(path): + lines = [] with open(path) as f: - return f.read().splitlines() + for line in f.read().splitlines(): + if line.startswith("https://"): + line = line.split("#")[1].split("egg=")[1] + lines.append(line) + return sorted(lines) setup( diff --git a/taskboot/build.py b/taskboot/build.py index d34b973..02d1ae2 100644 --- a/taskboot/build.py +++ b/taskboot/build.py @@ -13,6 +13,7 @@ import yaml from taskboot.config import Configuration +from taskboot.docker import DinD from taskboot.docker import Docker from taskboot.docker import Img from taskboot.docker import patch_dockerfile @@ -53,6 +54,8 @@ def build_image(target, args): build_tool = Img(cache=args.cache) elif args.build_tool == "docker": build_tool = Docker() + elif args.build_tool == "dind": + build_tool = DinD() else: raise ValueError("Unsupported build tool: {}".format(args.build_tool)) diff --git a/taskboot/cli.py b/taskboot/cli.py index b4047f9..fa6be3d 100644 --- a/taskboot/cli.py +++ b/taskboot/cli.py @@ -94,7 +94,7 @@ def main(): build.add_argument( "--build-tool", dest="build_tool", - choices=["img", "docker"], + choices=["img", "docker", "dind"], default=os.environ.get("BUILD_TOOL") or "img", help="Tool to build docker images.", ) diff --git a/taskboot/docker.py b/taskboot/docker.py index c4cfb2a..847ac8d 100644 --- a/taskboot/docker.py +++ b/taskboot/docker.py @@ -15,6 +15,7 @@ import tarfile import tempfile +import docker as really_old_docker from dockerfile_parse import DockerfileParser logger = logging.getLogger(__name__) @@ -26,6 +27,10 @@ IMG_NAME_REGEX = re.compile(r"(?P[\/\w\-\._]+):?(?P\S*)") +# Taskcluster uses a really outdated version of Docker daemon API +# so we need to use a *really* outdated client too +TASKCLUSTER_DIND_API_VERSION = "1.18" + def read_archive_tags(path): tar = tarfile.open(path) @@ -281,6 +286,88 @@ def push(self, tag): self.run(["push", "--state", self.state, tag]) +class DinD(Tool): + """ + Interface to the Docker In Docker Taskcluster feature + """ + + def __init__(self, cache=None): + # Check version of remote daemon + self.client = really_old_docker.from_env(version=TASKCLUSTER_DIND_API_VERSION) + version = self.client.version() + assert ( + version["ApiVersion"] == TASKCLUSTER_DIND_API_VERSION + ), f"DinD version mismatch: {version}" + + def list_images(self): + """ + List images stored on remote daemon + """ + + def _list_images(): + for image in self.client.images(all=True): + for repo_tag in image["RepoTags"]: + repo, tag = parse_image_name(repo_tag) + image.update({"tag": tag, "repository": repo}) + yield image + + return [ + { + "repository": image["repository"], + "tag": image["tag"], + "size": image["VirtualSize"], + "created": image["Created"], + "digest": image["Id"], + } + for image in _list_images() + ] + + def build(self, context_dir, dockerfile, tags, build_args=[]): + logger.info(f"Building docker image with DinD {dockerfile}") + build_output = self.client.build( + path=context_dir, dockerfile=dockerfile, buildargs=build_args, tag=tags + ) + + # The build is not processed if the generator is not used + for line in build_output: + try: + state = json.loads(line) + if "stream" in state: + out = state["stream"].rstrip() + elif "status" in state: + if "id" in state: + out = f"[{state['id']}] {state['status']}" + else: + out = state["status"] + progress = state.get("progressDetail") + if progress and "current" in progress and "total" in progress: + percent = round(100.0 * progress["current"] / progress["total"]) + out += f" {percent}%" + logger.info(f"DinD build: {out}") + except (KeyError, json.decoder.JSONDecodeError): + logger.info(f"DinD build: {line}") + + logger.info("Built image {}".format(", ".join(tags))) + + def save(self, tags, path): + assert isinstance(tags, list) + assert len(tags) > 0, "Missing tags to save" + + # save the image using only one tag + main_tag = tags[0] + logger.info("Saving image {} to {}".format(main_tag, path)) + + image = self.client.get_image(main_tag) + with open(path, "wb") as dest: + dest.write(image.data) + + def login(self, *args, **kwargs): + raise NotImplementedError("Cannot login using dind") + + def push(self, *args, **kwargs): + raise NotImplementedError("Cannot push using dind") + + class Skopeo(Tool): """ Interface to the skopeo tool, used to copy local images to remote repositories diff --git a/tests/dockerfile.empty b/tests/dockerfile.empty index fdb3f90..d707bde 100644 --- a/tests/dockerfile.empty +++ b/tests/dockerfile.empty @@ -1,6 +1,6 @@ FROM alpine RUN mkdir -p /etc -RUN echo 'Hello from taskboot' /etc/boot +RUN echo 'Hello from taskboot' > /etc/boot -CMD ['cat', '/etc/boot'] +CMD cat /etc/boot