From 3aad2e7a686429f40856e074712a1dd4d3fc9d0e Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 21 Oct 2019 15:20:59 -0400 Subject: [PATCH 1/8] Reduce docker image size & introduce per-python prefect images --- .circleci/config.yml | 86 +++++++++++++++---- Dockerfile | 22 +++-- src/prefect/environments/storage/docker.py | 61 +++++++++---- .../storage/test_docker_storage.py | 42 +++++++-- 4 files changed, 162 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e9eff3b65b3..f40341f04590 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 references: workspace_root: &workspace_root /tmp/workspace @@ -6,6 +6,7 @@ references: attach_workspace: at: *workspace_root + jobs: # ---------------------------------- # Check formatting @@ -147,7 +148,7 @@ jobs: # Run unit tests in Python 3.5-3.7 # ---------------------------------- - test-3.5: + test-35: docker: - image: python:3.5 @@ -176,7 +177,7 @@ jobs: paths: - coverage - test-3.6: + test-36: docker: - image: python:3.6 steps: @@ -204,7 +205,7 @@ jobs: paths: - coverage - test-3.7: + test-37: docker: - image: python:3.7 steps: @@ -266,41 +267,74 @@ jobs: name: Upload Coverage command: bash <(curl -s https://codecov.io/bash) -cF python -s "/tmp/workspace/coverage/" - build_image: + build-docker-image: docker: - image: docker + parameters: + python-version: + type: string + tag-latest: + type: boolean + default: false + environment: + PYTHON_VERSION: << parameters.python-version >> + PYTHON_TAG: python<< parameters.python-version >> steps: - - setup_remote_docker - checkout - - run: - name: Docker Build - command: docker build -t prefecthq/prefect . + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build image + command: >- + docker build + --build-arg GIT_SHA=$CIRCLE_SHA1 + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + --build-arg PREFECT_VERSION=$CIRCLE_TAG + -t prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} + -t prefecthq/prefect:$PYTHON_TAG + . + - when: + condition: << parameters.tag-latest >> + steps: + - run: + name: Tag latest image + command: | + docker tag prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} prefecthq/prefect:latest - run: name: Test image command: | docker run -dit prefecthq/prefect /bin/bash -c 'curl -fL0 https://raw.githubusercontent.com/PrefectHQ/prefect/master/examples/retries_with_mapping.py | python' - run: - name: Authenticate with Docker Hub and push + name: Push versioned tags command: | docker login --username $DOCKER_HUB_USER --password $DOCKER_HUB_PW - docker push prefecthq/prefect + docker push prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} + docker push prefecthq/prefect:$PYTHON_TAG + - when: + condition: << parameters.tag-latest >> + steps: + - run: + name: Push latest tag + command: | + docker login --username $DOCKER_HUB_USER --password $DOCKER_HUB_PW + docker push prefecthq/prefect:latest workflows: version: 2 "Run tests": jobs: - - test-3.5 - - test-3.6 - - test-3.7 + - test-35 + - test-36 + - test-37 - test-lower-prefect - test-vanilla-prefect - test-py352-import-prefect - test-airflow - upload-coverage: requires: - - test-3.5 - - test-3.6 + - test-35 + - test-36 - test-vanilla-prefect - test-airflow @@ -312,7 +346,25 @@ workflows: "Build docker images": jobs: - - build_image: + - build-docker-image: + python-version: "3.5" + filters: + branches: + only: master + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ + - build-docker-image: + python-version: "3.6" + filters: + branches: + only: master + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ + - build-docker-image: + python-version: "3.7" + tag-latest: true filters: branches: only: master + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 72437519daa2..0261a66980d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,20 @@ -ARG PYTHON_VERSION=3.6 +ARG PYTHON_VERSION=${PYTHON_VERSION:-3.6} +FROM python:${PYTHON_VERSION}-slim -FROM python:${PYTHON_VERSION} -LABEL maintainer="help@prefect.io" -ARG GIT_POINTER=master +RUN apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/* -RUN pip install git+https://github.com/PrefectHQ/prefect.git@${GIT_POINTER}#egg=prefect[kubernetes] +ARG PREFECT_VERSION +RUN pip install git+https://github.com/PrefectHQ/prefect.git@${PREFECT_VERSION}#egg=prefect[kubernetes] RUN mkdir /root/.prefect/ + +ARG GIT_SHA +ARG BUILD_DATE + +LABEL maintainer="help@prefect.io" +LABEL io.prefect.python-version=${PYTHON_VERSION} +LABEL org.label-schema.schema-version = "1.0" +LABEL org.label-schema.name="prefect" +LABEL org.label-schema.url="https://www.prefect.io/" +LABEL org.label-schema.version=${PREFECT_VERSION} +LABEL org.label-schema.vcs-ref=${GIT_SHA} +LABEL org.label-schema.build-date=${BUILD_DATE} \ No newline at end of file diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 61ef7068b2bb..32c7fa64d09d 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -2,6 +2,7 @@ import json import logging import os +import re import shutil import sys import tempfile @@ -63,14 +64,6 @@ def __init__( ) -> None: self.registry_url = registry_url - if base_image is None: - python_version = "{}.{}".format( - sys.version_info.major, sys.version_info.minor - ) - self.base_image = "python:{}".format(python_version) - else: - self.base_image = base_image - if sys.platform == "win32": default_url = "npipe:////./pipe/docker_engine" else: @@ -79,12 +72,17 @@ def __init__( self.image_name = image_name self.image_tag = image_tag self.python_dependencies = python_dependencies or [] + self.python_dependencies.append("wheel") + self.env_vars = env_vars or {} + self.env_vars["PREFECT__USER_CONFIG_PATH"] = "/root/.prefect/config.toml" + self.files = files or {} self.flows = dict() # type: Dict[str, str] self._flows = dict() # type: Dict[str, "prefect.core.flow.Flow"] self.base_url = base_url or default_url self.local_image = local_image + self.extra_commands = [] version = prefect.__version__.split("+") if prefect_version is None: @@ -92,6 +90,32 @@ def __init__( else: self.prefect_version = prefect_version + if base_image is None: + python_version = "{}.{}".format( + sys.version_info.major, sys.version_info.minor + ) + if re.match("^[0-9]+\.[0-9]+\.[0-9]+$", self.prefect_version) != None: + # note: this does not necessarily mean that we have built/pushed all previous prefect versions to dockerhub + self.base_image = "prefecthq/prefect:{}-python{}".format( + self.prefect_version, python_version + ) + elif self.prefect_version == "master": + # use the latest image for the given python version + self.base_image = "prefecthq/prefect:python{}".format(python_version) + else: + # create an image from python:*-slim directly + self.base_image = "python:{}-slim".format(python_version) + self.extra_commands.extend( + [ + "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", + "pip install git+https://github.com/PrefectHQ/prefect.git@{}#egg=prefect[kubernetes]".format( + self.prefect_version + ), + ] + ) + else: + self.base_image = base_image + not_absolute = [ file_path for file_path in self.files if not os.path.isabs(file_path) ] @@ -311,11 +335,11 @@ def create_dockerfile_object(self, directory: str = None) -> None: with open(os.path.join(directory, "Dockerfile"), "w+") as dockerfile: - # Generate RUN pip install commands for python dependencies - pip_installs = "" + # Generate single pip install command for python dependencies + pip_installs = "RUN pip install " if self.python_dependencies: for dependency in self.python_dependencies: - pip_installs += "RUN pip install {}\n".format(dependency) + pip_installs += "{} ".format(dependency) # Generate ENV variables to load into the image env_vars = "" @@ -355,6 +379,11 @@ def create_dockerfile_object(self, directory: str = None) -> None: source="{}.flow".format(clean_name), dest=flow_location ) + # Write all extra commands that should be run in the image + extra_commands = "" + for cmd in self.extra_commands: + extra_commands += "RUN {}\n".format(cmd) + # Write a healthcheck script into the image with open( os.path.join(os.path.dirname(__file__), "_healthcheck.py"), "r" @@ -369,28 +398,24 @@ def create_dockerfile_object(self, directory: str = None) -> None: FROM {base_image} RUN pip install pip --upgrade - RUN pip install wheel + {extra_commands} {pip_installs} - RUN mkdir /root/.prefect/ + RUN mkdir -p /root/.prefect/ {copy_flows} COPY healthcheck.py /root/.prefect/healthcheck.py {copy_files} - ENV PREFECT__USER_CONFIG_PATH="/root/.prefect/config.toml" {env_vars} - # update version if base image already has prefect installed - RUN pip install -U git+https://github.com/PrefectHQ/prefect.git@{version}#egg=prefect[kubernetes] - RUN python /root/.prefect/healthcheck.py '[{flow_file_paths}]' '{python_version}' """.format( + extra_commands=extra_commands, base_image=self.base_image, pip_installs=pip_installs, copy_flows=copy_flows, copy_files=copy_files, env_vars=env_vars, - version=self.prefect_version, flow_file_paths=", ".join( ['"{}"'.format(k) for k in self.flows.values()] ), diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index af2bcb48fd0b..f73b2ff1e417 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -47,11 +47,13 @@ def test_empty_docker_storage(monkeypatch, platform, url): storage = Docker() assert not storage.registry_url - assert storage.base_image.startswith("python:") + assert storage.base_image.startswith("prefecthq/prefect:python") assert not storage.image_name assert not storage.image_tag - assert not storage.python_dependencies - assert not storage.env_vars + assert storage.python_dependencies == ["wheel"] + assert storage.env_vars == { + "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml" + } assert not storage.files assert storage.prefect_version assert storage.base_url == url @@ -63,7 +65,7 @@ def test_docker_init_responds_to_python_version(monkeypatch, version_info): version_mock = MagicMock(major=version_info[0], minor=version_info[1]) monkeypatch.setattr(sys, "version_info", version_mock) storage = Docker() - assert storage.base_image == "python:{}.{}".format(*version_info) + assert storage.base_image == "prefecthq/prefect:python{}.{}".format(*version_info) @pytest.mark.parametrize( @@ -97,8 +99,11 @@ def test_initialized_docker_storage(): assert storage.base_image == "test3" assert storage.image_name == "test4" assert storage.image_tag == "test5" - assert storage.python_dependencies == ["test"] - assert storage.env_vars == {"test": "1"} + assert storage.python_dependencies == ["test", "wheel"] + assert storage.env_vars == { + "test": "1", + "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml", + } assert storage.base_url == "test_url" assert storage.prefect_version == "my-branch" assert storage.local_image @@ -301,8 +306,26 @@ def test_create_dockerfile_from_base_image(): assert "FROM python:3.6" in output -def test_create_dockerfile_from_prefect_version(): - storage = Docker(prefect_version="master") +@pytest.mark.parametrize( + "prefect_version", + [ + ("0.5.3", ("FROM prefecthq/prefect:0.5.3-python3.6",)), + ("master", ("FROM prefecthq/prefect:python3.6",)), + ( + "424be6b5ed8d3be85064de4b95b5c3d7cb665510", + ( + "FROM python:3.6-slim", + "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", + "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510#egg=prefect[kubernetes]", + ), + ), + ], +) +def test_create_dockerfile_from_prefect_version(monkeypatch, prefect_version): + version_mock = MagicMock(major=3, minor=6) + monkeypatch.setattr(sys, "version_info", version_mock) + + storage = Docker(prefect_version=prefect_version[0]) with tempfile.TemporaryDirectory() as tempdir: storage.create_dockerfile_object(directory=tempdir) @@ -310,7 +333,8 @@ def test_create_dockerfile_from_prefect_version(): with open(os.path.join(tempdir, "Dockerfile"), "r") as dockerfile: output = dockerfile.read() - assert "prefect.git@master" in output + for content in prefect_version[1]: + assert content in output def test_create_dockerfile_with_weird_flow_name(): From 91d107dc560b875ac5e4aacedae228bd650ba055 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 09:41:55 -0400 Subject: [PATCH 2/8] added type hint for docker store extra_commands --- src/prefect/environments/storage/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 32c7fa64d09d..31251c946c4a 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -82,7 +82,7 @@ def __init__( self._flows = dict() # type: Dict[str, "prefect.core.flow.Flow"] self.base_url = base_url or default_url self.local_image = local_image - self.extra_commands = [] + self.extra_commands = [] # type: List[str] version = prefect.__version__.split("+") if prefect_version is None: From 7576d7ba0b9c813d9af2004a67c74ef7a458a60d Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 11:43:09 -0400 Subject: [PATCH 3/8] explicitly check for user given env in dockerfile render --- tests/environments/storage/test_docker_storage.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index f73b2ff1e417..2b78ecc2bf8e 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -1,4 +1,5 @@ import os +import re import sys import tempfile from unittest.mock import MagicMock @@ -386,9 +387,20 @@ def test_create_dockerfile_from_everything(): with open(os.path.join(tempdir, "Dockerfile"), "r") as dockerfile: output = dockerfile.read() + print(output) + assert "FROM test3" in output assert "COPY test ./test2" in output - assert "ENV test=1" in output + + # ensure there is a "ENV ... test=1" in the output + results = re.search( + r"ENV(\s+[a-zA-Z0-9_]*\=[^\\]*\\\s*$)*\s*(?Ptest=1)", + output, + re.MULTILINE, + ) + assert results != None + assert results.group("result") == "test=1" + assert "COPY healthcheck.py /root/.prefect/healthcheck.py" in output assert "COPY test.flow /root/.prefect/test.prefect" in output assert "COPY other.flow /root/.prefect/other.prefect" in output From 613713b7945f1c06b7bc0e2fbae2e6f4f637f059 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 15:30:34 -0400 Subject: [PATCH 4/8] only allow docker builds for tagged commits on master --- .circleci/config.yml | 14 +++++++++++--- src/prefect/environments/storage/docker.py | 1 - 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f40341f04590..f3492a362c6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,14 @@ jobs: PYTHON_TAG: python<< parameters.python-version >> steps: - checkout + - run: + name: Master branch check + command: | + apk add git + if [[ $(git branch --contains $CIRCLE_SHA1 --points-at master | wc -l) -ne 1 ]]; then + echo "commit $CIRCLE_SHA1 is not a member of the master branch" + exit 1 + fi - setup_remote_docker: docker_layer_caching: true - run: @@ -350,14 +358,14 @@ workflows: python-version: "3.5" filters: branches: - only: master + ignore: /.*/ tags: only: /^[0-9]+\.[0-9]+\.[0-9]+$/ - build-docker-image: python-version: "3.6" filters: branches: - only: master + ignore: /.*/ tags: only: /^[0-9]+\.[0-9]+\.[0-9]+$/ - build-docker-image: @@ -365,6 +373,6 @@ workflows: tag-latest: true filters: branches: - only: master + ignore: /.*/ tags: only: /^[0-9]+\.[0-9]+\.[0-9]+$/ \ No newline at end of file diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 31251c946c4a..0f86ad286306 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -95,7 +95,6 @@ def __init__( sys.version_info.major, sys.version_info.minor ) if re.match("^[0-9]+\.[0-9]+\.[0-9]+$", self.prefect_version) != None: - # note: this does not necessarily mean that we have built/pushed all previous prefect versions to dockerhub self.base_image = "prefecthq/prefect:{}-python{}".format( self.prefect_version, python_version ) From 6194846f04c710b24ab0669af49ed4aa0b53b74e Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 15:49:03 -0400 Subject: [PATCH 5/8] swap job names and parameters to underscores --- .circleci/config.yml | 69 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3492a362c6e..849580d84217 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ jobs: # test a standard install of prefect # is importable in python 3.5.2 # there was a typing bug in 3.5.2 that this attempts to catch - test-py352-import-prefect: + test_py352_import_prefect: docker: - image: python:3.5.2 @@ -75,7 +75,7 @@ jobs: # test a standard install of prefect # with all requriements pinned to their lowest allowed versions # to ensure our requirements.txt file is accurate - test-lower-prefect: + test_lower_prefect: docker: - image: python:3.5.2 @@ -115,7 +115,7 @@ jobs: # test a standard install of prefect # this ensures we correctly capture all ImportError sitautions # caused by many package dependency options - test-vanilla-prefect: + test_vanilla_prefect: docker: - image: python:3.6 @@ -148,7 +148,7 @@ jobs: # Run unit tests in Python 3.5-3.7 # ---------------------------------- - test-35: + test_35: docker: - image: python:3.5 @@ -177,7 +177,7 @@ jobs: paths: - coverage - test-36: + test_36: docker: - image: python:3.6 steps: @@ -205,7 +205,7 @@ jobs: paths: - coverage - test-37: + test_37: docker: - image: python:3.7 steps: @@ -227,7 +227,7 @@ jobs: name: Run tests command: pytest -rfEsx . - test-airflow: + test_airflow: docker: - image: continuumio/miniconda3:4.6.14 steps: @@ -258,7 +258,7 @@ jobs: paths: - coverage - upload-coverage: + upload_coverage: docker: - image: python:3.6 steps: @@ -267,21 +267,22 @@ jobs: name: Upload Coverage command: bash <(curl -s https://codecov.io/bash) -cF python -s "/tmp/workspace/coverage/" - build-docker-image: + build_docker_image: docker: - image: docker parameters: - python-version: + python_version: type: string - tag-latest: + tag_latest: type: boolean default: false environment: - PYTHON_VERSION: << parameters.python-version >> - PYTHON_TAG: python<< parameters.python-version >> + PYTHON_VERSION: << parameters.python_version >> + PYTHON_TAG: python<< parameters.python_version >> steps: - checkout - run: + # todo: is there a better way to ensure that this is a commit on master? name: Master branch check command: | apk add git @@ -302,7 +303,7 @@ jobs: -t prefecthq/prefect:$PYTHON_TAG . - when: - condition: << parameters.tag-latest >> + condition: << parameters.tag_latest >> steps: - run: name: Tag latest image @@ -319,7 +320,7 @@ jobs: docker push prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} docker push prefecthq/prefect:$PYTHON_TAG - when: - condition: << parameters.tag-latest >> + condition: << parameters.tag_latest >> steps: - run: name: Push latest tag @@ -332,19 +333,19 @@ workflows: "Run tests": jobs: - - test-35 - - test-36 - - test-37 - - test-lower-prefect - - test-vanilla-prefect - - test-py352-import-prefect - - test-airflow - - upload-coverage: + - test_35 + - test_36 + - test_37 + - test_lower_prefect + - test_vanilla_prefect + - test_py352_import_prefect + - test_airflow + - upload_coverage: requires: - - test-35 - - test-36 - - test-vanilla-prefect - - test-airflow + - test_35 + - test_36 + - test_vanilla_prefect + - test_airflow "Check code style and docs": jobs: @@ -354,23 +355,23 @@ workflows: "Build docker images": jobs: - - build-docker-image: - python-version: "3.5" + - build_docker_image: + python_version: "3.5" filters: branches: ignore: /.*/ tags: only: /^[0-9]+\.[0-9]+\.[0-9]+$/ - - build-docker-image: - python-version: "3.6" + - build_docker_image: + python_version: "3.6" filters: branches: ignore: /.*/ tags: only: /^[0-9]+\.[0-9]+\.[0-9]+$/ - - build-docker-image: - python-version: "3.7" - tag-latest: true + - build_docker_image: + python_version: "3.7" + tag_latest: true filters: branches: ignore: /.*/ From 6b9b411123d09704c8bffd6c4fe74959da7fbbe2 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 16:11:05 -0400 Subject: [PATCH 6/8] update docs and changelog --- CHANGELOG.md | 3 ++- src/prefect/environments/storage/docker.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba51c549ca51..221fcaeab6a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Enhancements - Add the ability to delete task tag limits using the client - [#1622](https://github.com/PrefectHQ/prefect/pull/1622) -- Adds an "Ask for help" button with a link to the prefect.io support page [#1637](https://github.com/PrefectHQ/prefect/pull/1637) +- Adds an "Ask for help" button with a link to the prefect.io support page - [#1637](https://github.com/PrefectHQ/prefect/pull/1637) +- Reduces the size of the `prefecthq/prefect` Docker image by ~400MB, which is now the base Docker image used in Flows - [#1648](https://github.com/PrefectHQ/prefect/pull/1648) ### Task Library diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 0f86ad286306..b5d3ab37db56 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -36,7 +36,8 @@ class Docker(Storage): Args: - registry_url (str, optional): URL of a registry to push the image to; image will not be pushed if not provided - - base_image (str, optional): the base image for this environment (e.g. `python:3.6`), defaults to `python:3.6` + - base_image (str, optional): the base image for this environment (e.g. `python:3.6`), defaults to the `prefecthq/prefect` image + matching your python version and prefect core library version used at runtime. - python_dependencies (List[str], optional): list of pip installable dependencies for the image - image_name (str, optional): name of the image to use when building, populated with a UUID after build - image_tag (str, optional): tag of the image to use when building, populated with a UUID after build From c8b90075e14083c1360840be424a722754b4f9a5 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 22 Oct 2019 17:22:43 -0400 Subject: [PATCH 7/8] remove print in test Co-Authored-By: Josh Meek <40716964+joshmeek@users.noreply.github.com> --- tests/environments/storage/test_docker_storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index 2b78ecc2bf8e..62ab341e9886 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -387,8 +387,6 @@ def test_create_dockerfile_from_everything(): with open(os.path.join(tempdir, "Dockerfile"), "r") as dockerfile: output = dockerfile.read() - print(output) - assert "FROM test3" in output assert "COPY test ./test2" in output From f0daf61611e834048b7776df106e721756f892bf Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 23 Oct 2019 10:02:21 -0400 Subject: [PATCH 8/8] ensure that an empty env var will fail a docker build --- .circleci/config.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 849580d84217..15244b5289cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -294,14 +294,15 @@ jobs: docker_layer_caching: true - run: name: Build image - command: >- - docker build - --build-arg GIT_SHA=$CIRCLE_SHA1 - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - --build-arg PREFECT_VERSION=$CIRCLE_TAG - -t prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} - -t prefecthq/prefect:$PYTHON_TAG - . + command: | + set -u + docker build \ + --build-arg GIT_SHA=$CIRCLE_SHA1 \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg PREFECT_VERSION=$CIRCLE_TAG \ + -t prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} \ + -t prefecthq/prefect:$PYTHON_TAG \ + . - when: condition: << parameters.tag_latest >> steps: