diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e9eff3b65b3..15244b5289cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 references: workspace_root: &workspace_root /tmp/workspace @@ -6,6 +6,7 @@ references: attach_workspace: at: *workspace_root + jobs: # ---------------------------------- # Check formatting @@ -56,7 +57,7 @@ jobs: # test a standard install of prefect # is importable in python 3.5.2 # there was a typing bug in 3.5.2 that this attempts to catch - test-py352-import-prefect: + test_py352_import_prefect: docker: - image: python:3.5.2 @@ -74,7 +75,7 @@ jobs: # test a standard install of prefect # with all requriements pinned to their lowest allowed versions # to ensure our requirements.txt file is accurate - test-lower-prefect: + test_lower_prefect: docker: - image: python:3.5.2 @@ -114,7 +115,7 @@ jobs: # test a standard install of prefect # this ensures we correctly capture all ImportError sitautions # caused by many package dependency options - test-vanilla-prefect: + test_vanilla_prefect: docker: - image: python:3.6 @@ -147,7 +148,7 @@ jobs: # Run unit tests in Python 3.5-3.7 # ---------------------------------- - test-3.5: + test_35: docker: - image: python:3.5 @@ -176,7 +177,7 @@ jobs: paths: - coverage - test-3.6: + test_36: docker: - image: python:3.6 steps: @@ -204,7 +205,7 @@ jobs: paths: - coverage - test-3.7: + test_37: docker: - image: python:3.7 steps: @@ -226,7 +227,7 @@ jobs: name: Run tests command: pytest -rfEsx . - test-airflow: + test_airflow: docker: - image: continuumio/miniconda3:4.6.14 steps: @@ -257,7 +258,7 @@ jobs: paths: - coverage - upload-coverage: + upload_coverage: docker: - image: python:3.6 steps: @@ -266,43 +267,86 @@ jobs: name: Upload Coverage command: bash <(curl -s https://codecov.io/bash) -cF python -s "/tmp/workspace/coverage/" - build_image: + build_docker_image: docker: - image: docker + parameters: + python_version: + type: string + tag_latest: + type: boolean + default: false + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTHON_TAG: python<< parameters.python_version >> steps: - - setup_remote_docker - checkout - run: - name: Docker Build - command: docker build -t prefecthq/prefect . + # todo: is there a better way to ensure that this is a commit on master? + name: Master branch check + command: | + apk add git + if [[ $(git branch --contains $CIRCLE_SHA1 --points-at master | wc -l) -ne 1 ]]; then + echo "commit $CIRCLE_SHA1 is not a member of the master branch" + exit 1 + fi + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build image + command: | + set -u + docker build \ + --build-arg GIT_SHA=$CIRCLE_SHA1 \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg PREFECT_VERSION=$CIRCLE_TAG \ + -t prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} \ + -t prefecthq/prefect:$PYTHON_TAG \ + . + - when: + condition: << parameters.tag_latest >> + steps: + - run: + name: Tag latest image + command: | + docker tag prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} prefecthq/prefect:latest - run: name: Test image command: | docker run -dit prefecthq/prefect /bin/bash -c 'curl -fL0 https://raw.githubusercontent.com/PrefectHQ/prefect/master/examples/retries_with_mapping.py | python' - run: - name: Authenticate with Docker Hub and push + name: Push versioned tags command: | docker login --username $DOCKER_HUB_USER --password $DOCKER_HUB_PW - docker push prefecthq/prefect + docker push prefecthq/prefect:${CIRCLE_TAG}-${PYTHON_TAG} + docker push prefecthq/prefect:$PYTHON_TAG + - when: + condition: << parameters.tag_latest >> + steps: + - run: + name: Push latest tag + command: | + docker login --username $DOCKER_HUB_USER --password $DOCKER_HUB_PW + docker push prefecthq/prefect:latest workflows: version: 2 "Run tests": jobs: - - test-3.5 - - test-3.6 - - test-3.7 - - test-lower-prefect - - test-vanilla-prefect - - test-py352-import-prefect - - test-airflow - - upload-coverage: + - test_35 + - test_36 + - test_37 + - test_lower_prefect + - test_vanilla_prefect + - test_py352_import_prefect + - test_airflow + - upload_coverage: requires: - - test-3.5 - - test-3.6 - - test-vanilla-prefect - - test-airflow + - test_35 + - test_36 + - test_vanilla_prefect + - test_airflow "Check code style and docs": jobs: @@ -312,7 +356,25 @@ workflows: "Build docker images": jobs: - - build_image: + - build_docker_image: + python_version: "3.5" + filters: + branches: + ignore: /.*/ + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ + - build_docker_image: + python_version: "3.6" + filters: + branches: + ignore: /.*/ + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ + - build_docker_image: + python_version: "3.7" + tag_latest: true filters: branches: - only: master + ignore: /.*/ + tags: + only: /^[0-9]+\.[0-9]+\.[0-9]+$/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ba51c549ca51..221fcaeab6a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Enhancements - Add the ability to delete task tag limits using the client - [#1622](https://github.com/PrefectHQ/prefect/pull/1622) -- Adds an "Ask for help" button with a link to the prefect.io support page [#1637](https://github.com/PrefectHQ/prefect/pull/1637) +- Adds an "Ask for help" button with a link to the prefect.io support page - [#1637](https://github.com/PrefectHQ/prefect/pull/1637) +- Reduces the size of the `prefecthq/prefect` Docker image by ~400MB, which is now the base Docker image used in Flows - [#1648](https://github.com/PrefectHQ/prefect/pull/1648) ### Task Library diff --git a/Dockerfile b/Dockerfile index 72437519daa2..0261a66980d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,20 @@ -ARG PYTHON_VERSION=3.6 +ARG PYTHON_VERSION=${PYTHON_VERSION:-3.6} +FROM python:${PYTHON_VERSION}-slim -FROM python:${PYTHON_VERSION} -LABEL maintainer="help@prefect.io" -ARG GIT_POINTER=master +RUN apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/* -RUN pip install git+https://github.com/PrefectHQ/prefect.git@${GIT_POINTER}#egg=prefect[kubernetes] +ARG PREFECT_VERSION +RUN pip install git+https://github.com/PrefectHQ/prefect.git@${PREFECT_VERSION}#egg=prefect[kubernetes] RUN mkdir /root/.prefect/ + +ARG GIT_SHA +ARG BUILD_DATE + +LABEL maintainer="help@prefect.io" +LABEL io.prefect.python-version=${PYTHON_VERSION} +LABEL org.label-schema.schema-version = "1.0" +LABEL org.label-schema.name="prefect" +LABEL org.label-schema.url="https://www.prefect.io/" +LABEL org.label-schema.version=${PREFECT_VERSION} +LABEL org.label-schema.vcs-ref=${GIT_SHA} +LABEL org.label-schema.build-date=${BUILD_DATE} \ No newline at end of file diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 61ef7068b2bb..b5d3ab37db56 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -2,6 +2,7 @@ import json import logging import os +import re import shutil import sys import tempfile @@ -35,7 +36,8 @@ class Docker(Storage): Args: - registry_url (str, optional): URL of a registry to push the image to; image will not be pushed if not provided - - base_image (str, optional): the base image for this environment (e.g. `python:3.6`), defaults to `python:3.6` + - base_image (str, optional): the base image for this environment (e.g. `python:3.6`), defaults to the `prefecthq/prefect` image + matching your python version and prefect core library version used at runtime. - python_dependencies (List[str], optional): list of pip installable dependencies for the image - image_name (str, optional): name of the image to use when building, populated with a UUID after build - image_tag (str, optional): tag of the image to use when building, populated with a UUID after build @@ -63,14 +65,6 @@ def __init__( ) -> None: self.registry_url = registry_url - if base_image is None: - python_version = "{}.{}".format( - sys.version_info.major, sys.version_info.minor - ) - self.base_image = "python:{}".format(python_version) - else: - self.base_image = base_image - if sys.platform == "win32": default_url = "npipe:////./pipe/docker_engine" else: @@ -79,12 +73,17 @@ def __init__( self.image_name = image_name self.image_tag = image_tag self.python_dependencies = python_dependencies or [] + self.python_dependencies.append("wheel") + self.env_vars = env_vars or {} + self.env_vars["PREFECT__USER_CONFIG_PATH"] = "/root/.prefect/config.toml" + self.files = files or {} self.flows = dict() # type: Dict[str, str] self._flows = dict() # type: Dict[str, "prefect.core.flow.Flow"] self.base_url = base_url or default_url self.local_image = local_image + self.extra_commands = [] # type: List[str] version = prefect.__version__.split("+") if prefect_version is None: @@ -92,6 +91,31 @@ def __init__( else: self.prefect_version = prefect_version + if base_image is None: + python_version = "{}.{}".format( + sys.version_info.major, sys.version_info.minor + ) + if re.match("^[0-9]+\.[0-9]+\.[0-9]+$", self.prefect_version) != None: + self.base_image = "prefecthq/prefect:{}-python{}".format( + self.prefect_version, python_version + ) + elif self.prefect_version == "master": + # use the latest image for the given python version + self.base_image = "prefecthq/prefect:python{}".format(python_version) + else: + # create an image from python:*-slim directly + self.base_image = "python:{}-slim".format(python_version) + self.extra_commands.extend( + [ + "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", + "pip install git+https://github.com/PrefectHQ/prefect.git@{}#egg=prefect[kubernetes]".format( + self.prefect_version + ), + ] + ) + else: + self.base_image = base_image + not_absolute = [ file_path for file_path in self.files if not os.path.isabs(file_path) ] @@ -311,11 +335,11 @@ def create_dockerfile_object(self, directory: str = None) -> None: with open(os.path.join(directory, "Dockerfile"), "w+") as dockerfile: - # Generate RUN pip install commands for python dependencies - pip_installs = "" + # Generate single pip install command for python dependencies + pip_installs = "RUN pip install " if self.python_dependencies: for dependency in self.python_dependencies: - pip_installs += "RUN pip install {}\n".format(dependency) + pip_installs += "{} ".format(dependency) # Generate ENV variables to load into the image env_vars = "" @@ -355,6 +379,11 @@ def create_dockerfile_object(self, directory: str = None) -> None: source="{}.flow".format(clean_name), dest=flow_location ) + # Write all extra commands that should be run in the image + extra_commands = "" + for cmd in self.extra_commands: + extra_commands += "RUN {}\n".format(cmd) + # Write a healthcheck script into the image with open( os.path.join(os.path.dirname(__file__), "_healthcheck.py"), "r" @@ -369,28 +398,24 @@ def create_dockerfile_object(self, directory: str = None) -> None: FROM {base_image} RUN pip install pip --upgrade - RUN pip install wheel + {extra_commands} {pip_installs} - RUN mkdir /root/.prefect/ + RUN mkdir -p /root/.prefect/ {copy_flows} COPY healthcheck.py /root/.prefect/healthcheck.py {copy_files} - ENV PREFECT__USER_CONFIG_PATH="/root/.prefect/config.toml" {env_vars} - # update version if base image already has prefect installed - RUN pip install -U git+https://github.com/PrefectHQ/prefect.git@{version}#egg=prefect[kubernetes] - RUN python /root/.prefect/healthcheck.py '[{flow_file_paths}]' '{python_version}' """.format( + extra_commands=extra_commands, base_image=self.base_image, pip_installs=pip_installs, copy_flows=copy_flows, copy_files=copy_files, env_vars=env_vars, - version=self.prefect_version, flow_file_paths=", ".join( ['"{}"'.format(k) for k in self.flows.values()] ), diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index af2bcb48fd0b..62ab341e9886 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -1,4 +1,5 @@ import os +import re import sys import tempfile from unittest.mock import MagicMock @@ -47,11 +48,13 @@ def test_empty_docker_storage(monkeypatch, platform, url): storage = Docker() assert not storage.registry_url - assert storage.base_image.startswith("python:") + assert storage.base_image.startswith("prefecthq/prefect:python") assert not storage.image_name assert not storage.image_tag - assert not storage.python_dependencies - assert not storage.env_vars + assert storage.python_dependencies == ["wheel"] + assert storage.env_vars == { + "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml" + } assert not storage.files assert storage.prefect_version assert storage.base_url == url @@ -63,7 +66,7 @@ def test_docker_init_responds_to_python_version(monkeypatch, version_info): version_mock = MagicMock(major=version_info[0], minor=version_info[1]) monkeypatch.setattr(sys, "version_info", version_mock) storage = Docker() - assert storage.base_image == "python:{}.{}".format(*version_info) + assert storage.base_image == "prefecthq/prefect:python{}.{}".format(*version_info) @pytest.mark.parametrize( @@ -97,8 +100,11 @@ def test_initialized_docker_storage(): assert storage.base_image == "test3" assert storage.image_name == "test4" assert storage.image_tag == "test5" - assert storage.python_dependencies == ["test"] - assert storage.env_vars == {"test": "1"} + assert storage.python_dependencies == ["test", "wheel"] + assert storage.env_vars == { + "test": "1", + "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml", + } assert storage.base_url == "test_url" assert storage.prefect_version == "my-branch" assert storage.local_image @@ -301,8 +307,26 @@ def test_create_dockerfile_from_base_image(): assert "FROM python:3.6" in output -def test_create_dockerfile_from_prefect_version(): - storage = Docker(prefect_version="master") +@pytest.mark.parametrize( + "prefect_version", + [ + ("0.5.3", ("FROM prefecthq/prefect:0.5.3-python3.6",)), + ("master", ("FROM prefecthq/prefect:python3.6",)), + ( + "424be6b5ed8d3be85064de4b95b5c3d7cb665510", + ( + "FROM python:3.6-slim", + "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", + "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510#egg=prefect[kubernetes]", + ), + ), + ], +) +def test_create_dockerfile_from_prefect_version(monkeypatch, prefect_version): + version_mock = MagicMock(major=3, minor=6) + monkeypatch.setattr(sys, "version_info", version_mock) + + storage = Docker(prefect_version=prefect_version[0]) with tempfile.TemporaryDirectory() as tempdir: storage.create_dockerfile_object(directory=tempdir) @@ -310,7 +334,8 @@ def test_create_dockerfile_from_prefect_version(): with open(os.path.join(tempdir, "Dockerfile"), "r") as dockerfile: output = dockerfile.read() - assert "prefect.git@master" in output + for content in prefect_version[1]: + assert content in output def test_create_dockerfile_with_weird_flow_name(): @@ -364,7 +389,16 @@ def test_create_dockerfile_from_everything(): assert "FROM test3" in output assert "COPY test ./test2" in output - assert "ENV test=1" in output + + # ensure there is a "ENV ... test=1" in the output + results = re.search( + r"ENV(\s+[a-zA-Z0-9_]*\=[^\\]*\\\s*$)*\s*(?Ptest=1)", + output, + re.MULTILINE, + ) + assert results != None + assert results.group("result") == "test=1" + assert "COPY healthcheck.py /root/.prefect/healthcheck.py" in output assert "COPY test.flow /root/.prefect/test.prefect" in output assert "COPY other.flow /root/.prefect/other.prefect" in output