From 2945a8d26d9b83028ddac1806a2a880586214969 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 12:46:12 -0400 Subject: [PATCH 01/13] Add healthcheck for flow environment dependencies --- .../environments/storage/_healthcheck.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/prefect/environments/storage/_healthcheck.py b/src/prefect/environments/storage/_healthcheck.py index 58fc1569889e..1665569fafbe 100644 --- a/src/prefect/environments/storage/_healthcheck.py +++ b/src/prefect/environments/storage/_healthcheck.py @@ -93,6 +93,36 @@ def result_handler_check(flows: list): print("Result Handler check: OK") +def environment_requirement_check(flows: list): + from prefect import environments + + # Tests for imports that are required by certain environments + for flow in flows: + if isinstance( + flow.environment, environments.DaskKubernetesEnvironment + ) or isinstance(flow.environment, environments.KubernetesJobEnvironment): + try: + import kubernetes + except ModuleNotFoundError: + raise ModuleNotFoundError( + "Using {} requires the `kubernetes` dependency.".format( + flow.environment.__class__.__name__ + ) + ) + elif isinstance(flow.environment, environments.FargateTaskEnvironment): + try: + import boto3 + import botocore + except ModuleNotFoundError: + raise ModuleNotFoundError( + "Using {} requires both `boto3` and `botocore` dependencies.".format( + flow.environment.__class__.__name__ + ) + ) + + print("Environment requirement check: OK") + + if __name__ == "__main__": flow_file_path, python_version = sys.argv[1:3] @@ -100,4 +130,5 @@ def result_handler_check(flows: list): system_check(python_version) flows = cloudpickle_deserialization_check(flow_file_path) result_handler_check(flows) + environment_requirement_check(flows) print("All health checks passed.") From ce022783e764f72b0118ace4a3729ad0ae10375b Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 12:47:39 -0400 Subject: [PATCH 02/13] Remove kubernetes dependency from docker storage --- src/prefect/environments/storage/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index b5d3ab37db56..2a7165ad8c17 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -108,7 +108,7 @@ def __init__( self.extra_commands.extend( [ "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", - "pip install git+https://github.com/PrefectHQ/prefect.git@{}#egg=prefect[kubernetes]".format( + "pip install git+https://github.com/PrefectHQ/prefect.git@{}#egg=prefect".format( self.prefect_version ), ] From 1129130e4e61bd877c5cd4a9c9361d2a791fda98 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 12:48:25 -0400 Subject: [PATCH 03/13] Update docker storage test output --- tests/environments/storage/test_docker_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index 62ab341e9886..d3763afce1e5 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -317,7 +317,7 @@ def test_create_dockerfile_from_base_image(): ( "FROM python:3.6-slim", "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", - "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510#egg=prefect[kubernetes]", + "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510#egg=prefect", ), ), ], From c3984d6e73c93be6a75444f4dadc12ff018c3e0c Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:08:19 -0400 Subject: [PATCH 04/13] Adjust environments and healthcheck for dynamic dependency specification --- src/prefect/environments/execution/base.py | 4 +++ .../environments/execution/dask/k8s.py | 4 +++ .../execution/fargate/fargate_task.py | 4 +++ src/prefect/environments/execution/k8s/job.py | 4 +++ src/prefect/environments/execution/local.py | 4 +++ src/prefect/environments/execution/remote.py | 4 +++ .../environments/storage/_healthcheck.py | 27 ++++++------------- .../execution/test_base_environment.py | 5 ++++ .../execution/test_dask_k8s_environment.py | 5 ++++ .../test_fargate_task_environment.py | 5 ++++ .../execution/test_k8s_job_environment.py | 5 ++++ .../execution/test_local_environment.py | 5 ++++ .../execution/test_remote_environment.py | 5 ++++ 13 files changed, 62 insertions(+), 19 deletions(-) diff --git a/src/prefect/environments/execution/base.py b/src/prefect/environments/execution/base.py index 6245fb72fa9a..b3b3796d87f0 100644 --- a/src/prefect/environments/execution/base.py +++ b/src/prefect/environments/execution/base.py @@ -52,6 +52,10 @@ def __init__( def __repr__(self) -> str: return "".format(type(self).__name__) + @property + def dependencies(self) -> list: + return [] + def setup(self, storage: "Storage") -> None: """ Sets up any infrastructure needed for this environment diff --git a/src/prefect/environments/execution/dask/k8s.py b/src/prefect/environments/execution/dask/k8s.py index ec6ae9e51410..173819e9c440 100644 --- a/src/prefect/environments/execution/dask/k8s.py +++ b/src/prefect/environments/execution/dask/k8s.py @@ -88,6 +88,10 @@ def __init__( super().__init__(labels=labels, on_start=on_start, on_exit=on_exit) + @property + def dependencies(self) -> list: + return ["kubernetes"] + def setup(self, storage: "Docker") -> None: # type: ignore if self.private_registry: from kubernetes import client, config diff --git a/src/prefect/environments/execution/fargate/fargate_task.py b/src/prefect/environments/execution/fargate/fargate_task.py index 5ff4b8ee5db0..e6cb2578e677 100644 --- a/src/prefect/environments/execution/fargate/fargate_task.py +++ b/src/prefect/environments/execution/fargate/fargate_task.py @@ -142,6 +142,10 @@ def _parse_kwargs(self, user_kwargs: dict) -> tuple: return task_definition_kwargs, task_run_kwargs + @property + def dependencies(self) -> list: + return ["boto3", "botocore"] + def setup(self, storage: "Docker") -> None: # type: ignore """ Register the task definition if it does not already exist. diff --git a/src/prefect/environments/execution/k8s/job.py b/src/prefect/environments/execution/k8s/job.py index 79b17f178e2f..070597e54841 100644 --- a/src/prefect/environments/execution/k8s/job.py +++ b/src/prefect/environments/execution/k8s/job.py @@ -59,6 +59,10 @@ def __init__( super().__init__(labels=labels, on_start=on_start, on_exit=on_exit) + @property + def dependencies(self) -> list: + return ["kubernetes"] + def execute( # type: ignore self, storage: "Docker", flow_location: str, **kwargs: Any ) -> None: diff --git a/src/prefect/environments/execution/local.py b/src/prefect/environments/execution/local.py index fcec0d8d7c85..2bffab0d50a4 100644 --- a/src/prefect/environments/execution/local.py +++ b/src/prefect/environments/execution/local.py @@ -27,6 +27,10 @@ def __init__( ) -> None: super().__init__(labels=labels, on_start=on_start, on_exit=on_exit) + @property + def dependencies(self) -> list: + return [] + def execute(self, storage: "Storage", flow_location: str, **kwargs: Any) -> None: """ Executes the flow for this environment from the storage parameter, diff --git a/src/prefect/environments/execution/remote.py b/src/prefect/environments/execution/remote.py index a81e434b5ffa..9489296bcbfc 100644 --- a/src/prefect/environments/execution/remote.py +++ b/src/prefect/environments/execution/remote.py @@ -46,6 +46,10 @@ def __init__( self.executor_kwargs = executor_kwargs or dict() super().__init__(labels=labels, on_start=on_start, on_exit=on_exit) + @property + def dependencies(self) -> list: + return [] + def execute( # type: ignore self, storage: "Storage", flow_location: str, **kwargs: Any ) -> None: diff --git a/src/prefect/environments/storage/_healthcheck.py b/src/prefect/environments/storage/_healthcheck.py index 1665569fafbe..ab459baf449f 100644 --- a/src/prefect/environments/storage/_healthcheck.py +++ b/src/prefect/environments/storage/_healthcheck.py @@ -5,6 +5,7 @@ """ import ast +import importlib import sys import warnings @@ -94,29 +95,17 @@ def result_handler_check(flows: list): def environment_requirement_check(flows: list): - from prefect import environments - - # Tests for imports that are required by certain environments + # Test for imports that are required by certain environments for flow in flows: - if isinstance( - flow.environment, environments.DaskKubernetesEnvironment - ) or isinstance(flow.environment, environments.KubernetesJobEnvironment): - try: - import kubernetes - except ModuleNotFoundError: - raise ModuleNotFoundError( - "Using {} requires the `kubernetes` dependency.".format( - flow.environment.__class__.__name__ - ) - ) - elif isinstance(flow.environment, environments.FargateTaskEnvironment): + # Load all required dependencies for an environment + required_imports = flow.environment.dependencies if flow.environment else [] + for requirement in required_imports: try: - import boto3 - import botocore + importlib.import_module(requirement) except ModuleNotFoundError: raise ModuleNotFoundError( - "Using {} requires both `boto3` and `botocore` dependencies.".format( - flow.environment.__class__.__name__ + "Using {} requires the `{}` dependency".format( + flow.environment.__class__.__name__, requirement ) ) diff --git a/tests/environments/execution/test_base_environment.py b/tests/environments/execution/test_base_environment.py index cdf3b7f649bc..162671b87933 100644 --- a/tests/environments/execution/test_base_environment.py +++ b/tests/environments/execution/test_base_environment.py @@ -29,6 +29,11 @@ def f(): assert environment.on_exit is f +def test_environment_dependencies(): + environment = Environment() + assert environment.dependencies == [] + + def test_setup_environment_passes(): environment = Environment() environment.setup(storage=Docker()) diff --git a/tests/environments/execution/test_dask_k8s_environment.py b/tests/environments/execution/test_dask_k8s_environment.py index bb2820e9e95d..49af5e46fbcf 100644 --- a/tests/environments/execution/test_dask_k8s_environment.py +++ b/tests/environments/execution/test_dask_k8s_environment.py @@ -40,6 +40,11 @@ def f(): assert environment.on_exit is f +def test_dask_environment_dependencies(): + environment = DaskKubernetesEnvironment() + assert environment.dependencies == ["kubernetes"] + + def test_create_dask_environment_identifier_label(): environment = DaskKubernetesEnvironment() assert environment.identifier_label diff --git a/tests/environments/execution/test_fargate_task_environment.py b/tests/environments/execution/test_fargate_task_environment.py index 6733dc191f11..5ab6a20a4313 100644 --- a/tests/environments/execution/test_fargate_task_environment.py +++ b/tests/environments/execution/test_fargate_task_environment.py @@ -38,6 +38,11 @@ def f(): assert environment.on_exit is f +def test_fargate_task_environment_dependencies(): + environment = FargateTaskEnvironment() + assert environment.dependencies == ["boto3", "botocore"] + + def test_create_fargate_task_environment_aws_creds_provided(): environment = FargateTaskEnvironment( labels=["foo"], diff --git a/tests/environments/execution/test_k8s_job_environment.py b/tests/environments/execution/test_k8s_job_environment.py index 83803e433bf7..f5aaf8b2e946 100644 --- a/tests/environments/execution/test_k8s_job_environment.py +++ b/tests/environments/execution/test_k8s_job_environment.py @@ -63,6 +63,11 @@ def f(): assert environment.on_exit is f +def test_k8s_job_environment_dependencies(): + environment = KubernetesJobEnvironment() + assert environment.dependencies == ["kubernetes"] + + def test_create_k8s_job_environment_identifier_label(): with tempfile.TemporaryDirectory() as directory: diff --git a/tests/environments/execution/test_local_environment.py b/tests/environments/execution/test_local_environment.py index b6e44a303625..f0deb62cfa52 100644 --- a/tests/environments/execution/test_local_environment.py +++ b/tests/environments/execution/test_local_environment.py @@ -28,6 +28,11 @@ def f(): assert environment.logger.name == "prefect.LocalEnvironment" +def test_environment_dependencies(): + environment = LocalEnvironment() + assert environment.dependencies == [] + + def test_setup_environment_passes(): environment = LocalEnvironment() environment.setup(storage=Docker()) diff --git a/tests/environments/execution/test_remote_environment.py b/tests/environments/execution/test_remote_environment.py index c396e8c1ea01..c6d6c5ff21ca 100644 --- a/tests/environments/execution/test_remote_environment.py +++ b/tests/environments/execution/test_remote_environment.py @@ -40,6 +40,11 @@ def f(): assert environment.on_exit is f +def test_remote_environment_dependencies(): + environment = RemoteEnvironment() + assert environment.dependencies == [] + + def test_environment_execute(): with tempfile.TemporaryDirectory() as directory: From 6cb80723f99603bfec0b02c36b08df918e4d7501 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:10:48 -0400 Subject: [PATCH 05/13] Remove `egg` from Docker storage prefect install --- src/prefect/environments/storage/docker.py | 2 +- tests/environments/storage/test_docker_storage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py index 2a7165ad8c17..efd14cd106c6 100644 --- a/src/prefect/environments/storage/docker.py +++ b/src/prefect/environments/storage/docker.py @@ -108,7 +108,7 @@ def __init__( self.extra_commands.extend( [ "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", - "pip install git+https://github.com/PrefectHQ/prefect.git@{}#egg=prefect".format( + "pip install git+https://github.com/PrefectHQ/prefect.git@{}".format( self.prefect_version ), ] diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py index d3763afce1e5..138d041c2ffe 100644 --- a/tests/environments/storage/test_docker_storage.py +++ b/tests/environments/storage/test_docker_storage.py @@ -317,7 +317,7 @@ def test_create_dockerfile_from_base_image(): ( "FROM python:3.6-slim", "apt update && apt install -y gcc git && rm -rf /var/lib/apt/lists/*", - "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510#egg=prefect", + "pip install git+https://github.com/PrefectHQ/prefect.git@424be6b5ed8d3be85064de4b95b5c3d7cb665510", ), ), ], From 69b72822af1304547aa8140701179a060afd9b5d Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:38:59 -0400 Subject: [PATCH 06/13] Add tests for environment dependency health check --- .../environments/storage/_healthcheck.py | 12 +++---- .../storage/test_docker_healthcheck.py | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/prefect/environments/storage/_healthcheck.py b/src/prefect/environments/storage/_healthcheck.py index ab459baf449f..130f76ace169 100644 --- a/src/prefect/environments/storage/_healthcheck.py +++ b/src/prefect/environments/storage/_healthcheck.py @@ -94,22 +94,22 @@ def result_handler_check(flows: list): print("Result Handler check: OK") -def environment_requirement_check(flows: list): +def environment_dependency_check(flows: list): # Test for imports that are required by certain environments for flow in flows: # Load all required dependencies for an environment required_imports = flow.environment.dependencies if flow.environment else [] - for requirement in required_imports: + for dependency in required_imports: try: - importlib.import_module(requirement) + importlib.import_module(dependency) except ModuleNotFoundError: raise ModuleNotFoundError( "Using {} requires the `{}` dependency".format( - flow.environment.__class__.__name__, requirement + flow.environment.__class__.__name__, dependency ) ) - print("Environment requirement check: OK") + print("Environment dependency check: OK") if __name__ == "__main__": @@ -119,5 +119,5 @@ def environment_requirement_check(flows: list): system_check(python_version) flows = cloudpickle_deserialization_check(flow_file_path) result_handler_check(flows) - environment_requirement_check(flows) + environment_dependency_check(flows) print("All health checks passed.") diff --git a/tests/environments/storage/test_docker_healthcheck.py b/tests/environments/storage/test_docker_healthcheck.py index 989bfcef5e3f..16d9b957a56d 100644 --- a/tests/environments/storage/test_docker_healthcheck.py +++ b/tests/environments/storage/test_docker_healthcheck.py @@ -2,11 +2,13 @@ import os import sys import tempfile +from unittest.mock import MagicMock import cloudpickle import pytest from prefect import Flow, Task, task +from prefect.environments import Environment, RemoteEnvironment from prefect.environments.storage import _healthcheck as healthchecks @@ -154,3 +156,36 @@ def down(): result = down(upstream_tasks=[up]) assert healthchecks.result_handler_check([f]) is None + + +class TestEnvironmentDependencyCheck: + def test_no_raise_on_normal_flow(self): + flow = Flow("THIS IS A TEST") + + assert healthchecks.environment_dependency_check([flow]) is None + + def test_no_raise_on_remote_env(self): + flow = Flow("THIS IS A TEST", environment=RemoteEnvironment()) + + assert healthchecks.environment_dependency_check([flow]) is None + + def test_no_raise_on_proper_imports(self): + class NewEnvironment(Environment): + @property + def dependencies(self) -> list: + return ["prefect"] + + flow = Flow("THIS IS A TEST", environment=NewEnvironment()) + + assert healthchecks.environment_dependency_check([flow]) is None + + def test_raise_on_missing_imports(self, monkeypatch): + class NewEnvironment(Environment): + @property + def dependencies(self) -> list: + return ["TEST"] + + flow = Flow("THIS IS A TEST", environment=NewEnvironment()) + + with pytest.raises(ModuleNotFoundError): + healthchecks.environment_dependency_check([flow]) From 41b4926f53bf013a77f25c2b9aca5bbed2957c29 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:40:47 -0400 Subject: [PATCH 07/13] Update CHANGELOG --- CHANGELOG.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 221fcaeab6a6..7b8cfcee5be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,12 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Features -- None +- Environments and Agents now support labeling for execution specification - [#1651](https://github.com/PrefectHQ/prefect/pull/1651) ### Enhancements - Add the ability to delete task tag limits using the client - [#1622](https://github.com/PrefectHQ/prefect/pull/1622) -- Adds an "Ask for help" button with a link to the prefect.io support page - [#1637](https://github.com/PrefectHQ/prefect/pull/1637) -- Reduces the size of the `prefecthq/prefect` Docker image by ~400MB, which is now the base Docker image used in Flows - [#1648](https://github.com/PrefectHQ/prefect/pull/1648) +- Add a new healthcheck for environment dependencies - [#1653](https://github.com/PrefectHQ/prefect/pull/1653) ### Task Library @@ -21,7 +20,6 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Fixes - Fix defaults for unspecified ARNs in the Fargate Agent - [#1634](https://github.com/PrefectHQ/prefect/pull/1634) -- Fix ShellTask return value on empty stdout - [#1632](https://github.com/PrefectHQ/prefect/pull/1632) ### Deprecations @@ -29,11 +27,11 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Breaking Changes -- None +- Fargate Agent now takes in all boto3 camel case arguments instead of specific snake case options - [#1649](https://github.com/PrefectHQ/prefect/pull/1649) ### Contributors -- [Tobias Schmidt](https://github.com/royalts) +- None ## 0.6.7 From 0a68f725a19b11b9904775873d6d3c3e074867aa Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:43:04 -0400 Subject: [PATCH 08/13] Update CHANGELOG --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8cfcee5be4..a8c49b46cff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Enhancements - Add the ability to delete task tag limits using the client - [#1622](https://github.com/PrefectHQ/prefect/pull/1622) +- Adds an "Ask for help" button with a link to the prefect.io support page - [#1637](https://github.com/PrefectHQ/prefect/pull/1637) +- Reduces the size of the `prefecthq/prefect` Docker image by ~400MB, which is now the base Docker image used in Flows - [#1648](https://github.com/PrefectHQ/prefect/pull/1648) - Add a new healthcheck for environment dependencies - [#1653](https://github.com/PrefectHQ/prefect/pull/1653) ### Task Library @@ -20,6 +22,7 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Fixes - Fix defaults for unspecified ARNs in the Fargate Agent - [#1634](https://github.com/PrefectHQ/prefect/pull/1634) +- Fix ShellTask return value on empty stdout - [#1632](https://github.com/PrefectHQ/prefect/pull/1632) ### Deprecations @@ -27,11 +30,11 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Breaking Changes -- Fargate Agent now takes in all boto3 camel case arguments instead of specific snake case options - [#1649](https://github.com/PrefectHQ/prefect/pull/1649) +- None ### Contributors -- None +- [Tobias Schmidt](https://github.com/royalts) ## 0.6.7 From 6fdedf2560b2f1d371516dd25512ca75414cf1d7 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:43:31 -0400 Subject: [PATCH 09/13] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8c49b46cff4..99642b4da949 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Features -- Environments and Agents now support labeling for execution specification - [#1651](https://github.com/PrefectHQ/prefect/pull/1651) +- None ### Enhancements From b307a18a1cdfbe9b30834c0915f79e3d7a3358d9 Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:50:46 -0400 Subject: [PATCH 10/13] Add test for missing dependencies property and update healthcheck --- src/prefect/environments/storage/_healthcheck.py | 5 ++++- tests/environments/storage/test_docker_healthcheck.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/prefect/environments/storage/_healthcheck.py b/src/prefect/environments/storage/_healthcheck.py index 130f76ace169..b737bb13f900 100644 --- a/src/prefect/environments/storage/_healthcheck.py +++ b/src/prefect/environments/storage/_healthcheck.py @@ -98,7 +98,10 @@ def environment_dependency_check(flows: list): # Test for imports that are required by certain environments for flow in flows: # Load all required dependencies for an environment - required_imports = flow.environment.dependencies if flow.environment else [] + if not hasattr(flow.environment, "dependencies"): + continue + + required_imports = flow.environment.dependencies for dependency in required_imports: try: importlib.import_module(dependency) diff --git a/tests/environments/storage/test_docker_healthcheck.py b/tests/environments/storage/test_docker_healthcheck.py index 16d9b957a56d..d7b6f6d6951b 100644 --- a/tests/environments/storage/test_docker_healthcheck.py +++ b/tests/environments/storage/test_docker_healthcheck.py @@ -179,6 +179,14 @@ def dependencies(self) -> list: assert healthchecks.environment_dependency_check([flow]) is None + def test_no_raise_on_missing_dependencies_property(self): + class NewEnvironment(Environment): + pass + + flow = Flow("THIS IS A TEST", environment=NewEnvironment()) + + assert healthchecks.environment_dependency_check([flow]) is None + def test_raise_on_missing_imports(self, monkeypatch): class NewEnvironment(Environment): @property From 4b2aa38d32e6942e011bf24026c85308f6f5f23a Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 13:54:36 -0400 Subject: [PATCH 11/13] Skip missing import test on python version < 3.6 --- tests/environments/storage/test_docker_healthcheck.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/environments/storage/test_docker_healthcheck.py b/tests/environments/storage/test_docker_healthcheck.py index d7b6f6d6951b..b38d9752ba56 100644 --- a/tests/environments/storage/test_docker_healthcheck.py +++ b/tests/environments/storage/test_docker_healthcheck.py @@ -187,6 +187,9 @@ class NewEnvironment(Environment): assert healthchecks.environment_dependency_check([flow]) is None + @pytest.mark.skipif( + sys.version_info < (3, 6), reason="3.5 does not support ModuleNotFoundError" + ) def test_raise_on_missing_imports(self, monkeypatch): class NewEnvironment(Environment): @property From fc71da2abd5c2b836d4566900abc56b98b33342c Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 16:05:33 -0400 Subject: [PATCH 12/13] Update CHANGELOG --- CHANGELOG.md | 2 +- src/prefect/agent/agent.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99642b4da949..b805fca36570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/ ### Breaking Changes -- None +- `kubernetes` is no longer installed by default in deployed flow images - [#1653](https://github.com/PrefectHQ/prefect/pull/1653) ### Contributors diff --git a/src/prefect/agent/agent.py b/src/prefect/agent/agent.py index e19b12015b34..910e4dfaa657 100644 --- a/src/prefect/agent/agent.py +++ b/src/prefect/agent/agent.py @@ -153,8 +153,8 @@ def agent_process(self, tenant_id: str) -> bool: ) ) - self.update_states(flow_runs) - self.deploy_flows(flow_runs) + # self.update_states(flow_runs) + # self.deploy_flows(flow_runs) self.logger.info( "Submitted {} flow run(s) for execution.".format(len(flow_runs)) ) From a4c8707611017287d375637a467d0b488654863d Mon Sep 17 00:00:00 2001 From: Josh Meek Date: Wed, 23 Oct 2019 16:19:12 -0400 Subject: [PATCH 13/13] Revert commented out debugging related agent functions --- src/prefect/agent/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prefect/agent/agent.py b/src/prefect/agent/agent.py index 910e4dfaa657..e19b12015b34 100644 --- a/src/prefect/agent/agent.py +++ b/src/prefect/agent/agent.py @@ -153,8 +153,8 @@ def agent_process(self, tenant_id: str) -> bool: ) ) - # self.update_states(flow_runs) - # self.deploy_flows(flow_runs) + self.update_states(flow_runs) + self.deploy_flows(flow_runs) self.logger.info( "Submitted {} flow run(s) for execution.".format(len(flow_runs)) )