diff --git a/metaflow/cli.py b/metaflow/cli.py index 8f316d326e9..e465af05a75 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -3,40 +3,16 @@ import traceback from datetime import datetime from functools import wraps -import metaflow.tracing as tracing +import metaflow.tracing as tracing from metaflow._vendor import click -from . import lint -from . import plugins -from . import parameters -from . import decorators -from . import metaflow_version -from . import namespace -from .metaflow_current import current -from .client.core import get_metadata +from . import decorators, lint, metaflow_version, namespace, parameters, plugins from .cli_args import cli_args -from .tagging_util import validate_tags -from .util import ( - resolve_identity, - decompress_list, - write_latest_run_id, - get_latest_run_id, -) -from .task import MetaflowTask +from .client.core import get_metadata +from .datastore import FlowDataStore, TaskDataStore, TaskDataStoreSet from .exception import CommandException, MetaflowException from .graph import FlowGraph -from .datastore import FlowDataStore, TaskDataStoreSet, TaskDataStore - -from .runtime import NativeRuntime -from .package import MetaflowPackage -from .plugins import ( - DATASTORES, - ENVIRONMENTS, - LOGGING_SIDECARS, - METADATA_PROVIDERS, - MONITOR_SIDECARS, -) from .metaflow_config import ( DEFAULT_DATASTORE, DEFAULT_ENVIRONMENT, @@ -45,12 +21,29 @@ DEFAULT_MONITOR, DEFAULT_PACKAGE_SUFFIXES, ) +from .metaflow_current import current from .metaflow_environment import MetaflowEnvironment +from .mflog import LOG_SOURCES, mflog +from .package import MetaflowPackage +from .plugins import ( + DATASTORES, + ENVIRONMENTS, + LOGGING_SIDECARS, + METADATA_PROVIDERS, + MONITOR_SIDECARS, +) from .pylint_wrapper import PyLint -from .R import use_r, metaflow_r_version -from .mflog import mflog, LOG_SOURCES +from .R import metaflow_r_version, use_r +from .runtime import NativeRuntime +from .tagging_util import validate_tags +from .task import MetaflowTask from .unbounded_foreach import UBF_CONTROL, UBF_TASK - +from .util import ( + decompress_list, + get_latest_run_id, + resolve_identity, + write_latest_run_id, +) ERASE_TO_EOL = "\033[K" HIGHLIGHT = "red" diff --git a/metaflow/runner/click_api.py b/metaflow/runner/click_api.py index d0367046f22..9757c2bbf55 100644 --- a/metaflow/runner/click_api.py +++ b/metaflow/runner/click_api.py @@ -7,38 +7,34 @@ """ ) -import inspect +import datetime import importlib +import inspect import itertools +import uuid from collections import OrderedDict -import uuid, datetime -from typing import ( - Optional, - List, - OrderedDict as TOrderedDict, - Any, - Union, - Dict, - Callable, -) +from typing import Any, Callable, Dict, List, Optional +from typing import OrderedDict as TOrderedDict +from typing import Union + from metaflow import FlowSpec, Parameter -from metaflow.cli import start from metaflow._vendor import click -from metaflow.parameters import JSONTypeClass -from metaflow.includefile import FilePathClass -from metaflow._vendor.typeguard import check_type, TypeCheckError from metaflow._vendor.click.types import ( - StringParamType, - IntParamType, - FloatParamType, BoolParamType, - UUIDParameterType, - Path, - DateTime, - Tuple, Choice, + DateTime, File, + FloatParamType, + IntParamType, + Path, + StringParamType, + Tuple, + UUIDParameterType, ) +from metaflow._vendor.typeguard import TypeCheckError, check_type +from metaflow.cli import start +from metaflow.includefile import FilePathClass +from metaflow.parameters import JSONTypeClass click_to_python_types = { StringParamType: str, diff --git a/metaflow/runner/metaflow_runner.py b/metaflow/runner/metaflow_runner.py index 3ea1fc15868..cdc9da77c6c 100644 --- a/metaflow/runner/metaflow_runner.py +++ b/metaflow/runner/metaflow_runner.py @@ -1,10 +1,12 @@ import os import sys -import time import tempfile +import time from typing import Dict, Iterator, Optional, Tuple + from metaflow import Run, metadata -from .subprocess_manager import SubprocessManager, CommandManager + +from .subprocess_manager import CommandManager, SubprocessManager def clear_and_set_os_environ(env: Dict): @@ -247,7 +249,6 @@ def __enter__(self) -> "Runner": async def __aenter__(self) -> "Runner": return self - def __get_executing_run(self, tfp_runner_attribute, command_obj): # When two 'Runner' executions are done sequentially i.e. one after the other # the 2nd run kinda uses the 1st run's previously set metadata and diff --git a/metaflow/runner/nbrun.py b/metaflow/runner/nbrun.py index fde1a2eafe5..e7327a16437 100644 --- a/metaflow/runner/nbrun.py +++ b/metaflow/runner/nbrun.py @@ -1,8 +1,9 @@ -import os import ast +import os import shutil import tempfile -from typing import Optional, Dict +from typing import Dict, Optional + from metaflow import Runner try: @@ -50,7 +51,7 @@ def __init__( profile: Optional[str] = None, env: Optional[Dict] = None, base_dir: Optional[str] = None, - **kwargs + **kwargs, ): self.cell = get_current_cell() self.flow = flow @@ -88,7 +89,7 @@ def __init__( flow_file=self.tmp_flow_file.name, profile=profile, env=self.env_vars, - **kwargs + **kwargs, ) def nbrun(self, **kwargs): diff --git a/metaflow/runner/subprocess_manager.py b/metaflow/runner/subprocess_manager.py index 84a6083f6c6..7e67851c4dd 100644 --- a/metaflow/runner/subprocess_manager.py +++ b/metaflow/runner/subprocess_manager.py @@ -1,13 +1,13 @@ +import asyncio import os -import sys -import time -import signal import shutil -import asyncio +import signal +import subprocess +import sys import tempfile import threading -import subprocess -from typing import List, Dict, Optional, Callable, Iterator, Tuple +import time +from typing import Callable, Dict, Iterator, List, Optional, Tuple def kill_process_and_descendants(pid, termination_timeout): diff --git a/test/core/run_tests.py b/test/core/run_tests.py index c80243aa2b1..f8aa0c0a9e5 100644 --- a/test/core/run_tests.py +++ b/test/core/run_tests.py @@ -1,27 +1,27 @@ -import sys -import os -import json -import uuid import glob import importlib -import tempfile +import json +import os import shutil import subprocess +import sys +import tempfile +import uuid from multiprocessing import Pool -from metaflow.cli import start, run from metaflow._vendor import click +from metaflow.cli import run, start skip_api_executor = False try: + from metaflow import Runner from metaflow.runner.click_api import ( MetaflowAPI, - extract_all_params, click_to_python_types, + extract_all_params, ) - from metaflow import Runner -except RuntimeError: +except ImportError: skip_api_executor = True from metaflow_test import MetaflowTest