Skip to content

Commit

Permalink
Add --[no-]install option to all affected commands and reuse for add
Browse files Browse the repository at this point in the history
  • Loading branch information
ReubenFrankel committed Apr 25, 2024
1 parent d390810 commit 55d4585
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 41 deletions.
29 changes: 12 additions & 17 deletions src/meltano/cli/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
import click
import requests

from meltano.cli.params import pass_project
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import (
CliError,
PartialInstrumentedCmd,
add_plugin,
add_required_plugins,
check_dependencies_met,
install_plugins,
)
from meltano.core.plugin import PluginRef, PluginType
from meltano.core.plugin_install_service import PluginInstallReason
Expand Down Expand Up @@ -103,11 +102,7 @@ def _load_yaml_from_ref(_ctx, _param, value: str | None) -> dict | None:
is_flag=True,
help="Update an existing plugin.",
)
@click.option(
"--no-install",
is_flag=True,
help="Do not install the plugin after adding it to the project.",
)
@install_option
@click.option(
"--force-install",
is_flag=True,
Expand All @@ -121,6 +116,7 @@ async def add( # noqa: C901 WPS238
project: Project,
plugin_type: str,
plugin_name: str,
install_plugins: InstallPlugins,
inherit_from: str | None = None,
variant: str | None = None,
as_name: str | None = None,
Expand Down Expand Up @@ -199,17 +195,16 @@ async def add( # noqa: C901 WPS238
)
tracker.track_command_event(CliEvent.inflight)

if not flags.get("no_install"):
success = await install_plugins(
project,
plugins,
reason=PluginInstallReason.ADD,
force=flags.get("force_install", False),
)
success = await install_plugins(
project,
plugins,
reason=PluginInstallReason.ADD,
force=flags.get("force_install", False),
)

if not success:
tracker.track_command_event(CliEvent.failed)
raise CliError("Failed to install plugin(s)") # noqa: EM101
if success is False:
tracker.track_command_event(CliEvent.failed)
raise CliError("Failed to install plugin(s)") # noqa: EM101

_print_plugins(plugins)
tracker.track_command_event(CliEvent.completed)
Expand Down
5 changes: 3 additions & 2 deletions src/meltano/cli/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,12 @@
import structlog

from meltano.cli.interactive import InteractiveConfig
from meltano.cli.params import pass_project
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import (
CliEnvironmentBehavior,
CliError,
InstrumentedGroup,
PartialInstrumentedCmd,
install_plugins,
)
from meltano.core.db import project_engine
from meltano.core.plugin import PluginType
Expand Down Expand Up @@ -397,10 +396,12 @@ def set_(
@config.command(cls=PartialInstrumentedCmd, name="test")
@pass_project(migrate=True)
@click.pass_context
@install_option
@run_async
async def test(
ctx,
project: Project,
install_plugins: InstallPlugins,
):
"""Test the configuration of a plugin."""
invoker = ctx.obj["invoker"]
Expand Down
39 changes: 29 additions & 10 deletions src/meltano/cli/elt.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,8 @@
import click
from structlog import stdlib as structlog_stdlib

from meltano.cli.params import pass_project
from meltano.cli.utils import (
CliEnvironmentBehavior,
CliError,
PartialInstrumentedCmd,
install_plugins,
)
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import CliEnvironmentBehavior, CliError, PartialInstrumentedCmd
from meltano.core.db import project_engine
from meltano.core.elt_context import ELTContextBuilder
from meltano.core.job import Job, JobFinder
Expand Down Expand Up @@ -122,6 +117,7 @@ class ELOptions:
@ELOptions.state_id
@ELOptions.force
@ELOptions.merge_state
@install_option
@click.pass_context
@pass_project(migrate=True)
@run_async
Expand All @@ -140,6 +136,7 @@ async def el( # WPS408
state_id: str,
force: bool,
merge_state: bool,
install_plugins: InstallPlugins,
):
"""
Run an EL pipeline to Extract and Load data.
Expand Down Expand Up @@ -167,6 +164,7 @@ async def el( # WPS408
state_id,
force,
merge_state,
install_plugins,
)


Expand All @@ -188,6 +186,7 @@ async def el( # WPS408
@ELOptions.state_id
@ELOptions.force
@ELOptions.merge_state
@install_option
@click.pass_context
@pass_project(migrate=True)
@run_async
Expand All @@ -207,6 +206,7 @@ async def elt( # WPS408
state_id: str,
force: bool,
merge_state: bool,
install_plugins: InstallPlugins,
):
"""
Run an ELT pipeline to Extract, Load, and Transform data.
Expand Down Expand Up @@ -235,6 +235,7 @@ async def elt( # WPS408
state_id,
force,
merge_state,
install_plugins,
)


Expand All @@ -254,6 +255,7 @@ async def _run_el_command(
state_id: str,
force: bool,
merge_state: bool,
install_plugins: InstallPlugins,
):
if platform.system() == "Windows":
raise CliError(
Expand Down Expand Up @@ -300,7 +302,15 @@ async def _run_el_command(
if dump:
await dump_file(context_builder, dump)
else:
await _run_job(tracker, project, job, session, context_builder, force=force)
await _run_job(
tracker,
project,
job,
session,
context_builder,
install_plugins,
force=force,
)
except Exception as err:
tracker.track_command_event(CliEvent.failed)
raise err
Expand Down Expand Up @@ -364,7 +374,15 @@ async def dump_file(context_builder, dumpable):
raise CliError(f"Could not dump {dumpable}: {err}") from err # noqa: EM102


async def _run_job(tracker, project, job, session, context_builder, force=False):
async def _run_job(
tracker,
project,
job,
session,
context_builder,
install_plugins: InstallPlugins,
force=False,
):
fail_stale_jobs(session, job.job_name)

if not force and (existing := JobFinder(job.job_name).latest_running(session)):
Expand All @@ -383,7 +401,7 @@ async def _run_job(tracker, project, job, session, context_builder, force=False)

log = logger.bind(name="meltano", run_id=str(job.run_id), state_id=job.job_name)

await _run_elt(tracker, log, context_builder, output_logger)
await _run_elt(tracker, log, context_builder, output_logger, install_plugins)


@asynccontextmanager
Expand Down Expand Up @@ -411,6 +429,7 @@ async def _run_elt(
log: structlog.BoundLogger,
context_builder: ELTContextBuilder,
output_logger: OutputLogger,
install_plugins: InstallPlugins,
):
elt_context = context_builder.context()
plugins = [elt_context.extractor, elt_context.loader]
Expand Down
5 changes: 3 additions & 2 deletions src/meltano/cli/invoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@
import click
import structlog

from meltano.cli.params import pass_project
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import (
CliEnvironmentBehavior,
CliError,
PartialInstrumentedCmd,
install_plugins,
propagate_stop_signals,
)
from meltano.core.db import project_engine
Expand Down Expand Up @@ -74,6 +73,7 @@
is_flag=True,
help="Execute plugins using containers where possible.",
)
@install_option
@click.pass_context
@pass_project(migrate=True)
@run_async
Expand All @@ -85,6 +85,7 @@ async def invoke( # noqa: C901
list_commands: bool,
plugin_name: str,
plugin_args: tuple[str, ...],
install_plugins: InstallPlugins,
containers: bool = False,
print_var: str | None = None,
):
Expand Down
20 changes: 19 additions & 1 deletion src/meltano/cli/params.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,21 @@
from __future__ import annotations

import functools
import typing as t

import click

from meltano.cli.utils import CliError
from meltano.cli.utils import CliError, install_plugins
from meltano.core.db import project_engine
from meltano.core.migration_service import MigrationError
from meltano.core.project_settings_service import ProjectSettingsService
from meltano.core.utils import async_noop

InstallPlugins = t.Callable[..., t.Coroutine[t.Any, t.Any, bool]]


def _install_plugins_fn(_ctx, _param, value: bool) -> InstallPlugins:
return install_plugins if value else async_noop


def database_uri_option(func):
Expand All @@ -29,6 +37,16 @@ def decorate(*args, database_uri=None, **kwargs):
return functools.update_wrapper(decorate, func)


install_option = click.option(
"--install/--no-install",
"install_plugins",
default=True,
show_default=True,
callback=_install_plugins_fn,
help="Whether or not to install the subject plugin(s) automatically.",
)


class pass_project: # noqa: N801
"""Pass current project to decorated CLI command function."""

Expand Down
11 changes: 4 additions & 7 deletions src/meltano/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,8 @@
import click
import structlog

from meltano.cli.params import pass_project
from meltano.cli.utils import (
CliEnvironmentBehavior,
CliError,
PartialInstrumentedCmd,
install_plugins,
)
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import CliEnvironmentBehavior, CliError, PartialInstrumentedCmd
from meltano.core.block.blockset import BlockSet
from meltano.core.block.parser import BlockParser, validate_block_sets
from meltano.core.block.plugin_command import PluginCommandBlock
Expand Down Expand Up @@ -99,6 +94,7 @@ def convert(self, value, param, ctx):
"blocks",
nargs=-1,
)
@install_option
@pass_project(migrate=True)
@click.pass_context
@run_async
Expand All @@ -113,6 +109,7 @@ async def run(
merge_state: bool,
run_id: uuid.UUID | None,
blocks: list[str],
install_plugins: InstallPlugins,
):
"""
Run a set of command blocks in series.
Expand Down
5 changes: 3 additions & 2 deletions src/meltano/cli/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,10 @@
import click
import structlog

from meltano.cli.params import pass_project
from meltano.cli.params import InstallPlugins, install_option, pass_project
from meltano.cli.utils import (
CliEnvironmentBehavior,
InstrumentedCmd,
install_plugins,
propagate_stop_signals,
)
from meltano.core.db import project_engine
Expand Down Expand Up @@ -85,11 +84,13 @@ async def run_test(self, name: str) -> int:
required=False,
nargs=-1,
)
@install_option
@pass_project(migrate=True)
@run_async
async def test(
project: Project,
all_tests: bool,
install_plugins: InstallPlugins,
plugin_tests: tuple[str, ...] = (),
):
"""
Expand Down
4 changes: 4 additions & 0 deletions src/meltano/core/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,10 @@ def noop(*_args, **_kwargs):
pass


async def async_noop(*_args, **_kwargs):
pass


def truthy(val: str) -> bool:
return str(val).lower() in TRUTHY

Expand Down

0 comments on commit 55d4585

Please sign in to comment.