diff --git a/src/python/pants/core/goals/deploy.py b/src/python/pants/core/goals/deploy.py new file mode 100644 index 00000000000..218b9370008 --- /dev/null +++ b/src/python/pants/core/goals/deploy.py @@ -0,0 +1,215 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from abc import ABCMeta +from dataclasses import dataclass +from itertools import chain +from typing import Iterable + +from pants.core.goals.package import PackageFieldSet +from pants.core.goals.publish import PublishFieldSet, PublishProcesses, PublishProcessesRequest +from pants.engine.console import Console +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.process import InteractiveProcess, InteractiveProcessResult +from pants.engine.rules import Effect, Get, MultiGet, collect_rules, goal_rule, rule_helper +from pants.engine.target import ( + FieldSet, + FieldSetsPerTarget, + FieldSetsPerTargetRequest, + NoApplicableTargetsBehavior, + Target, + TargetRootsToFieldSets, + TargetRootsToFieldSetsRequest, +) +from pants.engine.unions import union + +logger = logging.getLogger(__name__) + + +@union +@dataclass(frozen=True) +class DeployFieldSet(FieldSet, metaclass=ABCMeta): + """The FieldSet type for the `deploy` goal. + + Union members may list any fields required to fulfill the instantiation of the `DeployProcess` + result of the deploy rule. + """ + + +@dataclass(frozen=True) +class DeployProcess: + """A process that when executed will have the side effect of deploying a target. + + To provide with the ability to deploy a given target, create a custom `DeployFieldSet` for + that given target and implement a rule that returns `DeployProcess` for that custom field set: + + Example: + + @dataclass(frozen=True) + class MyDeploymentFieldSet(DeployFieldSet): + pass + + @rule + async def my_deployment_process(field_set: MyDeploymentFieldSet) -> DeployProcess: + # Create the underlying process that executes the deployment + process = Process(...) + return DeployProcess( + name="my_deployment", + process=InteractiveProcess.from_process(process) + ) + + def rules(): + return [ + *collect_rules(), + UnionRule(DeployFieldSet, MyDeploymentFieldSet) + ] + + Use the `publish_dependencies` field to provide with a list of targets that produce packages + which need to be externally published before the deployment process is executed. + """ + + name: str + process: InteractiveProcess | None + publish_dependencies: tuple[Target, ...] = () + description: str | None = None + + +class DeploySubsystem(GoalSubsystem): + name = "experimental-deploy" + help = "Perform a deployment process." + + required_union_implementation = (DeployFieldSet,) + + +@dataclass(frozen=True) +class Deploy(Goal): + subsystem_cls = DeploySubsystem + + +@rule_helper +async def _find_publish_processes(targets: Iterable[Target]) -> PublishProcesses: + package_field_sets, publish_field_sets = await MultiGet( + Get(FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, targets)), + Get(FieldSetsPerTarget, FieldSetsPerTargetRequest(PublishFieldSet, targets)), + ) + + return await Get( + PublishProcesses, + PublishProcessesRequest( + package_field_sets=package_field_sets.field_sets, + publish_field_sets=publish_field_sets.field_sets, + ), + ) + + +@rule_helper +async def _invoke_process( + console: Console, + process: InteractiveProcess | None, + *, + names: Iterable[str], + success_status: str, + description: str | None = None, +) -> tuple[int, tuple[str, ...]]: + results = [] + + if not process: + sigil = console.sigil_skipped() + status = "skipped" + if description: + status += f" {description}" + for name in names: + results.append(f"{sigil} {name} {status}.") + return 0, tuple(results) + + logger.debug(f"Execute {process}") + res = await Effect(InteractiveProcessResult, InteractiveProcess, process) + if res.exit_code == 0: + sigil = console.sigil_succeeded() + status = success_status + prep = "to" + else: + sigil = console.sigil_failed() + status = "failed" + prep = "for" + + if description: + status += f" {prep} {description}" + + for name in names: + results.append(f"{sigil} {name} {status}") + + return res.exit_code, tuple(results) + + +@goal_rule +async def run_deploy(console: Console, deploy_subsystem: DeploySubsystem) -> Deploy: + target_roots_to_deploy_field_sets = await Get( + TargetRootsToFieldSets, + TargetRootsToFieldSetsRequest( + DeployFieldSet, + goal_description=f"the `{deploy_subsystem.name}` goal", + no_applicable_targets_behavior=NoApplicableTargetsBehavior.error, + ), + ) + + deploy_processes = await MultiGet( + Get(DeployProcess, DeployFieldSet, field_set) + for field_set in target_roots_to_deploy_field_sets.field_sets + ) + + publish_targets = set( + chain.from_iterable([deploy.publish_dependencies for deploy in deploy_processes]) + ) + publish_processes = await _find_publish_processes(publish_targets) + + exit_code: int = 0 + results: list[str] = [] + + if publish_processes: + logger.info("Publishing dependencies...") + + # Publish all deployment dependencies first. + for publish in publish_processes: + ec, statuses = await _invoke_process( + console, + publish.process, + names=publish.names, + description=publish.description, + success_status="published", + ) + exit_code = ec if ec != 0 else exit_code + results.extend(statuses) + + # Only proceed to deploy of all dependencies have been successfully published + if exit_code == 0 and deploy_processes: + logger.info("Deploying targets...") + + for deploy in deploy_processes: + # Invoke the deployment. + ec, statuses = await _invoke_process( + console, + deploy.process, + names=[deploy.name], + success_status="deployed", + description=deploy.description, + ) + exit_code = ec if ec != 0 else exit_code + results.extend(statuses) + + console.print_stderr("") + if not results: + sigil = console.sigil_skipped() + console.print_stderr(f"{sigil} Nothing deployed.") + + for line in results: + console.print_stderr(line) + + return Deploy(exit_code) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/core/goals/deploy_test.py b/src/python/pants/core/goals/deploy_test.py new file mode 100644 index 00000000000..892845bc579 --- /dev/null +++ b/src/python/pants/core/goals/deploy_test.py @@ -0,0 +1,218 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from textwrap import dedent + +import pytest + +from pants.core.goals.deploy import Deploy, DeployFieldSet, DeployProcess +from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet +from pants.core.goals.publish import ( + PublishFieldSet, + PublishPackages, + PublishProcesses, + PublishRequest, +) +from pants.core.register import rules as core_rules +from pants.engine import process +from pants.engine.fs import EMPTY_DIGEST +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.process import InteractiveProcess +from pants.engine.rules import Get, rule +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + DependenciesRequest, + StringField, + StringSequenceField, + Target, + Targets, +) +from pants.engine.unions import UnionRule +from pants.testutil.rule_runner import RuleRunner + + +class MockDestinationField(StringField): + alias = "destination" + + +class MockRepositoriesField(StringSequenceField): + alias = "repositories" + + +class MockDependenciesField(Dependencies): + pass + + +class MockPackageTarget(Target): + alias = "mock_package" + core_fields = ( + *COMMON_TARGET_FIELDS, + MockRepositoriesField, + ) + + +class MockDeployTarget(Target): + alias = "mock_deploy" + core_fields = ( + *COMMON_TARGET_FIELDS, + MockDestinationField, + MockDependenciesField, + ) + + +@dataclass(frozen=True) +class MockPublishRequest(PublishRequest): + pass + + +@dataclass(frozen=True) +class MockPackageFieldSet(PackageFieldSet): + required_fields = (MockRepositoriesField,) + + repositories: MockRepositoriesField + + +@dataclass(frozen=True) +class MockPublishFieldSet(PublishFieldSet): + publish_request_type = MockPublishRequest + required_fields = (MockRepositoriesField,) + + repositories: MockRepositoriesField + + +@dataclass(frozen=True) +class MockDeployFieldSet(DeployFieldSet): + required_fields = (MockDestinationField,) + + destination: MockDestinationField + dependencies: MockDependenciesField + + +@rule +async def mock_package(request: MockPackageFieldSet) -> BuiltPackage: + artifact = BuiltPackageArtifact( + relpath=request.address.path_safe_spec, + extra_log_lines=tuple( + [f"test package into: {repo}" for repo in request.repositories.value] + if request.repositories.value + else [] + ), + ) + return BuiltPackage(digest=EMPTY_DIGEST, artifacts=(artifact,)) + + +@rule +async def mock_publish(request: MockPublishRequest) -> PublishProcesses: + if not request.field_set.repositories.value: + return PublishProcesses() + + return PublishProcesses( + PublishPackages( + names=tuple( + artifact.relpath + for pkg in request.packages + for artifact in pkg.artifacts + if artifact.relpath + ), + process=None if repo == "skip" else InteractiveProcess(["echo", repo]), + description="(requested)" if repo == "skip" else repo, + ) + for repo in request.field_set.repositories.value + ) + + +@rule +async def mock_deploy(field_set: MockDeployFieldSet) -> DeployProcess: + if not field_set.destination.value: + return DeployProcess(name="test-deploy", publish_dependencies=(), process=None) + + dependencies = await Get(Targets, DependenciesRequest(field_set.dependencies)) + dest = field_set.destination.value + return DeployProcess( + name="test-deploy", + publish_dependencies=tuple(dependencies), + description="(requested)" if dest == "skip" else None, + process=None + if dest == "skip" + else InteractiveProcess(["echo", dest], run_in_workspace=True), + ) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *core_rules(), + *process.rules(), + mock_publish, + mock_deploy, + mock_package, + *MockPublishFieldSet.rules(), + UnionRule(PackageFieldSet, MockPackageFieldSet), + UnionRule(DeployFieldSet, MockDeployFieldSet), + ], + target_types=[MockDeployTarget, MockPackageTarget], + ) + + +def test_fail_when_no_deploy_targets_matched(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/BUILD": dedent( + """\ + mock_package(name="foo") + """ + ) + } + ) + + with pytest.raises(ExecutionError, match="No applicable files or targets matched"): + rule_runner.run_goal_rule(Deploy, args=("::",)) + + +def test_skip_deploy(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/BUILD": dedent( + """\ + mock_deploy(name="inst", destination="skip") + """ + ) + } + ) + + result = rule_runner.run_goal_rule(Deploy, args=("src:inst",)) + + assert result.exit_code == 0 + assert "test-deploy skipped (requested)." in result.stderr + + +@pytest.mark.skip("Can not run interactive process from test..?") +@pytest.mark.no_error_if_skipped +def test_mocked_deploy(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/BUILD": dedent( + """\ + mock_package( + name="dependency", + repositories=["https://www.example.com"], + ) + + mock_deploy( + name="main", + dependencies=[":dependency"], + destination="production", + ) + """ + ) + } + ) + + result = rule_runner.run_goal_rule(Deploy, args=("src:main",)) + + assert result.exit_code == 0 + assert "https://www.example.com" in result.stderr + assert "production" in result.stderr diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index 53425a48a01..ed260cdcb0e 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -9,6 +9,7 @@ from pants.build_graph.build_file_aliases import BuildFileAliases from pants.core.goals import ( check, + deploy, export, fmt, generate_lockfiles, @@ -52,6 +53,7 @@ def rules(): return [ # goals *check.rules(), + *deploy.rules(), *export.rules(), *fmt.rules(), *generate_lockfiles.rules(),