From cef2cc2b8d2018bedccaf43f8a288676bfe436f8 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 10:54:20 +0200 Subject: [PATCH 01/28] refactor(install): P0 -- create install engine package skeleton Sets up the structural foundation for decomposing `commands/install.py` (2905 LOC, with one 1444-line function) into a proper engine package with explicit phase boundaries and a typed shared context. Adds: - `src/apm_cli/install/` engine package with phases/, helpers/, presentation/ subpackages. - `InstallContext` dataclass stub. Fields are added incrementally as phases are extracted, turning the legacy implicit lexical scope into explicit, audit-friendly data flow. - `tests/unit/install/test_architecture_invariants.py` to pin the structural contract. The 500-LOC budget guard is staged (skipped) until extraction completes in P3.R2. Why a sibling package instead of `commands/install/`: the existing module is heavily monkeypatched (~30 `@patch("apm_cli.commands.install.X")` sites). Turning it into a package would create a name collision and force test rewrites or late-lookup gymnastics. The Click adapter at `commands/install.py` will stay a single module that re-exports engine symbols, preserving every existing test patch verbatim. This commit is import-only -- no behaviour change. Targeted install tests remain green (173 tests). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/__init__.py | 19 +++++++ src/apm_cli/install/context.py | 44 +++++++++++++++ src/apm_cli/install/helpers/__init__.py | 1 + src/apm_cli/install/phases/__init__.py | 1 + src/apm_cli/install/presentation/__init__.py | 1 + tests/unit/install/__init__.py | 0 .../install/test_architecture_invariants.py | 53 +++++++++++++++++++ 7 files changed, 119 insertions(+) create mode 100644 src/apm_cli/install/__init__.py create mode 100644 src/apm_cli/install/context.py create mode 100644 src/apm_cli/install/helpers/__init__.py create mode 100644 src/apm_cli/install/phases/__init__.py create mode 100644 src/apm_cli/install/presentation/__init__.py create mode 100644 tests/unit/install/__init__.py create mode 100644 tests/unit/install/test_architecture_invariants.py diff --git a/src/apm_cli/install/__init__.py b/src/apm_cli/install/__init__.py new file mode 100644 index 00000000..3f756f4a --- /dev/null +++ b/src/apm_cli/install/__init__.py @@ -0,0 +1,19 @@ +"""APM install engine. + +This package implements the install pipeline that the +`apm_cli.commands.install` Click command delegates to. + +Architecture (in progress; see refactor/install-modularization branch): + + pipeline.py orchestrator that calls each phase in order + context.py InstallContext dataclass (state passed between phases) + options.py InstallOptions dataclass (parsed CLI options) + validation.py manifest validation (dependency syntax, existence checks) + + phases/ one module per pipeline phase + helpers/ cross-cutting helpers (security scan, gitignore) + presentation/ dry-run preview + final result rendering + +The engine is import-safe (no Click decorators at top level) so phase modules +can be unit-tested directly without invoking the CLI. +""" diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py new file mode 100644 index 00000000..43fb2ed9 --- /dev/null +++ b/src/apm_cli/install/context.py @@ -0,0 +1,44 @@ +"""Mutable state passed between install pipeline phases. + +Each phase is a function ``def run(ctx: InstallContext) -> None`` that reads +the inputs already populated by earlier phases and writes its own outputs to +the context. Keeping shared state on a single typed object turns implicit +shared lexical scope (the legacy 1444-line `_install_apm_dependencies`) into +explicit data flow that is easy to audit and to test phase-by-phase. + +Fields are added to this dataclass incrementally as phases are extracted from +the legacy entry point. A field belongs here if and only if it is read or +written by more than one phase. Phase-local state should stay local. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + + +@dataclass +class InstallContext: + """State shared across install pipeline phases. + + Currently a stub. Fields are added by the phase extractions in P1 and P2 + of the install.py modularization refactor. + + Required-on-construction fields go above the ``field(default=...)`` + barrier; outputs accumulated by phases use ``field(default_factory=...)``. + """ + + project_root: Path + apm_dir: Path + + dry_run: bool = False + force: bool = False + verbose: bool = False + dev: bool = False + only_packages: Optional[List[str]] = None + + intended_dep_keys: Set[str] = field(default_factory=set) + package_deployed_files: Dict[str, List[str]] = field(default_factory=dict) + package_types: Dict[str, Dict[str, Any]] = field(default_factory=dict) + package_hashes: Dict[str, Dict[str, str]] = field(default_factory=dict) diff --git a/src/apm_cli/install/helpers/__init__.py b/src/apm_cli/install/helpers/__init__.py new file mode 100644 index 00000000..e43fa2c9 --- /dev/null +++ b/src/apm_cli/install/helpers/__init__.py @@ -0,0 +1 @@ +"""Cross-cutting install helpers (security scan, gitignore).""" diff --git a/src/apm_cli/install/phases/__init__.py b/src/apm_cli/install/phases/__init__.py new file mode 100644 index 00000000..935ac6fa --- /dev/null +++ b/src/apm_cli/install/phases/__init__.py @@ -0,0 +1 @@ +"""Install pipeline phases.""" diff --git a/src/apm_cli/install/presentation/__init__.py b/src/apm_cli/install/presentation/__init__.py new file mode 100644 index 00000000..64762157 --- /dev/null +++ b/src/apm_cli/install/presentation/__init__.py @@ -0,0 +1 @@ +"""Install presentation layer (dry-run preview, final result rendering).""" diff --git a/tests/unit/install/__init__.py b/tests/unit/install/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/install/test_architecture_invariants.py b/tests/unit/install/test_architecture_invariants.py new file mode 100644 index 00000000..b48b8971 --- /dev/null +++ b/tests/unit/install/test_architecture_invariants.py @@ -0,0 +1,53 @@ +"""Architectural invariants for the install engine package. + +These tests are the structural defence against regression to a +god-function/god-module design. They are intentionally activated as the +modularization refactor progresses; LOC budgets are set to current actuals +and tightened as more code is extracted. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +ENGINE_ROOT = Path(__file__).resolve().parents[3] / "src" / "apm_cli" / "install" + + +def _line_count(path: Path) -> int: + return sum(1 for _ in path.read_text(encoding="utf-8").splitlines()) + + +def test_engine_package_exists(): + """The engine package must exist as a sibling of commands/.""" + assert ENGINE_ROOT.is_dir(), f"{ENGINE_ROOT} is missing" + assert (ENGINE_ROOT / "__init__.py").is_file() + assert (ENGINE_ROOT / "context.py").is_file() + assert (ENGINE_ROOT / "phases").is_dir() + assert (ENGINE_ROOT / "helpers").is_dir() + assert (ENGINE_ROOT / "presentation").is_dir() + + +def test_install_context_importable(): + """InstallContext is the contract carrying state between phases.""" + from apm_cli.install.context import InstallContext + + assert hasattr(InstallContext, "__dataclass_fields__"), ( + "InstallContext must be a dataclass" + ) + + +@pytest.mark.skip(reason="LOC budget activated in P3.R2 once extraction is complete") +def test_no_install_module_exceeds_500_loc(): + """No file in the engine package should grow past 500 LOC. + + This is the structural guard against the install.py mega-function ever + growing back. Activated in P3.R2 of the refactor with the final budget. + """ + offenders = [] + for path in ENGINE_ROOT.rglob("*.py"): + n = _line_count(path) + if n > 500: + offenders.append((path.relative_to(ENGINE_ROOT), n)) + assert not offenders, f"Modules exceeding 500 LOC: {offenders}" From be0fd2da1f5b04edd23ea85eab68353bfb5bd8ce Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 11:04:12 +0200 Subject: [PATCH 02/28] refactor(install): P1.L1 -- extract validation.py from install.py Moves three leaf validation helpers (package-existence checks, local-path diagnostics) into the install engine package as `apm_cli/install/validation.py`. These functions had zero coupling to the rest of install.py -- they take packages and return booleans or strings -- so this is a pure relocation. Functions moved: - _validate_package_exists - _local_path_failure_reason - _local_path_no_markers_hint `_validate_and_add_packages_to_apm_yml` remains in commands/install.py because it calls _validate_package_exists and _local_path_failure_reason via module-level name lookup, and 30+ tests patch `apm_cli.commands.install._validate_package_exists` to intercept those calls. Keeping the orchestrator co-located with the re-exported names preserves all @patch targets without any test modifications. `commands/install.py` re-exports the three moved names so existing test patches keep working verbatim. Targeted install tests (175) + full unit suite (3972) green. install.py LOC: 2905 -> 2630. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 301 ++-------------------------- src/apm_cli/install/validation.py | 322 ++++++++++++++++++++++++++++++ 2 files changed, 335 insertions(+), 288 deletions(-) create mode 100644 src/apm_cli/install/validation.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 357fd5c1..0265b56a 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -50,6 +50,19 @@ def _hash_deployed(rel_paths, project_root: Path) -> dict: return out from ..utils.github_host import default_host, is_valid_fqdn from ..utils.path_security import safe_rmtree + +# Re-export validation leaf helpers so that existing test patches like +# @patch("apm_cli.commands.install._validate_package_exists") keep working. +# _validate_and_add_packages_to_apm_yml stays here (not moved) because it +# calls _validate_package_exists and _local_path_failure_reason via module- +# level name lookup -- keeping it co-located means @patch on this module +# intercepts those calls without test changes. +from apm_cli.install.validation import ( + _local_path_failure_reason, + _local_path_no_markers_hint, + _validate_package_exists, +) + from ._helpers import ( _create_minimal_apm_yml, _get_default_config, @@ -334,294 +347,6 @@ def _validate_and_add_packages_to_apm_yml(packages, dry_run=False, dev=False, lo return validated_packages, outcome -def _local_path_failure_reason(dep_ref): - """Return a specific failure reason for local path deps, or None for remote.""" - if not (dep_ref.is_local and dep_ref.local_path): - return None - local = Path(dep_ref.local_path).expanduser() - if not local.is_absolute(): - local = Path.cwd() / local - local = local.resolve() - if not local.exists(): - return "path does not exist" - if not local.is_dir(): - return "path is not a directory" - # Directory exists but has no package markers - return "no apm.yml, SKILL.md, or plugin.json found" - - -def _local_path_no_markers_hint(local_dir, verbose_log=None): - """Scan two levels for sub-packages and print a hint if any are found.""" - from apm_cli.utils.helpers import find_plugin_json - - markers = ("apm.yml", "SKILL.md") - found = [] - for child in sorted(local_dir.iterdir()): - if not child.is_dir(): - continue - if any((child / m).exists() for m in markers) or find_plugin_json(child) is not None: - found.append(child) - # Also check one more level (e.g. skills//) - for grandchild in sorted(child.iterdir()) if child.is_dir() else []: - if not grandchild.is_dir(): - continue - if any((grandchild / m).exists() for m in markers) or find_plugin_json(grandchild) is not None: - found.append(grandchild) - - if not found: - return - - _rich_info(" [i] Found installable package(s) inside this directory:") - for p in found[:5]: - _rich_echo(f" apm install {p}", color="dim") - if len(found) > 5: - _rich_echo(f" ... and {len(found) - 5} more", color="dim") - - -def _validate_package_exists(package, verbose=False, auth_resolver=None): - """Validate that a package exists and is accessible on GitHub, Azure DevOps, or locally.""" - import os - import subprocess - import tempfile - from apm_cli.core.auth import AuthResolver - - verbose_log = (lambda msg: _rich_echo(f" {msg}", color="dim")) if verbose else None - # Use provided resolver or create new one if not in a CLI session context - if auth_resolver is None: - auth_resolver = AuthResolver() - - try: - # Parse the package to check if it's a virtual package or ADO - from apm_cli.models.apm_package import DependencyReference - from apm_cli.deps.github_downloader import GitHubPackageDownloader - - dep_ref = DependencyReference.parse(package) - - # For local packages, validate directory exists and has valid package content - if dep_ref.is_local and dep_ref.local_path: - local = Path(dep_ref.local_path).expanduser() - if not local.is_absolute(): - local = Path.cwd() / local - local = local.resolve() - if not local.is_dir(): - return False - # Must contain apm.yml, SKILL.md, or plugin.json - if (local / "apm.yml").exists() or (local / "SKILL.md").exists(): - return True - from apm_cli.utils.helpers import find_plugin_json - if find_plugin_json(local) is not None: - return True - # Directory exists but lacks package markers -- surface a hint - _local_path_no_markers_hint(local, verbose_log) - return False - - # For virtual packages, use the downloader's validation method - if dep_ref.is_virtual: - ctx = auth_resolver.resolve_for_dep(dep_ref) - host = dep_ref.host or default_host() - org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None - if verbose_log: - verbose_log(f"Auth resolved: host={host}, org={org}, source={ctx.source}, type={ctx.token_type}") - virtual_downloader = GitHubPackageDownloader(auth_resolver=auth_resolver) - result = virtual_downloader.validate_virtual_package_exists(dep_ref) - if not result and verbose_log: - try: - err_ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) - for line in err_ctx.splitlines(): - verbose_log(line) - except Exception: - pass - return result - - # For Azure DevOps or GitHub Enterprise (non-github.com hosts), - # use the downloader which handles authentication properly - if dep_ref.is_azure_devops() or (dep_ref.host and dep_ref.host != "github.com"): - from apm_cli.utils.github_host import is_github_hostname, is_azure_devops_hostname - - # Determine host type before building the URL so we know whether to - # embed a token. Generic (non-GitHub, non-ADO) hosts are excluded - # from APM-managed auth; they rely on git credential helpers via the - # relaxed validate_env below. - is_generic = not is_github_hostname(dep_ref.host) and not is_azure_devops_hostname(dep_ref.host) - - # For GHES / ADO: resolve per-dependency auth up front so the URL - # carries an embedded token and avoids triggering OS credential - # helper popups during git ls-remote validation. - _url_token = None - if not is_generic: - _dep_ctx = auth_resolver.resolve_for_dep(dep_ref) - _url_token = _dep_ctx.token - - ado_downloader = GitHubPackageDownloader(auth_resolver=auth_resolver) - # Set the host - if dep_ref.host: - ado_downloader.github_host = dep_ref.host - - # Build authenticated URL using the resolved per-dep token. - package_url = ado_downloader._build_repo_url( - dep_ref.repo_url, use_ssh=False, dep_ref=dep_ref, token=_url_token - ) - - # For generic hosts (not GitHub, not ADO), relax the env so native - # credential helpers (SSH keys, macOS Keychain, etc.) can work. - # This mirrors _clone_with_fallback() which does the same relaxation. - if is_generic: - validate_env = {k: v for k, v in ado_downloader.git_env.items() - if k not in ('GIT_ASKPASS', 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_NOSYSTEM')} - validate_env['GIT_TERMINAL_PROMPT'] = '0' - else: - validate_env = {**os.environ, **ado_downloader.git_env} - - if verbose_log: - verbose_log(f"Trying git ls-remote for {dep_ref.host}") - - # For generic hosts, try SSH first (no credentials needed when SSH - # keys are configured) before falling back to HTTPS. - urls_to_try = [] - if is_generic: - ssh_url = ado_downloader._build_repo_url( - dep_ref.repo_url, use_ssh=True, dep_ref=dep_ref - ) - urls_to_try = [ssh_url, package_url] - else: - urls_to_try = [package_url] - - result = None - for probe_url in urls_to_try: - cmd = ["git", "ls-remote", "--heads", "--exit-code", probe_url] - result = subprocess.run( - cmd, - capture_output=True, - text=True, - encoding="utf-8", - timeout=30, - env=validate_env, - ) - if result.returncode == 0: - break - - if verbose_log: - if result.returncode == 0: - verbose_log(f"git ls-remote rc=0 for {package}") - else: - # Sanitize stderr to avoid leaking tokens - stderr_snippet = (result.stderr or "").strip()[:200] - for env_var in ("GIT_ASKPASS", "GIT_CONFIG_GLOBAL"): - stderr_snippet = stderr_snippet.replace( - validate_env.get(env_var, ""), "***" - ) - verbose_log(f"git ls-remote rc={result.returncode}: {stderr_snippet}") - - return result.returncode == 0 - - # For GitHub.com, use AuthResolver with unauth-first fallback - host = dep_ref.host or default_host() - org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None - host_info = auth_resolver.classify_host(host) - - if verbose_log: - ctx = auth_resolver.resolve(host, org=org) - verbose_log(f"Auth resolved: host={host}, org={org}, source={ctx.source}, type={ctx.token_type}") - - def _check_repo(token, git_env): - """Check repo accessibility via GitHub API (or git ls-remote for non-GitHub).""" - import urllib.request - import urllib.error - - api_base = host_info.api_base - api_url = f"{api_base}/repos/{dep_ref.repo_url}" - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "apm-cli", - } - if token: - headers["Authorization"] = f"Bearer {token}" - - req = urllib.request.Request(api_url, headers=headers) - try: - resp = urllib.request.urlopen(req, timeout=15) - if verbose_log: - verbose_log(f"API {api_url} -> {resp.status}") - return True - except urllib.error.HTTPError as e: - if verbose_log: - verbose_log(f"API {api_url} -> {e.code} {e.reason}") - if e.code == 404 and token: - # 404 with token could mean no access — raise to trigger fallback - raise RuntimeError(f"API returned {e.code}") - raise RuntimeError(f"API returned {e.code}: {e.reason}") - except Exception as e: - if verbose_log: - verbose_log(f"API request failed: {e}") - raise - - try: - return auth_resolver.try_with_fallback( - host, _check_repo, - org=org, - unauth_first=True, - verbose_callback=verbose_log, - ) - except Exception: - if verbose_log: - try: - ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) - for line in ctx.splitlines(): - verbose_log(line) - except Exception: - pass - return False - - except Exception: - # If parsing fails, assume it's a regular GitHub package - host = default_host() - org = package.split('/')[0] if '/' in package else None - repo_path = package # owner/repo format - - def _check_repo_fallback(token, git_env): - import urllib.request - import urllib.error - - host_info = auth_resolver.classify_host(host) - api_url = f"{host_info.api_base}/repos/{repo_path}" - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "apm-cli", - } - if token: - headers["Authorization"] = f"Bearer {token}" - - req = urllib.request.Request(api_url, headers=headers) - try: - resp = urllib.request.urlopen(req, timeout=15) - return True - except urllib.error.HTTPError as e: - if verbose_log: - verbose_log(f"API fallback -> {e.code} {e.reason}") - raise RuntimeError(f"API returned {e.code}") - except Exception as e: - if verbose_log: - verbose_log(f"API fallback failed: {e}") - raise - - try: - return auth_resolver.try_with_fallback( - host, _check_repo_fallback, - org=org, - unauth_first=True, - verbose_callback=verbose_log, - ) - except Exception: - if verbose_log: - try: - ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) - for line in ctx.splitlines(): - verbose_log(line) - except Exception: - pass - return False - - # --------------------------------------------------------------------------- # Install command # --------------------------------------------------------------------------- diff --git a/src/apm_cli/install/validation.py b/src/apm_cli/install/validation.py new file mode 100644 index 00000000..aee2f815 --- /dev/null +++ b/src/apm_cli/install/validation.py @@ -0,0 +1,322 @@ +"""Manifest validation: package existence checks, dependency syntax canonicalisation. + +This module contains the leaf validation helpers extracted from +``apm_cli.commands.install``. They are pure functions of their arguments +with zero coupling to the install pipeline, which is why they could be +relocated verbatim. + +The orchestrator ``_validate_and_add_packages_to_apm_yml`` remains in +``commands/install.py`` because dozens of tests patch +``apm_cli.commands.install._validate_package_exists`` and rely on +module-level name resolution inside the orchestrator to intercept the call. +Keeping the orchestrator co-located with the re-exported name preserves +``@patch`` compatibility without any test modifications. + +Functions +--------- +_validate_package_exists + Probe GitHub API / git-ls-remote / local FS to confirm a package ref + is accessible. +_local_path_failure_reason + Return a human-readable reason when a local-path dep fails validation. +_local_path_no_markers_hint + Scan a local directory for nested installable packages and hint the user. +""" + +from pathlib import Path + +from ..utils.console import _rich_echo, _rich_info +from ..utils.github_host import default_host + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + + +def _local_path_failure_reason(dep_ref): + """Return a specific failure reason for local path deps, or None for remote.""" + if not (dep_ref.is_local and dep_ref.local_path): + return None + local = Path(dep_ref.local_path).expanduser() + if not local.is_absolute(): + local = Path.cwd() / local + local = local.resolve() + if not local.exists(): + return "path does not exist" + if not local.is_dir(): + return "path is not a directory" + # Directory exists but has no package markers + return "no apm.yml, SKILL.md, or plugin.json found" + + +def _local_path_no_markers_hint(local_dir, verbose_log=None): + """Scan two levels for sub-packages and print a hint if any are found.""" + from apm_cli.utils.helpers import find_plugin_json + + markers = ("apm.yml", "SKILL.md") + found = [] + for child in sorted(local_dir.iterdir()): + if not child.is_dir(): + continue + if any((child / m).exists() for m in markers) or find_plugin_json(child) is not None: + found.append(child) + # Also check one more level (e.g. skills//) + for grandchild in sorted(child.iterdir()) if child.is_dir() else []: + if not grandchild.is_dir(): + continue + if any((grandchild / m).exists() for m in markers) or find_plugin_json(grandchild) is not None: + found.append(grandchild) + + if not found: + return + + _rich_info(" [i] Found installable package(s) inside this directory:") + for p in found[:5]: + _rich_echo(f" apm install {p}", color="dim") + if len(found) > 5: + _rich_echo(f" ... and {len(found) - 5} more", color="dim") + + +def _validate_package_exists(package, verbose=False, auth_resolver=None): + """Validate that a package exists and is accessible on GitHub, Azure DevOps, or locally.""" + import os + import subprocess + import tempfile + from apm_cli.core.auth import AuthResolver + + verbose_log = (lambda msg: _rich_echo(f" {msg}", color="dim")) if verbose else None + # Use provided resolver or create new one if not in a CLI session context + if auth_resolver is None: + auth_resolver = AuthResolver() + + try: + # Parse the package to check if it's a virtual package or ADO + from apm_cli.models.apm_package import DependencyReference + from apm_cli.deps.github_downloader import GitHubPackageDownloader + + dep_ref = DependencyReference.parse(package) + + # For local packages, validate directory exists and has valid package content + if dep_ref.is_local and dep_ref.local_path: + local = Path(dep_ref.local_path).expanduser() + if not local.is_absolute(): + local = Path.cwd() / local + local = local.resolve() + if not local.is_dir(): + return False + # Must contain apm.yml, SKILL.md, or plugin.json + if (local / "apm.yml").exists() or (local / "SKILL.md").exists(): + return True + from apm_cli.utils.helpers import find_plugin_json + if find_plugin_json(local) is not None: + return True + # Directory exists but lacks package markers -- surface a hint + _local_path_no_markers_hint(local, verbose_log) + return False + + # For virtual packages, use the downloader's validation method + if dep_ref.is_virtual: + ctx = auth_resolver.resolve_for_dep(dep_ref) + host = dep_ref.host or default_host() + org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None + if verbose_log: + verbose_log(f"Auth resolved: host={host}, org={org}, source={ctx.source}, type={ctx.token_type}") + virtual_downloader = GitHubPackageDownloader(auth_resolver=auth_resolver) + result = virtual_downloader.validate_virtual_package_exists(dep_ref) + if not result and verbose_log: + try: + err_ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) + for line in err_ctx.splitlines(): + verbose_log(line) + except Exception: + pass + return result + + # For Azure DevOps or GitHub Enterprise (non-github.com hosts), + # use the downloader which handles authentication properly + if dep_ref.is_azure_devops() or (dep_ref.host and dep_ref.host != "github.com"): + from apm_cli.utils.github_host import is_github_hostname, is_azure_devops_hostname + + # Determine host type before building the URL so we know whether to + # embed a token. Generic (non-GitHub, non-ADO) hosts are excluded + # from APM-managed auth; they rely on git credential helpers via the + # relaxed validate_env below. + is_generic = not is_github_hostname(dep_ref.host) and not is_azure_devops_hostname(dep_ref.host) + + # For GHES / ADO: resolve per-dependency auth up front so the URL + # carries an embedded token and avoids triggering OS credential + # helper popups during git ls-remote validation. + _url_token = None + if not is_generic: + _dep_ctx = auth_resolver.resolve_for_dep(dep_ref) + _url_token = _dep_ctx.token + + ado_downloader = GitHubPackageDownloader(auth_resolver=auth_resolver) + # Set the host + if dep_ref.host: + ado_downloader.github_host = dep_ref.host + + # Build authenticated URL using the resolved per-dep token. + package_url = ado_downloader._build_repo_url( + dep_ref.repo_url, use_ssh=False, dep_ref=dep_ref, token=_url_token + ) + + # For generic hosts (not GitHub, not ADO), relax the env so native + # credential helpers (SSH keys, macOS Keychain, etc.) can work. + # This mirrors _clone_with_fallback() which does the same relaxation. + if is_generic: + validate_env = {k: v for k, v in ado_downloader.git_env.items() + if k not in ('GIT_ASKPASS', 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_NOSYSTEM')} + validate_env['GIT_TERMINAL_PROMPT'] = '0' + else: + validate_env = {**os.environ, **ado_downloader.git_env} + + if verbose_log: + verbose_log(f"Trying git ls-remote for {dep_ref.host}") + + # For generic hosts, try SSH first (no credentials needed when SSH + # keys are configured) before falling back to HTTPS. + urls_to_try = [] + if is_generic: + ssh_url = ado_downloader._build_repo_url( + dep_ref.repo_url, use_ssh=True, dep_ref=dep_ref + ) + urls_to_try = [ssh_url, package_url] + else: + urls_to_try = [package_url] + + result = None + for probe_url in urls_to_try: + cmd = ["git", "ls-remote", "--heads", "--exit-code", probe_url] + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + env=validate_env, + ) + if result.returncode == 0: + break + + if verbose_log: + if result.returncode == 0: + verbose_log(f"git ls-remote rc=0 for {package}") + else: + # Sanitize stderr to avoid leaking tokens + stderr_snippet = (result.stderr or "").strip()[:200] + for env_var in ("GIT_ASKPASS", "GIT_CONFIG_GLOBAL"): + stderr_snippet = stderr_snippet.replace( + validate_env.get(env_var, ""), "***" + ) + verbose_log(f"git ls-remote rc={result.returncode}: {stderr_snippet}") + + return result.returncode == 0 + + # For GitHub.com, use AuthResolver with unauth-first fallback + host = dep_ref.host or default_host() + org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None + host_info = auth_resolver.classify_host(host) + + if verbose_log: + ctx = auth_resolver.resolve(host, org=org) + verbose_log(f"Auth resolved: host={host}, org={org}, source={ctx.source}, type={ctx.token_type}") + + def _check_repo(token, git_env): + """Check repo accessibility via GitHub API (or git ls-remote for non-GitHub).""" + import urllib.request + import urllib.error + + api_base = host_info.api_base + api_url = f"{api_base}/repos/{dep_ref.repo_url}" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "apm-cli", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request(api_url, headers=headers) + try: + resp = urllib.request.urlopen(req, timeout=15) + if verbose_log: + verbose_log(f"API {api_url} -> {resp.status}") + return True + except urllib.error.HTTPError as e: + if verbose_log: + verbose_log(f"API {api_url} -> {e.code} {e.reason}") + if e.code == 404 and token: + # 404 with token could mean no access — raise to trigger fallback + raise RuntimeError(f"API returned {e.code}") + raise RuntimeError(f"API returned {e.code}: {e.reason}") + except Exception as e: + if verbose_log: + verbose_log(f"API request failed: {e}") + raise + + try: + return auth_resolver.try_with_fallback( + host, _check_repo, + org=org, + unauth_first=True, + verbose_callback=verbose_log, + ) + except Exception: + if verbose_log: + try: + ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) + for line in ctx.splitlines(): + verbose_log(line) + except Exception: + pass + return False + + except Exception: + # If parsing fails, assume it's a regular GitHub package + host = default_host() + org = package.split('/')[0] if '/' in package else None + repo_path = package # owner/repo format + + def _check_repo_fallback(token, git_env): + import urllib.request + import urllib.error + + host_info = auth_resolver.classify_host(host) + api_url = f"{host_info.api_base}/repos/{repo_path}" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "apm-cli", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request(api_url, headers=headers) + try: + resp = urllib.request.urlopen(req, timeout=15) + return True + except urllib.error.HTTPError as e: + if verbose_log: + verbose_log(f"API fallback -> {e.code} {e.reason}") + raise RuntimeError(f"API returned {e.code}") + except Exception as e: + if verbose_log: + verbose_log(f"API fallback failed: {e}") + raise + + try: + return auth_resolver.try_with_fallback( + host, _check_repo_fallback, + org=org, + unauth_first=True, + verbose_callback=verbose_log, + ) + except Exception: + if verbose_log: + try: + ctx = auth_resolver.build_error_context(host, f"accessing {package}", org=org) + for line in ctx.splitlines(): + verbose_log(line) + except Exception: + pass + return False From 706f1d829ed5661ace0666c0bebd2f20734b5cd0 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 11:10:34 +0200 Subject: [PATCH 03/28] refactor(install): P1.L2 -- extract local_content.py phase module Moves local-content leaf helpers (root project's .apm/ as implicit local package per #714, local-path dependencies from apm.yml) into `apm_cli/install/phases/local_content.py`. Three of the four planned functions form a coherent feature with no coupling to the integration pipeline -- they take a project root and return booleans or copy files. Functions moved: - _project_has_root_primitives - _has_local_apm_content - _copy_local_package Function KEPT in commands/install.py: - _integrate_local_content (calls _integrate_package_primitives via bare-name lookup; 5 tests patch apm_cli.commands.install._integrate_package_primitives to intercept that call -- same L1 lesson as _validate_and_add_packages_to_apm_yml) `commands/install.py` re-exports the three moved names so existing callers (_install_apm_dependencies) and any future @patch targets keep working. Targeted install tests (196) + full unit suite (3972) green. install.py LOC: 2630 -> 2562. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 92 ++------------- src/apm_cli/install/phases/local_content.py | 121 ++++++++++++++++++++ 2 files changed, 133 insertions(+), 80 deletions(-) create mode 100644 src/apm_cli/install/phases/local_content.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 0265b56a..b48b12e9 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -63,6 +63,18 @@ def _hash_deployed(rel_paths, project_root: Path) -> dict: _validate_package_exists, ) +# Re-export local-content leaf helpers so that callers inside this module +# (e.g. _install_apm_dependencies) and any future test patches against +# "apm_cli.commands.install._copy_local_package" keep working. +# _integrate_local_content stays here (not moved) because it calls +# _integrate_package_primitives via bare-name lookup and tests patch +# apm_cli.commands.install._integrate_package_primitives to intercept it. +from apm_cli.install.phases.local_content import ( + _copy_local_package, + _has_local_apm_content, + _project_has_root_primitives, +) + from ._helpers import ( _create_minimal_apm_yml, _get_default_config, @@ -97,23 +109,6 @@ def _hash_deployed(rel_paths, project_root: Path) -> dict: _APM_IMPORT_ERROR = str(e) -# --------------------------------------------------------------------------- -# Root primitive detection helper -# --------------------------------------------------------------------------- - -def _project_has_root_primitives(project_root) -> bool: - """Return True when *project_root* has a .apm/ directory of its own. - - Used to decide whether ``apm install`` should enter the integration - pipeline even when no external APM dependencies are declared (#714). - The integrators themselves determine whether the directory contains - anything actionable, so we only check for the directory's existence. - """ - from pathlib import Path as _Path - root = _Path(project_root) - return (root / ".apm").is_dir() - - # --------------------------------------------------------------------------- # Validation helpers # --------------------------------------------------------------------------- @@ -1055,24 +1050,6 @@ def _log_integration(msg): return result -def _has_local_apm_content(project_root): - """Check if the project has local .apm/ content worth integrating. - - Returns True if .apm/ exists and contains at least one primitive file - in a recognized subdirectory (skills, instructions, agents/chatmodes, - prompts, hooks, commands). - """ - apm_dir = project_root / ".apm" - if not apm_dir.is_dir(): - return False - _PRIMITIVE_DIRS = ("skills", "instructions", "chatmodes", "agents", "prompts", "hooks", "commands") - for subdir_name in _PRIMITIVE_DIRS: - subdir = apm_dir / subdir_name - if subdir.is_dir() and any(p.is_file() for p in subdir.rglob("*")): - return True - return False - - def _integrate_local_content( project_root, *, @@ -1138,51 +1115,6 @@ def _integrate_local_content( ) -def _copy_local_package(dep_ref, install_path, project_root): - """Copy a local package to apm_modules/. - - Args: - dep_ref: DependencyReference with is_local=True - install_path: Target path under apm_modules/ - project_root: Project root for resolving relative paths - - Returns: - install_path on success, None on failure - """ - import shutil - - local = Path(dep_ref.local_path).expanduser() - if not local.is_absolute(): - local = (project_root / local).resolve() - else: - local = local.resolve() - - if not local.is_dir(): - _rich_error(f"Local package path does not exist: {dep_ref.local_path}") - return None - from apm_cli.utils.helpers import find_plugin_json - if ( - not (local / "apm.yml").exists() - and not (local / "SKILL.md").exists() - and find_plugin_json(local) is None - ): - _rich_error( - f"Local package is not a valid APM package (no apm.yml, SKILL.md, or plugin.json): {dep_ref.local_path}" - ) - return None - - # Ensure parent exists and clean target (always re-copy for local deps) - install_path.parent.mkdir(parents=True, exist_ok=True) - if install_path.exists(): - # install_path is already validated by get_install_path() (Layer 2), - # but use safe_rmtree for defense-in-depth. - apm_modules_dir = install_path.parent.parent # _local/ → apm_modules - safe_rmtree(install_path, apm_modules_dir) - - shutil.copytree(local, install_path, dirs_exist_ok=False, symlinks=True) - return install_path - - def _install_apm_dependencies( apm_package: "APMPackage", update_refs: bool = False, diff --git a/src/apm_cli/install/phases/local_content.py b/src/apm_cli/install/phases/local_content.py new file mode 100644 index 00000000..19b3d1cf --- /dev/null +++ b/src/apm_cli/install/phases/local_content.py @@ -0,0 +1,121 @@ +"""Local-content integration: deploy primitives the user authored locally. + +This module handles two related scenarios: + +1. **Root project as implicit local package (#714)** -- when the project's own + ``.apm/`` directory contains skills, instructions, agents, prompts, hooks, + or commands, ``apm install`` deploys them to target directories exactly like + dependency primitives. ``_project_has_root_primitives`` and + ``_has_local_apm_content`` detect this case. + +2. **Local-path dependencies from apm.yml** -- ``_copy_local_package`` copies + a locally-referenced package into ``apm_modules/`` so the downstream + integration pipeline can treat it uniformly. + +The orchestrator ``_integrate_local_content`` remains in +``apm_cli.commands.install`` because it calls ``_integrate_package_primitives`` +via bare-name lookup, and tests patch +``apm_cli.commands.install._integrate_package_primitives`` to intercept that +call. Keeping the orchestrator co-located with the re-exported name preserves +``@patch`` compatibility without any test modifications. + +Functions +--------- +_project_has_root_primitives + Return True when the project root contains a ``.apm/`` directory. +_has_local_apm_content + Return True when ``.apm/`` contains at least one primitive file. +_copy_local_package + Copy a local-path dependency into ``apm_modules/``. +""" + +from pathlib import Path + +from apm_cli.utils.console import _rich_error +from apm_cli.utils.path_security import safe_rmtree + + +# --------------------------------------------------------------------------- +# Root primitive detection helpers +# --------------------------------------------------------------------------- + + +def _project_has_root_primitives(project_root) -> bool: + """Return True when *project_root* has a .apm/ directory of its own. + + Used to decide whether ``apm install`` should enter the integration + pipeline even when no external APM dependencies are declared (#714). + The integrators themselves determine whether the directory contains + anything actionable, so we only check for the directory's existence. + """ + from pathlib import Path as _Path + root = _Path(project_root) + return (root / ".apm").is_dir() + + +def _has_local_apm_content(project_root): + """Check if the project has local .apm/ content worth integrating. + + Returns True if .apm/ exists and contains at least one primitive file + in a recognized subdirectory (skills, instructions, agents/chatmodes, + prompts, hooks, commands). + """ + apm_dir = project_root / ".apm" + if not apm_dir.is_dir(): + return False + _PRIMITIVE_DIRS = ("skills", "instructions", "chatmodes", "agents", "prompts", "hooks", "commands") + for subdir_name in _PRIMITIVE_DIRS: + subdir = apm_dir / subdir_name + if subdir.is_dir() and any(p.is_file() for p in subdir.rglob("*")): + return True + return False + + +# --------------------------------------------------------------------------- +# Local-path dependency copy +# --------------------------------------------------------------------------- + + +def _copy_local_package(dep_ref, install_path, project_root): + """Copy a local package to apm_modules/. + + Args: + dep_ref: DependencyReference with is_local=True + install_path: Target path under apm_modules/ + project_root: Project root for resolving relative paths + + Returns: + install_path on success, None on failure + """ + import shutil + + local = Path(dep_ref.local_path).expanduser() + if not local.is_absolute(): + local = (project_root / local).resolve() + else: + local = local.resolve() + + if not local.is_dir(): + _rich_error(f"Local package path does not exist: {dep_ref.local_path}") + return None + from apm_cli.utils.helpers import find_plugin_json + if ( + not (local / "apm.yml").exists() + and not (local / "SKILL.md").exists() + and find_plugin_json(local) is None + ): + _rich_error( + f"Local package is not a valid APM package (no apm.yml, SKILL.md, or plugin.json): {dep_ref.local_path}" + ) + return None + + # Ensure parent exists and clean target (always re-copy for local deps) + install_path.parent.mkdir(parents=True, exist_ok=True) + if install_path.exists(): + # install_path is already validated by get_install_path() (Layer 2), + # but use safe_rmtree for defense-in-depth. + apm_modules_dir = install_path.parent.parent # _local/ → apm_modules + safe_rmtree(install_path, apm_modules_dir) + + shutil.copytree(local, install_path, dirs_exist_ok=False, symlinks=True) + return install_path From 7548afd1e27623e6f03a70e76f5bc0b915113ee0 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 11:14:51 +0200 Subject: [PATCH 04/28] refactor(install): P1.L3 -- introduce LockfileBuilder + relocate _hash_deployed Moves the per-file content-hash helper (added in #762) into the install engine package as `apm_cli/install/phases/lockfile.py`, alongside a `LockfileBuilder` class skeleton. The bulk of lockfile assembly currently lives inline inside `_install_apm_dependencies`; P2.S6 will fold that into LockfileBuilder. This commit relocates only the leaf helper to keep the change small and the test patches stable. `commands/install.py` aliases the new function back to `_hash_deployed` so the regression test pinned in #762 (`test_hash_deployed_is_module_level_and_works`) keeps working without modification. install.py LOC: 2562 -> 2547. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 23 ++-------- src/apm_cli/install/phases/lockfile.py | 61 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 src/apm_cli/install/phases/lockfile.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index b48b12e9..dbca42ce 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -25,29 +25,14 @@ from ..models.results import InstallResult from ..core.command_logger import InstallLogger, _ValidationOutcome from ..utils.console import _rich_echo, _rich_error, _rich_info, _rich_success -from ..utils.content_hash import compute_file_hash as _compute_file_hash_for_provenance from ..utils.diagnostics import DiagnosticCollector -def _hash_deployed(rel_paths, project_root: Path) -> dict: - """Hash currently-on-disk deployed files for provenance. +# Re-export lockfile hash helper so existing call sites and the regression +# test pinned in #762 (test_hash_deployed_is_module_level_and_works) keep +# working via "apm_cli.commands.install._hash_deployed". +from apm_cli.install.phases.lockfile import compute_deployed_hashes as _hash_deployed - Module-level so both the local-package persist site (in - ``_integrate_local_content``) and the remote-package lockfile-build - site (in ``_install_apm_dependencies``) share one implementation. - Returns ``{rel_path: "sha256:"}`` for files that exist as regular - files; symlinks and unreadable paths are silently omitted (they cannot - contribute meaningful provenance). - """ - out: dict = {} - for _rel in rel_paths or (): - _full = project_root / _rel - if _full.is_file() and not _full.is_symlink(): - try: - out[_rel] = _compute_file_hash_for_provenance(_full) - except Exception: - pass - return out from ..utils.github_host import default_host, is_valid_fqdn from ..utils.path_security import safe_rmtree diff --git a/src/apm_cli/install/phases/lockfile.py b/src/apm_cli/install/phases/lockfile.py new file mode 100644 index 00000000..7462cac4 --- /dev/null +++ b/src/apm_cli/install/phases/lockfile.py @@ -0,0 +1,61 @@ +"""Lockfile assembly: build a ``LockFile`` from install artefacts. + +This module hosts the ``LockfileBuilder`` that assembles a +:class:`~apm_cli.deps.lockfile.LockFile` from the artefacts produced by +earlier install phases (deployed files, types, hashes, marketplace +provenance, dependency graph). + +Currently exposes only ``compute_deployed_hashes()`` — the per-file +content-hash helper relocated from ``commands/install.py`` +(:pypi:`#762`). P2.S6 will fold the inline lockfile assembly logic +that lives inside ``_install_apm_dependencies`` into +:class:`LockfileBuilder`. +""" + +from pathlib import Path + +from apm_cli.utils.content_hash import compute_file_hash + + +def compute_deployed_hashes(rel_paths, project_root: Path) -> dict: + """Hash currently-on-disk deployed files for provenance. + + Module-level so both the local-package persist site (in + ``_integrate_local_content``) and the remote-package lockfile-build + site (in ``_install_apm_dependencies``) share one implementation. + Returns ``{rel_path: "sha256:"}`` for files that exist as regular + files; symlinks and unreadable paths are silently omitted (they cannot + contribute meaningful provenance). + """ + out: dict = {} + for _rel in rel_paths or (): + _full = project_root / _rel + if _full.is_file() and not _full.is_symlink(): + try: + out[_rel] = compute_file_hash(_full) + except Exception: + pass + return out + + +class LockfileBuilder: + """Incrementally assembles a ``LockFile`` from install artefacts. + + Currently a thin skeleton that delegates to + :func:`compute_deployed_hashes`. The following builder methods will + be added in **P2.S6** when the inline lockfile assembly logic inside + ``_install_apm_dependencies`` is folded in: + + - ``with_installed(dep_key, locked_dep)`` + - ``with_deployed_files(dep_key, files)`` + - ``with_types(dep_key, package_type)`` + - ``with_provenance(dep_key, marketplace_info)`` + - ``build() -> LockFile`` + """ + + def __init__(self, project_root: Path) -> None: + self.project_root = project_root + + def compute_deployed_hashes(self, rel_paths) -> dict[str, str]: + """Delegate to the module-level canonical implementation.""" + return compute_deployed_hashes(rel_paths, self.project_root) From 2e918411bc9469966914fd6d32460bd302b0e8c0 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 11:17:58 +0200 Subject: [PATCH 05/28] refactor(install): P1.L4 -- extract pre-deploy security scan helper Moves `_pre_deploy_security_scan` into the install engine package as `apm_cli/install/helpers/security_scan.py`. The helper has a clean single responsibility (run the MCP scanner before any deploy) and no coupling to the rest of install.py. Note: the plan also called for a gitignore helper extraction, but `_update_gitignore_for_apm_modules` already lives in `commands/_helpers.py`. L4 reduces to security_scan only. `commands/install.py` re-exports the moved name so `tests/unit/test_install_scanning.py`'s `from apm_cli.commands.install import _pre_deploy_security_scan` keeps working without modification. install.py LOC: 2547 -> 2513. P1 (leaf extractions) complete. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 46 +++--------------- src/apm_cli/install/helpers/security_scan.py | 51 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 40 deletions(-) create mode 100644 src/apm_cli/install/helpers/security_scan.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index dbca42ce..49df5027 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -60,6 +60,12 @@ _project_has_root_primitives, ) +# Re-export the pre-deploy security scan so that bare-name call sites inside +# this module and ``tests/unit/test_install_scanning.py``'s direct import +# (``from apm_cli.commands.install import _pre_deploy_security_scan``) keep +# working without modification. +from apm_cli.install.helpers.security_scan import _pre_deploy_security_scan + from ._helpers import ( _create_minimal_apm_yml, _get_default_config, @@ -866,46 +872,6 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # --------------------------------------------------------------------------- -def _pre_deploy_security_scan( - install_path: Path, - diagnostics: DiagnosticCollector, - package_name: str = "", - force: bool = False, - logger=None, -) -> bool: - """Scan package source files for hidden characters BEFORE deployment. - - Delegates to :class:`SecurityGate` for the scan->classify->decide pipeline. - Inline CLI feedback (error/info lines) is kept here because it is - install-specific formatting. - - Returns: - True if deployment should proceed, False to block. - """ - from ..security.gate import BLOCK_POLICY, SecurityGate - - verdict = SecurityGate.scan_files( - install_path, policy=BLOCK_POLICY, force=force - ) - if not verdict.has_findings: - return True - - # Record into diagnostics (consistent messaging via gate) - SecurityGate.report(verdict, diagnostics, package=package_name, force=force) - - if verdict.should_block: - if logger: - logger.error( - f" Blocked: {package_name or 'package'} contains " - f"critical hidden character(s)" - ) - logger.progress(f" └─ Inspect source: {install_path}") - logger.progress(" └─ Use --force to deploy anyway") - return False - - return True - - def _integrate_package_primitives( package_info, project_root, diff --git a/src/apm_cli/install/helpers/security_scan.py b/src/apm_cli/install/helpers/security_scan.py new file mode 100644 index 00000000..0e5b2ae0 --- /dev/null +++ b/src/apm_cli/install/helpers/security_scan.py @@ -0,0 +1,51 @@ +"""Pre-deploy security scan that runs before any file is written to the project tree. + +Wraps the :class:`~apm_cli.security.gate.SecurityGate` scanner used by the +install pipeline. The scan detects hidden characters (zero-width joiners, +bidirectional overrides, etc.) that could be used to smuggle malicious payloads +into prompts, skills, or agent definitions. +""" + +from pathlib import Path + +from apm_cli.utils.diagnostics import DiagnosticCollector + + +def _pre_deploy_security_scan( + install_path: Path, + diagnostics: DiagnosticCollector, + package_name: str = "", + force: bool = False, + logger=None, +) -> bool: + """Scan package source files for hidden characters BEFORE deployment. + + Delegates to :class:`SecurityGate` for the scan->classify->decide pipeline. + Inline CLI feedback (error/info lines) is kept here because it is + install-specific formatting. + + Returns: + True if deployment should proceed, False to block. + """ + from apm_cli.security.gate import BLOCK_POLICY, SecurityGate + + verdict = SecurityGate.scan_files( + install_path, policy=BLOCK_POLICY, force=force + ) + if not verdict.has_findings: + return True + + # Record into diagnostics (consistent messaging via gate) + SecurityGate.report(verdict, diagnostics, package=package_name, force=force) + + if verdict.should_block: + if logger: + logger.error( + f" Blocked: {package_name or 'package'} contains " + f"critical hidden character(s)" + ) + logger.progress(f" └─ Inspect source: {install_path}") + logger.progress(" └─ Use --force to deploy anyway") + return False + + return True From 0b981cb74b6f62ead1865482ec908e573669be52 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 11:44:23 +0200 Subject: [PATCH 06/28] refactor(install): P2.S1 -- extract resolve & targets phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the resolve and targets phases from _install_apm_dependencies into dedicated modules under src/apm_cli/install/phases/. InstallContext extended with all fields for resolve + targets outputs. The mega-function now delegates to phase.run(ctx) and reads results back at the seam so the remaining ~1100 lines are untouched. install.py body: 2513 → 2282 LOC (−231) New: phases/resolve.py 313 LOC, phases/targets.py 100 LOC Rubber-duck findings (1 critical, 1 medium, 1 low): CRITICAL – Fixed: GitHubPackageDownloader test patch bypass. resolve.py imported the class by name (from apm_cli.deps.github_downloader import GitHubPackageDownloader), so tests patching apm_cli.commands.install.GitHubPackageDownloader were silent no-ops. Fix: resolve.py now uses module-attribute access (_ghd_mod.GitHubPackageDownloader); 10 test patches updated to canonical path apm_cli.deps.github_downloader. MEDIUM – Noted: try/except boundary shift. Resolution errors now propagate raw instead of being wrapped in 'Failed to resolve APM dependencies: ...'. No test asserts on the wrapped message. External callers may need updating. LOW – Preserved: callback_failures latent bug (dict assignment on set at original line 1182). Faithfully extracted; filed for follow-up. Verification gates (all green): 1. Import check: phases load cleanly 2. Targeted suite: 90 passed 3. Full unit suite: 4528 passed (5 pre-existing failures excluded) 4. Integration suite: 87 passed --- src/apm_cli/commands/install.py | 351 +++--------------- src/apm_cli/install/context.py | 57 ++- src/apm_cli/install/phases/resolve.py | 313 ++++++++++++++++ src/apm_cli/install/phases/targets.py | 100 +++++ .../integration/test_selective_install_mcp.py | 16 +- tests/unit/test_install_command.py | 4 +- 6 files changed, 531 insertions(+), 310 deletions(-) create mode 100644 src/apm_cli/install/phases/resolve.py create mode 100644 src/apm_cli/install/phases/targets.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 49df5027..353295c4 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1096,7 +1096,7 @@ def _install_apm_dependencies( if not APM_DEPS_AVAILABLE: raise RuntimeError("APM dependency system not available") - from apm_cli.core.scope import InstallScope, get_deploy_root, get_apm_dir, get_modules_dir + from apm_cli.core.scope import InstallScope, get_deploy_root, get_apm_dir if scope is None: scope = InstallScope.PROJECT @@ -1108,307 +1108,76 @@ def _install_apm_dependencies( apm_dir = get_apm_dir(scope) # Check whether the project root itself has local .apm/ primitives (#714). - # Users should be able to keep root-level .apm/ rules alongside their apm.yml - # without creating a dummy sub-package stub. _root_has_local_primitives = _project_has_root_primitives(project_root) if not all_apm_deps and not _root_has_local_primitives: return InstallResult() - # T5: Check for existing lockfile - use locked versions for reproducible installs - from apm_cli.deps.lockfile import LockFile, get_lockfile_path - lockfile_path = get_lockfile_path(apm_dir) - existing_lockfile = None - lockfile_count = 0 - if lockfile_path.exists(): - existing_lockfile = LockFile.read(lockfile_path) - if existing_lockfile and existing_lockfile.dependencies: - lockfile_count = len(existing_lockfile.dependencies) - if logger: - if update_refs: - logger.verbose_detail(f"Loaded apm.lock.yaml for SHA comparison ({lockfile_count} dependencies)") - else: - logger.verbose_detail(f"Using apm.lock.yaml ({lockfile_count} locked dependencies)") - if logger.verbose: - for locked_dep in existing_lockfile.get_all_dependencies(): - _sha = locked_dep.resolved_commit[:8] if locked_dep.resolved_commit else "" - _ref = locked_dep.resolved_ref if hasattr(locked_dep, 'resolved_ref') and locked_dep.resolved_ref else "" - logger.lockfile_entry(locked_dep.get_unique_key(), ref=_ref, sha=_sha) - - apm_modules_dir = get_modules_dir(scope) - apm_modules_dir.mkdir(parents=True, exist_ok=True) - - # Use provided resolver or create new one if not in a CLI session context - if auth_resolver is None: - auth_resolver = AuthResolver() - - # Create downloader early so it can be used for transitive dependency resolution - downloader = GitHubPackageDownloader(auth_resolver=auth_resolver) - - # Track direct dependency keys so the download callback can distinguish them from transitive - direct_dep_keys = builtins.set(dep.get_unique_key() for dep in all_apm_deps) - - # Track paths already downloaded by the resolver callback to avoid re-downloading - # Maps dep_key -> resolved_commit (SHA or None) so the cached path can use it - callback_downloaded = {} - - # Collect transitive dep failures during resolution — they'll be routed to - # diagnostics after the DiagnosticCollector is created (later in the flow). - transitive_failures: list[tuple[str, str]] = [] # (dep_display, message) - - # Track dep keys that failed in download_callback so the main install loop - # skips them instead of re-trying and producing a duplicate error entry. - callback_failures: builtins.set = builtins.set() - - # Create a download callback for transitive dependency resolution - # This allows the resolver to fetch packages on-demand during tree building - def download_callback(dep_ref, modules_dir, parent_chain=""): - """Download a package during dependency resolution. - - Args: - dep_ref: The dependency to download. - modules_dir: Target apm_modules directory. - parent_chain: Human-readable breadcrumb (e.g. "root > mid") - showing which dependency path led to this transitive dep. - """ - install_path = dep_ref.get_install_path(modules_dir) - if install_path.exists(): - return install_path - try: - # Handle local packages: copy instead of git clone - if dep_ref.is_local and dep_ref.local_path: - if scope is InstallScope.USER: - # Cannot resolve local paths at user scope - callback_failures[dep_ref.get_unique_key()] = ( - f"local package '{dep_ref.local_path}' skipped at user scope" - ) - return None - result_path = _copy_local_package(dep_ref, install_path, project_root) - if result_path: - callback_downloaded[dep_ref.get_unique_key()] = None - return result_path - return None - - # T5: Use locked commit if available (reproducible installs) - locked_ref = None - if existing_lockfile: - locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key()) - if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached": - locked_ref = locked_dep.resolved_commit - - # Build a DependencyReference with the right ref to avoid lossy - # str() -> parse() round-trips (#382). - from dataclasses import replace as _dc_replace - if locked_ref and not update_refs: - download_dep = _dc_replace(dep_ref, reference=locked_ref) - else: - download_dep = dep_ref - - # Silent download - no progress display for transitive deps - result = downloader.download_package(download_dep, install_path) - # Capture resolved commit SHA for lockfile - resolved_sha = None - if result and hasattr(result, 'resolved_reference') and result.resolved_reference: - resolved_sha = result.resolved_reference.resolved_commit - callback_downloaded[dep_ref.get_unique_key()] = resolved_sha - return install_path - except Exception as e: - dep_display = dep_ref.get_display_name() - dep_key = dep_ref.get_unique_key() - is_direct = dep_key in direct_dep_keys - - # Distinguish direct vs transitive failure messages so users - # don't see a misleading "transitive dep" label for top-level deps. - if is_direct: - fail_msg = ( - f"Failed to download dependency " - f"{dep_ref.repo_url}: {e}" - ) - else: - chain_hint = f" (via {parent_chain})" if parent_chain else "" - fail_msg = ( - f"Failed to resolve transitive dep " - f"{dep_ref.repo_url}{chain_hint}: {e}" - ) - - # Verbose: inline detail - if logger: - logger.verbose_detail(f" {fail_msg}") - elif verbose: - _rich_error(f" └─ {fail_msg}") - # Collect for deferred diagnostics summary (always, even non-verbose) - callback_failures.add(dep_key) - transitive_failures.append((dep_display, fail_msg)) - return None - - # Resolve dependencies with transitive download support - resolver = APMDependencyResolver( - apm_modules_dir=apm_modules_dir, - download_callback=download_callback + # ------------------------------------------------------------------ + # Build InstallContext from function args + computed state + # ------------------------------------------------------------------ + from apm_cli.install.context import InstallContext + + ctx = InstallContext( + project_root=project_root, + apm_dir=apm_dir, + apm_package=apm_package, + update_refs=update_refs, + verbose=verbose, + only_packages=only_packages, + force=force, + parallel_downloads=parallel_downloads, + logger=logger, + scope=scope, + auth_resolver=auth_resolver, + target_override=target, + marketplace_provenance=marketplace_provenance, + all_apm_deps=all_apm_deps, + root_has_local_primitives=_root_has_local_primitives, ) - try: - dependency_graph = resolver.resolve_dependencies(apm_dir) + # ------------------------------------------------------------------ + # Phase 1: Resolve dependencies + # ------------------------------------------------------------------ + from apm_cli.install.phases import resolve as _resolve_phase + _resolve_phase.run(ctx) - # Verbose: show resolved tree summary + if not ctx.deps_to_install and not ctx.root_has_local_primitives: if logger: - tree = dependency_graph.dependency_tree - direct_count = len(tree.get_nodes_at_depth(1)) - transitive_count = len(tree.nodes) - direct_count - if transitive_count > 0: - logger.verbose_detail( - f"Resolved dependency tree: {direct_count} direct + " - f"{transitive_count} transitive deps (max depth {tree.max_depth})" - ) - for node in tree.nodes.values(): - if node.depth > 1: - logger.verbose_detail( - f" {node.get_ancestor_chain()}" - ) - else: - logger.verbose_detail(f"Resolved {direct_count} direct dependencies (no transitive)") - - # Check for circular dependencies - if dependency_graph.circular_dependencies: - if logger: - logger.error("Circular dependencies detected:") - for circular in dependency_graph.circular_dependencies: - cycle_path = " -> ".join(circular.cycle_path) - if logger: - logger.error(f" {cycle_path}") - raise RuntimeError("Cannot install packages with circular dependencies") - - # Get flattened dependencies for installation - flat_deps = dependency_graph.flattened_dependencies - deps_to_install = flat_deps.get_installation_list() - - # If specific packages were requested, filter to only those - # **and their full transitive dependency subtrees** so that - # sub-deps (and their MCP servers) are installed and recorded - # in the lockfile. - if only_packages: - # Build identity set from user-supplied package specs. - # Accepts any input form: git URLs, FQDN, shorthand. - only_identities = builtins.set() - for p in only_packages: - try: - ref = DependencyReference.parse(p) - only_identities.add(ref.get_identity()) - except Exception: - only_identities.add(p) - - # Expand the set to include transitive descendants of the - # requested packages so their MCP servers, primitives, etc. - # are correctly installed and written to the lockfile. - tree = dependency_graph.dependency_tree - - def _collect_descendants(node, visited=None): - """Walk the tree and add every child identity (cycle-safe).""" - if visited is None: - visited = builtins.set() - for child in node.children: - identity = child.dependency_ref.get_identity() - if identity not in visited: - visited.add(identity) - only_identities.add(identity) - _collect_descendants(child, visited) - - for node in tree.nodes.values(): - if node.dependency_ref.get_identity() in only_identities: - _collect_descendants(node) - - deps_to_install = [ - dep for dep in deps_to_install - if dep.get_identity() in only_identities - ] - - if not deps_to_install and not _root_has_local_primitives: - if logger: - logger.nothing_to_install() - return InstallResult() - - # ------------------------------------------------------------------ - # Orphan detection: packages in lockfile no longer in the manifest. - # Only relevant for a full install (not apm install ). - # We compute this NOW, before the download loop, so we know which old - # lockfile entries to remove from the merge and which deployed files - # to clean up after the loop. - # ------------------------------------------------------------------ - intended_dep_keys: builtins.set = builtins.set( - d.get_unique_key() for d in deps_to_install - ) - - # apm_modules directory already created above - - # Auto-detect target for integration (same logic as compile) - from apm_cli.core.target_detection import ( - detect_target, - get_target_description, - ) - - # Get config target from apm.yml if available - config_target = apm_package.target - - # Resolve effective explicit target: CLI --target wins, then apm.yml - _explicit = target or config_target or None - - # Determine active targets. When --target or apm.yml target is set - # the user's choice wins. Otherwise auto-detect from existing dirs, - # falling back to copilot when nothing is found. - from apm_cli.integration.targets import resolve_targets as _resolve_targets - - _is_user = scope is InstallScope.USER - _targets = _resolve_targets( - project_root, user_scope=_is_user, explicit_target=_explicit, - ) - - # Log target detection results - if logger and _targets: - _scope_label = "global" if _is_user else "project" - _target_names = ", ".join( - f"{t.name} (~/{t.root_dir}/)" - if _is_user else t.name - for t in _targets - ) - logger.verbose_detail( - f"Active {_scope_label} targets: {_target_names}" - ) - if _is_user: - from apm_cli.deps.lockfile import get_lockfile_path - logger.verbose_detail( - f"Lockfile: {get_lockfile_path(apm_dir)}" - ) - - for _t in _targets: - if not _t.auto_create: - continue - _root = _t.root_dir - _target_dir = project_root / _root - if not _target_dir.exists(): - _target_dir.mkdir(parents=True, exist_ok=True) - if logger: - logger.verbose_detail( - f"Created {_root}/ ({_t.name} target)" - ) + logger.nothing_to_install() + return InstallResult() - detected_target, detection_reason = detect_target( - project_root=project_root, - explicit_target=_explicit, - config_target=config_target, - ) + try: + # -------------------------------------------------------------- + # Phase 2: Target detection + integrator initialization + # -------------------------------------------------------------- + from apm_cli.install.phases import targets as _targets_phase + _targets_phase.run(ctx) + + # -------------------------------------------------------------- + # Seam: read phase outputs into locals for remaining code. + # This minimises diff below -- subsequent phases (download, + # integrate, cleanup, lockfile) continue using bare-name locals. + # Future S-phases will fold them into the context one by one. + # -------------------------------------------------------------- + deps_to_install = ctx.deps_to_install + intended_dep_keys = ctx.intended_dep_keys + dependency_graph = ctx.dependency_graph + existing_lockfile = ctx.existing_lockfile + lockfile_path = ctx.lockfile_path + apm_modules_dir = ctx.apm_modules_dir + downloader = ctx.downloader + callback_downloaded = ctx.callback_downloaded + callback_failures = ctx.callback_failures + transitive_failures = ctx.transitive_failures + _targets = ctx.targets + prompt_integrator = ctx.integrators["prompt"] + agent_integrator = ctx.integrators["agent"] + skill_integrator = ctx.integrators["skill"] + command_integrator = ctx.integrators["command"] + hook_integrator = ctx.integrators["hook"] + instruction_integrator = ctx.integrators["instruction"] - # Initialize integrators - prompt_integrator = PromptIntegrator() - agent_integrator = AgentIntegrator() - from apm_cli.integration.skill_integrator import SkillIntegrator, should_install_skill - from apm_cli.integration.command_integrator import CommandIntegrator - from apm_cli.integration.hook_integrator import HookIntegrator - from apm_cli.integration.instruction_integrator import InstructionIntegrator - - skill_integrator = SkillIntegrator() - command_integrator = CommandIntegrator() - hook_integrator = HookIntegrator() - instruction_integrator = InstructionIntegrator() diagnostics = DiagnosticCollector(verbose=verbose) # Drain transitive failures collected during resolution into diagnostics diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py index 43fb2ed9..294b84db 100644 --- a/src/apm_cli/install/context.py +++ b/src/apm_cli/install/context.py @@ -2,42 +2,81 @@ Each phase is a function ``def run(ctx: InstallContext) -> None`` that reads the inputs already populated by earlier phases and writes its own outputs to -the context. Keeping shared state on a single typed object turns implicit -shared lexical scope (the legacy 1444-line `_install_apm_dependencies`) into -explicit data flow that is easy to audit and to test phase-by-phase. +the context. Keeping shared state on a single typed object turns implicit +shared lexical scope (the legacy 1444-line ``_install_apm_dependencies``) +into explicit data flow that is easy to audit and to test phase-by-phase. Fields are added to this dataclass incrementally as phases are extracted from -the legacy entry point. A field belongs here if and only if it is read or -written by more than one phase. Phase-local state should stay local. +the legacy entry point. A field belongs here if and only if it is read or +written by more than one phase. Phase-local state should stay local. """ from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple @dataclass class InstallContext: """State shared across install pipeline phases. - Currently a stub. Fields are added by the phase extractions in P1 and P2 - of the install.py modularization refactor. - Required-on-construction fields go above the ``field(default=...)`` barrier; outputs accumulated by phases use ``field(default_factory=...)``. + + Fields are grouped by the phase that first populates them. A trailing + comment ``# `` marks the originating phase for auditability. """ + # ------------------------------------------------------------------ + # Required on construction (caller supplies before any phase runs) + # ------------------------------------------------------------------ project_root: Path apm_dir: Path + # ------------------------------------------------------------------ + # Inputs: populated by the caller from CLI args / APMPackage + # ------------------------------------------------------------------ + apm_package: Any = None # APMPackage + update_refs: bool = False + scope: Any = None # InstallScope (defaults to PROJECT) + auth_resolver: Any = None # AuthResolver + marketplace_provenance: Optional[Dict[str, Any]] = None + parallel_downloads: int = 4 + logger: Any = None # InstallLogger + target_override: Optional[str] = None # CLI --target value + dry_run: bool = False force: bool = False verbose: bool = False dev: bool = False only_packages: Optional[List[str]] = None + # ------------------------------------------------------------------ + # Resolve phase outputs + # ------------------------------------------------------------------ + all_apm_deps: List[Any] = field(default_factory=list) # resolve + root_has_local_primitives: bool = False # resolve + deps_to_install: List[Any] = field(default_factory=list) # resolve + dependency_graph: Any = None # resolve + existing_lockfile: Any = None # resolve + lockfile_path: Optional[Path] = None # resolve + apm_modules_dir: Optional[Path] = None # resolve + downloader: Any = None # resolve (GitHubPackageDownloader) + callback_downloaded: Dict[str, Any] = field(default_factory=dict) # resolve + callback_failures: Set[str] = field(default_factory=set) # resolve + transitive_failures: List[Tuple[str, str]] = field(default_factory=list) # resolve + + # ------------------------------------------------------------------ + # Targets phase outputs + # ------------------------------------------------------------------ + targets: List[Any] = field(default_factory=list) # targets + integrators: Dict[str, Any] = field(default_factory=dict) # targets + + # ------------------------------------------------------------------ + # Downstream phase accumulators (written by integrate, read by cleanup/lockfile) + # ------------------------------------------------------------------ intended_dep_keys: Set[str] = field(default_factory=set) package_deployed_files: Dict[str, List[str]] = field(default_factory=dict) package_types: Dict[str, Dict[str, Any]] = field(default_factory=dict) diff --git a/src/apm_cli/install/phases/resolve.py b/src/apm_cli/install/phases/resolve.py new file mode 100644 index 00000000..8feb3050 --- /dev/null +++ b/src/apm_cli/install/phases/resolve.py @@ -0,0 +1,313 @@ +"""Dependency resolution phase. + +Reads ``ctx.apm_package``, ``ctx.update_refs``, ``ctx.scope``, etc.; +populates ``ctx.deps_to_install``, ``ctx.intended_dep_keys``, +``ctx.dependency_graph``, ``ctx.existing_lockfile``, and several ancillary +fields consumed by later phases (download, integrate, cleanup, lockfile). + +This is the first phase of the install pipeline. It covers: + +1. Lockfile loading (``apm.lock.yaml``) +2. ``apm_modules/`` directory creation +3. Auth resolver defaulting + downloader construction +4. Transitive dependency resolution via ``APMDependencyResolver`` +5. ``--only`` filtering (restrict to named packages + their subtrees) +6. ``intended_dep_keys`` computation (the manifest-intent set used by + orphan cleanup in a later phase) +""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: "InstallContext") -> None: + """Execute the resolve phase. + + On return every field listed in the *Resolve phase outputs* section of + :class:`~apm_cli.install.context.InstallContext` is populated. + """ + from apm_cli.core.auth import AuthResolver + from apm_cli.core.scope import InstallScope, get_modules_dir + from apm_cli.deps.apm_resolver import APMDependencyResolver + from apm_cli.deps import github_downloader as _ghd_mod + from apm_cli.deps.lockfile import LockFile, get_lockfile_path + from apm_cli.install.phases.local_content import _copy_local_package + from apm_cli.models.apm_package import DependencyReference + from apm_cli.utils.console import _rich_error + + # ------------------------------------------------------------------ + # 1. Lockfile loading + # ------------------------------------------------------------------ + lockfile_path = get_lockfile_path(ctx.apm_dir) + ctx.lockfile_path = lockfile_path + existing_lockfile = None + lockfile_count = 0 + if lockfile_path.exists(): + existing_lockfile = LockFile.read(lockfile_path) + if existing_lockfile and existing_lockfile.dependencies: + lockfile_count = len(existing_lockfile.dependencies) + if ctx.logger: + if ctx.update_refs: + ctx.logger.verbose_detail( + f"Loaded apm.lock.yaml for SHA comparison ({lockfile_count} dependencies)" + ) + else: + ctx.logger.verbose_detail( + f"Using apm.lock.yaml ({lockfile_count} locked dependencies)" + ) + if ctx.logger.verbose: + for locked_dep in existing_lockfile.get_all_dependencies(): + _sha = ( + locked_dep.resolved_commit[:8] + if locked_dep.resolved_commit + else "" + ) + _ref = ( + locked_dep.resolved_ref + if hasattr(locked_dep, "resolved_ref") + and locked_dep.resolved_ref + else "" + ) + ctx.logger.lockfile_entry( + locked_dep.get_unique_key(), ref=_ref, sha=_sha + ) + ctx.existing_lockfile = existing_lockfile + + # ------------------------------------------------------------------ + # 2. apm_modules directory + # ------------------------------------------------------------------ + apm_modules_dir = get_modules_dir(ctx.scope) + apm_modules_dir.mkdir(parents=True, exist_ok=True) + ctx.apm_modules_dir = apm_modules_dir + + # ------------------------------------------------------------------ + # 3. Auth resolver + downloader + # ------------------------------------------------------------------ + if ctx.auth_resolver is None: + ctx.auth_resolver = AuthResolver() + + downloader = _ghd_mod.GitHubPackageDownloader(auth_resolver=ctx.auth_resolver) + ctx.downloader = downloader + + # ------------------------------------------------------------------ + # 4. Tracking variables (phase-local except where noted) + # ------------------------------------------------------------------ + # direct_dep_keys is phase-local (only read inside download_callback) + direct_dep_keys = builtins.set( + dep.get_unique_key() for dep in ctx.all_apm_deps + ) + # These three escape to later phases via ctx + callback_downloaded: builtins.dict = {} + transitive_failures: builtins.list = [] + callback_failures: builtins.set = builtins.set() + + # ------------------------------------------------------------------ + # 5. Download callback for transitive resolution + # ------------------------------------------------------------------ + # Capture frequently-used ctx fields as locals for the closure. + # This matches the original code's closure over function-level locals. + scope = ctx.scope + project_root = ctx.project_root + update_refs = ctx.update_refs + logger = ctx.logger + verbose = ctx.verbose + + def download_callback(dep_ref, modules_dir, parent_chain=""): + """Download a package during dependency resolution. + + Args: + dep_ref: The dependency to download. + modules_dir: Target apm_modules directory. + parent_chain: Human-readable breadcrumb (e.g. "root > mid") + showing which dependency path led to this transitive dep. + """ + install_path = dep_ref.get_install_path(modules_dir) + if install_path.exists(): + return install_path + try: + # Handle local packages: copy instead of git clone + if dep_ref.is_local and dep_ref.local_path: + if scope is InstallScope.USER: + # Cannot resolve local paths at user scope + callback_failures[dep_ref.get_unique_key()] = ( + f"local package '{dep_ref.local_path}' skipped at user scope" + ) + return None + result_path = _copy_local_package( + dep_ref, install_path, project_root + ) + if result_path: + callback_downloaded[dep_ref.get_unique_key()] = None + return result_path + return None + + # T5: Use locked commit if available (reproducible installs) + locked_ref = None + if existing_lockfile: + locked_dep = existing_lockfile.get_dependency( + dep_ref.get_unique_key() + ) + if ( + locked_dep + and locked_dep.resolved_commit + and locked_dep.resolved_commit != "cached" + ): + locked_ref = locked_dep.resolved_commit + + # Build a DependencyReference with the right ref to avoid lossy + # str() -> parse() round-trips (#382). + from dataclasses import replace as _dc_replace + + if locked_ref and not update_refs: + download_dep = _dc_replace(dep_ref, reference=locked_ref) + else: + download_dep = dep_ref + + # Silent download - no progress display for transitive deps + result = downloader.download_package(download_dep, install_path) + # Capture resolved commit SHA for lockfile + resolved_sha = None + if ( + result + and hasattr(result, "resolved_reference") + and result.resolved_reference + ): + resolved_sha = result.resolved_reference.resolved_commit + callback_downloaded[dep_ref.get_unique_key()] = resolved_sha + return install_path + except Exception as e: + dep_display = dep_ref.get_display_name() + dep_key = dep_ref.get_unique_key() + is_direct = dep_key in direct_dep_keys + + # Distinguish direct vs transitive failure messages so users + # don't see a misleading "transitive dep" label for top-level deps. + if is_direct: + fail_msg = ( + f"Failed to download dependency " + f"{dep_ref.repo_url}: {e}" + ) + else: + chain_hint = ( + f" (via {parent_chain})" if parent_chain else "" + ) + fail_msg = ( + f"Failed to resolve transitive dep " + f"{dep_ref.repo_url}{chain_hint}: {e}" + ) + + # Verbose: inline detail + if logger: + logger.verbose_detail(f" {fail_msg}") + elif verbose: + _rich_error(f" \u2514\u2500 {fail_msg}") + # Collect for deferred diagnostics summary (always, even non-verbose) + callback_failures.add(dep_key) + transitive_failures.append((dep_display, fail_msg)) + return None + + # ------------------------------------------------------------------ + # 6. Resolver creation + dependency resolution + # ------------------------------------------------------------------ + resolver = APMDependencyResolver( + apm_modules_dir=apm_modules_dir, + download_callback=download_callback, + ) + + dependency_graph = resolver.resolve_dependencies(ctx.apm_dir) + ctx.dependency_graph = dependency_graph + + # Verbose: show resolved tree summary + if ctx.logger: + tree = dependency_graph.dependency_tree + direct_count = len(tree.get_nodes_at_depth(1)) + transitive_count = len(tree.nodes) - direct_count + if transitive_count > 0: + ctx.logger.verbose_detail( + f"Resolved dependency tree: {direct_count} direct + " + f"{transitive_count} transitive deps (max depth {tree.max_depth})" + ) + for node in tree.nodes.values(): + if node.depth > 1: + ctx.logger.verbose_detail( + f" {node.get_ancestor_chain()}" + ) + else: + ctx.logger.verbose_detail( + f"Resolved {direct_count} direct dependencies (no transitive)" + ) + + # Check for circular dependencies + if dependency_graph.circular_dependencies: + if ctx.logger: + ctx.logger.error("Circular dependencies detected:") + for circular in dependency_graph.circular_dependencies: + cycle_path = " -> ".join(circular.cycle_path) + if ctx.logger: + ctx.logger.error(f" {cycle_path}") + raise RuntimeError("Cannot install packages with circular dependencies") + + # Get flattened dependencies for installation + flat_deps = dependency_graph.flattened_dependencies + deps_to_install = flat_deps.get_installation_list() + + # ------------------------------------------------------------------ + # 7. --only filtering + # ------------------------------------------------------------------ + if ctx.only_packages: + # Build identity set from user-supplied package specs. + # Accepts any input form: git URLs, FQDN, shorthand. + only_identities = builtins.set() + for p in ctx.only_packages: + try: + ref = DependencyReference.parse(p) + only_identities.add(ref.get_identity()) + except Exception: + only_identities.add(p) + + # Expand the set to include transitive descendants of the + # requested packages so their MCP servers, primitives, etc. + # are correctly installed and written to the lockfile. + tree = dependency_graph.dependency_tree + + def _collect_descendants(node, visited=None): + """Walk the tree and add every child identity (cycle-safe).""" + if visited is None: + visited = builtins.set() + for child in node.children: + identity = child.dependency_ref.get_identity() + if identity not in visited: + visited.add(identity) + only_identities.add(identity) + _collect_descendants(child, visited) + + for node in tree.nodes.values(): + if node.dependency_ref.get_identity() in only_identities: + _collect_descendants(node) + + deps_to_install = [ + dep + for dep in deps_to_install + if dep.get_identity() in only_identities + ] + + ctx.deps_to_install = deps_to_install + + # ------------------------------------------------------------------ + # 8. Orphan detection: intended_dep_keys + # ------------------------------------------------------------------ + ctx.intended_dep_keys = builtins.set( + d.get_unique_key() for d in deps_to_install + ) + + # ------------------------------------------------------------------ + # Write ancillary state to ctx for later phases + # ------------------------------------------------------------------ + ctx.callback_downloaded = callback_downloaded + ctx.callback_failures = callback_failures + ctx.transitive_failures = transitive_failures diff --git a/src/apm_cli/install/phases/targets.py b/src/apm_cli/install/phases/targets.py new file mode 100644 index 00000000..2a73c7dd --- /dev/null +++ b/src/apm_cli/install/phases/targets.py @@ -0,0 +1,100 @@ +"""Target detection and integrator initialization phase. + +Reads ``ctx.target_override``, ``ctx.apm_package``, ``ctx.scope``, +``ctx.project_root``; populates ``ctx.targets`` (list of +:class:`~apm_cli.integration.targets.TargetProfile`) and +``ctx.integrators`` (dict of per-primitive-type integrator instances). + +This is the second phase of the install pipeline, running after resolve. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: "InstallContext") -> None: + """Execute the targets phase. + + On return ``ctx.targets`` and ``ctx.integrators`` are populated. + """ + from apm_cli.core.scope import InstallScope + from apm_cli.core.target_detection import ( + detect_target, + ) + from apm_cli.integration import AgentIntegrator, PromptIntegrator + from apm_cli.integration.command_integrator import CommandIntegrator + from apm_cli.integration.hook_integrator import HookIntegrator + from apm_cli.integration.instruction_integrator import InstructionIntegrator + from apm_cli.integration.skill_integrator import SkillIntegrator + from apm_cli.integration.targets import resolve_targets as _resolve_targets + + # Get config target from apm.yml if available + config_target = ctx.apm_package.target + + # Resolve effective explicit target: CLI --target wins, then apm.yml + _explicit = ctx.target_override or config_target or None + + # Determine active targets. When --target or apm.yml target is set + # the user's choice wins. Otherwise auto-detect from existing dirs, + # falling back to copilot when nothing is found. + _is_user = ctx.scope is InstallScope.USER + _targets = _resolve_targets( + ctx.project_root, + user_scope=_is_user, + explicit_target=_explicit, + ) + + # Log target detection results + if ctx.logger and _targets: + _scope_label = "global" if _is_user else "project" + _target_names = ", ".join( + f"{t.name} (~/{t.root_dir}/)" if _is_user else t.name + for t in _targets + ) + ctx.logger.verbose_detail( + f"Active {_scope_label} targets: {_target_names}" + ) + if _is_user: + from apm_cli.deps.lockfile import get_lockfile_path + + ctx.logger.verbose_detail( + f"Lockfile: {get_lockfile_path(ctx.apm_dir)}" + ) + + for _t in _targets: + if not _t.auto_create: + continue + _root = _t.root_dir + _target_dir = ctx.project_root / _root + if not _target_dir.exists(): + _target_dir.mkdir(parents=True, exist_ok=True) + if ctx.logger: + ctx.logger.verbose_detail( + f"Created {_root}/ ({_t.name} target)" + ) + + # Legacy detect_target call -- return values are not consumed by any + # downstream code but the call is preserved for behaviour parity with + # the pre-refactor mega-function. + detect_target( + project_root=ctx.project_root, + explicit_target=_explicit, + config_target=config_target, + ) + + # ------------------------------------------------------------------ + # Initialize integrators + # ------------------------------------------------------------------ + ctx.targets = _targets + ctx.integrators = { + "prompt": PromptIntegrator(), + "agent": AgentIntegrator(), + "skill": SkillIntegrator(), + "command": CommandIntegrator(), + "hook": HookIntegrator(), + "instruction": InstructionIntegrator(), + } diff --git a/tests/integration/test_selective_install_mcp.py b/tests/integration/test_selective_install_mcp.py index 1db66038..11c40745 100644 --- a/tests/integration/test_selective_install_mcp.py +++ b/tests/integration/test_selective_install_mcp.py @@ -117,7 +117,7 @@ class TestSelectiveInstallTransitiveMCPIntegration: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.commands.install._validate_package_exists", return_value=True) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_lockfile_records_transitive_mcp_servers( self, mock_dl_cls, mock_mcp_install, mock_validate, mock_updates, cli_env ): @@ -145,7 +145,7 @@ def test_lockfile_records_transitive_mcp_servers( @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.commands.install._validate_package_exists", return_value=True) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_install_mcp_receives_transitive_deps( self, mock_dl_cls, mock_mcp_install, mock_validate, mock_updates, cli_env ): @@ -172,7 +172,7 @@ class TestDeepChainIntegration: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.commands.install._validate_package_exists", return_value=True) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_deep_chain_mcp_in_lockfile( self, mock_dl_cls, mock_mcp_install, mock_validate, mock_updates, tmp_path ): @@ -222,7 +222,7 @@ class TestDiamondDependencyIntegration: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.commands.install._validate_package_exists", return_value=True) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_diamond_mcp_in_lockfile( self, mock_dl_cls, mock_mcp_install, mock_validate, mock_updates, tmp_path ): @@ -276,7 +276,7 @@ class TestMultiPackageSelectiveInstallIntegration: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.commands.install._validate_package_exists", return_value=True) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_multiple_packages_mcp_merged( self, mock_dl_cls, mock_mcp_install, mock_validate, mock_updates, tmp_path ): @@ -331,7 +331,7 @@ class TestFullInstallTransitiveMCPIntegration: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_full_install_collects_transitive_mcp( self, mock_dl_cls, mock_mcp_install, mock_updates, cli_env ): @@ -354,7 +354,7 @@ class TestStaleRemovalAfterUpdate: @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) @patch("apm_cli.integration.mcp_integrator.MCPIntegrator.install", return_value=0) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_stale_mcp_removed_on_update( self, mock_dl_cls, mock_mcp_install, mock_updates, tmp_path ): @@ -414,7 +414,7 @@ class TestNoMCPWhenOnlyAPM: lockfile mcp_servers must be preserved.""" @patch("apm_cli.commands._helpers.check_for_updates", return_value=None) - @patch("apm_cli.commands.install.GitHubPackageDownloader") + @patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") def test_only_apm_preserves_mcp_servers( self, mock_dl_cls, mock_updates, cli_env ): diff --git a/tests/unit/test_install_command.py b/tests/unit/test_install_command.py index 98bb1923..a27ecda3 100644 --- a/tests/unit/test_install_command.py +++ b/tests/unit/test_install_command.py @@ -511,7 +511,7 @@ def test_direct_dep_failure_says_download_dependency(self, tmp_path, monkeypatch apm_package = APMPackage.from_apm_yml(tmp_path / "apm.yml") # Patch the downloader to always fail - with patch("apm_cli.commands.install.GitHubPackageDownloader") as MockDownloader: + with patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") as MockDownloader: mock_dl = MockDownloader.return_value mock_dl.download_package.side_effect = RuntimeError("auth failed") @@ -560,7 +560,7 @@ def test_callback_failure_not_duplicated_in_main_loop(self, tmp_path, monkeypatc })) apm_package = APMPackage.from_apm_yml(tmp_path / "apm.yml") - with patch("apm_cli.commands.install.GitHubPackageDownloader") as MockDownloader: + with patch("apm_cli.deps.github_downloader.GitHubPackageDownloader") as MockDownloader: mock_dl = MockDownloader.return_value mock_dl.download_package.side_effect = RuntimeError("auth failed") From 24315a04ae4550928c58d2b84c4128adf7efcb19 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:00:18 +0200 Subject: [PATCH 07/28] refactor(install): P2.S2 -- extract parallel download phase Extract the Phase 4 (#171) parallel pre-download block from _install_apm_dependencies into src/apm_cli/install/phases/download.py. The phase reads ctx.deps_to_install, ctx.existing_lockfile, ctx.update_refs, ctx.parallel_downloads, ctx.apm_modules_dir, ctx.downloader, and ctx.callback_downloaded (all already on InstallContext from S1). New InstallContext fields added: - pre_download_results: Dict[str, Any] (dep_key -> PackageInfo) - pre_downloaded_keys: Set[str] install.py retains bridge aliases (_pre_download_results, _pre_downloaded_keys) that read from ctx so downstream code at the sequential integration loop (~lines 1483, 1767-1768) is untouched. concurrent.futures import removed from install.py (no longer used there). rich.progress imports in download.py are local to the if-block. install.py LOC: 2282 -> 2211 (-71) New: phases/download.py 132 LOC Rubber-duck findings: CLEAN -- No test patches invalidated (zero patches target download symbols at apm_cli.commands.install.X; detect_ref_change/build_download_ref are imported from apm_cli.drift in both old and new code). CLEAN -- Progress UI lifecycle preserved (transient context manager). CLEAN -- Error-swallowing semantic preserved (except Exception: silent). CLEAN -- parallel_downloads=0 skip path works correctly. LATENT BUG (preserved, not fixed) -- HEAD-skip path silently swallows GitPython ImportError/corruption, causing unnecessary re-downloads. Verification gates (all green): 1. Import check: download.py + install.py load cleanly 2. Targeted suite: 102 passed, 1 skipped 3. Full unit suite: 3972 passed, 1 skipped 4. Cleanup/prune/orphan suite: 32 passed Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 89 ++-------------- src/apm_cli/install/context.py | 6 ++ src/apm_cli/install/phases/download.py | 135 +++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 80 deletions(-) create mode 100644 src/apm_cli/install/phases/download.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 353295c4..4d9b0c16 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1276,87 +1276,16 @@ def _install_apm_dependencies( installed_count = 0 unpinned_count = 0 - # Phase 4 (#171): Parallel package downloads using ThreadPoolExecutor - # Pre-download all non-cached packages in parallel for wall-clock speedup. - # Results are stored and consumed by the sequential integration loop below. - from concurrent.futures import ThreadPoolExecutor, as_completed as _futures_completed - - _pre_download_results = {} # dep_key -> PackageInfo - _need_download = [] - for _pd_ref in deps_to_install: - _pd_key = _pd_ref.get_unique_key() - _pd_path = (apm_modules_dir / _pd_ref.alias) if _pd_ref.alias else _pd_ref.get_install_path(apm_modules_dir) - # Skip local packages — they are copied, not downloaded - if _pd_ref.is_local: - continue - # Skip if already downloaded during BFS resolution - if _pd_key in callback_downloaded: - continue - # Detect if manifest ref changed from what's recorded in the lockfile. - # detect_ref_change() handles all transitions including None→ref. - _pd_locked_chk = ( - existing_lockfile.get_dependency(_pd_key) - if existing_lockfile - else None - ) - _pd_ref_changed = detect_ref_change( - _pd_ref, _pd_locked_chk, update_refs=update_refs - ) - # Skip if lockfile SHA matches local HEAD. - # Normal mode: only when the ref hasn't changed in the manifest. - # Update mode: defer to the sequential loop which resolves the - # remote ref and compares -- if unchanged, the download is skipped - # entirely; if changed, it falls back to sequential download. - if (_pd_path.exists() and _pd_locked_chk - and _pd_locked_chk.resolved_commit - and _pd_locked_chk.resolved_commit != "cached" - and (update_refs or not _pd_ref_changed)): - try: - from git import Repo as _PDGitRepo - if _PDGitRepo(_pd_path).head.commit.hexsha == _pd_locked_chk.resolved_commit: - continue - except Exception: - pass - # Build download ref (use locked commit for reproducibility). - # build_download_ref() uses the manifest ref when ref_changed is True. - _pd_dlref = build_download_ref( - _pd_ref, existing_lockfile, update_refs=update_refs, ref_changed=_pd_ref_changed - ) - _need_download.append((_pd_ref, _pd_path, _pd_dlref)) - - if _need_download and parallel_downloads > 0: - with Progress( - SpinnerColumn(), - TextColumn("[cyan]{task.description}[/cyan]"), - BarColumn(), - TaskProgressColumn(), - transient=True, - ) as _dl_progress: - _max_workers = min(parallel_downloads, len(_need_download)) - with ThreadPoolExecutor(max_workers=_max_workers) as _executor: - _futures = {} - for _pd_ref, _pd_path, _pd_dlref in _need_download: - _pd_disp = str(_pd_ref) if _pd_ref.is_virtual else _pd_ref.repo_url - _pd_short = _pd_disp.split("/")[-1] if "/" in _pd_disp else _pd_disp - _pd_tid = _dl_progress.add_task(description=f"Fetching {_pd_short}", total=None) - _pd_fut = _executor.submit( - downloader.download_package, _pd_dlref, _pd_path, - progress_task_id=_pd_tid, progress_obj=_dl_progress, - ) - _futures[_pd_fut] = (_pd_ref, _pd_tid, _pd_disp) - for _pd_fut in _futures_completed(_futures): - _pd_ref, _pd_tid, _pd_disp = _futures[_pd_fut] - _pd_key = _pd_ref.get_unique_key() - try: - _pd_info = _pd_fut.result() - _pre_download_results[_pd_key] = _pd_info - _dl_progress.update(_pd_tid, visible=False) - _dl_progress.refresh() - except Exception: - _dl_progress.remove_task(_pd_tid) - # Silent: sequential loop below will retry and report errors + # -------------------------------------------------------------- + # Phase 4 (#171): Parallel package pre-download + # -------------------------------------------------------------- + from apm_cli.install.phases import download as _download_phase + _download_phase.run(ctx) - _pre_downloaded_keys = builtins.set(_pre_download_results.keys()) + # Bridge: alias ctx outputs into locals so downstream code + # (lines ~1554, ~1838-1839) continues working without changes. + _pre_download_results = ctx.pre_download_results + _pre_downloaded_keys = ctx.pre_downloaded_keys # Create progress display for sequential integration # Reuse the shared auth_resolver (already created in this invocation) so diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py index 294b84db..be889a57 100644 --- a/src/apm_cli/install/context.py +++ b/src/apm_cli/install/context.py @@ -74,6 +74,12 @@ class InstallContext: targets: List[Any] = field(default_factory=list) # targets integrators: Dict[str, Any] = field(default_factory=dict) # targets + # ------------------------------------------------------------------ + # Download phase outputs + # ------------------------------------------------------------------ + pre_download_results: Dict[str, Any] = field(default_factory=dict) # download + pre_downloaded_keys: Set[str] = field(default_factory=set) # download + # ------------------------------------------------------------------ # Downstream phase accumulators (written by integrate, read by cleanup/lockfile) # ------------------------------------------------------------------ diff --git a/src/apm_cli/install/phases/download.py b/src/apm_cli/install/phases/download.py new file mode 100644 index 00000000..1e820359 --- /dev/null +++ b/src/apm_cli/install/phases/download.py @@ -0,0 +1,135 @@ +"""Parallel package pre-download phase. + +Reads ``ctx.deps_to_install``, ``ctx.existing_lockfile``, +``ctx.update_refs``, ``ctx.parallel_downloads``, ``ctx.apm_modules_dir``, +``ctx.downloader``, and ``ctx.callback_downloaded``; populates +``ctx.pre_download_results`` (dep_key -> PackageInfo) and +``ctx.pre_downloaded_keys`` (set of dep_keys that were pre-downloaded). + +This is Phase 4 (#171) of the install pipeline. Packages that were already +fetched during BFS resolution (callback_downloaded), local packages, and +those whose lockfile SHA matches the on-disk HEAD are skipped. Remaining +packages are fetched in parallel via :class:`ThreadPoolExecutor` with a Rich +progress UI. Failures are silently swallowed -- the sequential integration +loop is the source of truth for error reporting. +""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: "InstallContext") -> None: + """Execute the parallel download phase. + + On return ``ctx.pre_download_results`` and ``ctx.pre_downloaded_keys`` + are populated. + """ + # Module-attribute access for late-patchability (same pattern as + # resolve.py). detect_ref_change / build_download_ref live in + # apm_cli.drift and tests import them from there, so direct import + # is safe here -- no test patches at apm_cli.commands.install.X. + from apm_cli.drift import build_download_ref, detect_ref_change + + deps_to_install = ctx.deps_to_install + existing_lockfile = ctx.existing_lockfile + update_refs = ctx.update_refs + parallel_downloads = ctx.parallel_downloads + apm_modules_dir = ctx.apm_modules_dir + downloader = ctx.downloader + callback_downloaded = ctx.callback_downloaded + + # Phase 4 (#171): Parallel package downloads using ThreadPoolExecutor + # Pre-download all non-cached packages in parallel for wall-clock speedup. + # Results are stored and consumed by the sequential integration loop below. + from concurrent.futures import ThreadPoolExecutor, as_completed as _futures_completed + + _pre_download_results = {} # dep_key -> PackageInfo + _need_download = [] + for _pd_ref in deps_to_install: + _pd_key = _pd_ref.get_unique_key() + _pd_path = (apm_modules_dir / _pd_ref.alias) if _pd_ref.alias else _pd_ref.get_install_path(apm_modules_dir) + # Skip local packages \u2014 they are copied, not downloaded + if _pd_ref.is_local: + continue + # Skip if already downloaded during BFS resolution + if _pd_key in callback_downloaded: + continue + # Detect if manifest ref changed from what's recorded in the lockfile. + # detect_ref_change() handles all transitions including None\u2192ref. + _pd_locked_chk = ( + existing_lockfile.get_dependency(_pd_key) + if existing_lockfile + else None + ) + _pd_ref_changed = detect_ref_change( + _pd_ref, _pd_locked_chk, update_refs=update_refs + ) + # Skip if lockfile SHA matches local HEAD. + # Normal mode: only when the ref hasn't changed in the manifest. + # Update mode: defer to the sequential loop which resolves the + # remote ref and compares -- if unchanged, the download is skipped + # entirely; if changed, it falls back to sequential download. + if (_pd_path.exists() and _pd_locked_chk + and _pd_locked_chk.resolved_commit + and _pd_locked_chk.resolved_commit != "cached" + and (update_refs or not _pd_ref_changed)): + try: + from git import Repo as _PDGitRepo + if _PDGitRepo(_pd_path).head.commit.hexsha == _pd_locked_chk.resolved_commit: + continue + except Exception: + pass + # Build download ref (use locked commit for reproducibility). + # build_download_ref() uses the manifest ref when ref_changed is True. + _pd_dlref = build_download_ref( + _pd_ref, existing_lockfile, update_refs=update_refs, ref_changed=_pd_ref_changed + ) + _need_download.append((_pd_ref, _pd_path, _pd_dlref)) + + if _need_download and parallel_downloads > 0: + from rich.progress import ( + Progress, + SpinnerColumn, + TextColumn, + BarColumn, + TaskProgressColumn, + ) + + with Progress( + SpinnerColumn(), + TextColumn("[cyan]{task.description}[/cyan]"), + BarColumn(), + TaskProgressColumn(), + transient=True, + ) as _dl_progress: + _max_workers = min(parallel_downloads, len(_need_download)) + with ThreadPoolExecutor(max_workers=_max_workers) as _executor: + _futures = {} + for _pd_ref, _pd_path, _pd_dlref in _need_download: + _pd_disp = str(_pd_ref) if _pd_ref.is_virtual else _pd_ref.repo_url + _pd_short = _pd_disp.split("/")[-1] if "/" in _pd_disp else _pd_disp + _pd_tid = _dl_progress.add_task(description=f"Fetching {_pd_short}", total=None) + _pd_fut = _executor.submit( + downloader.download_package, _pd_dlref, _pd_path, + progress_task_id=_pd_tid, progress_obj=_dl_progress, + ) + _futures[_pd_fut] = (_pd_ref, _pd_tid, _pd_disp) + for _pd_fut in _futures_completed(_futures): + _pd_ref, _pd_tid, _pd_disp = _futures[_pd_fut] + _pd_key = _pd_ref.get_unique_key() + try: + _pd_info = _pd_fut.result() + _pre_download_results[_pd_key] = _pd_info + _dl_progress.update(_pd_tid, visible=False) + _dl_progress.refresh() + except Exception: + _dl_progress.remove_task(_pd_tid) + # Silent: sequential loop below will retry and report errors + + ctx.pre_download_results = _pre_download_results + ctx.pre_downloaded_keys = builtins.set(_pre_download_results.keys()) From 8a43ca7f0f9d12ba6b65f454c275164f8c4fc584 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:16:08 +0200 Subject: [PATCH 08/28] refactor(install): P2.S3 -- extract sequential integration phase Seam: lines 1290-2004 of install.py -> install/phases/integrate.py Extract the ~700 LOC sequential integration loop into src/apm_cli/install/phases/integrate.py as a single run(ctx) function. This covers per-dependency integration (local copy, cached, fresh download) and root-project primitives integration (#714). install.py shrinks from 2211 to 1511 lines (-700). New InstallContext fields (context.py): Pre-integrate inputs: diagnostics, registry_config, managed_files, installed_packages Integrate outputs: installed_count, unpinned_count, total_prompts_integrated, total_agents_integrated, total_skills_integrated, total_sub_skills_promoted, total_instructions_integrated, total_commands_integrated, total_hooks_integrated, total_links_resolved Test-patch contract preserved: _integrate_package_primitives (4 call sites), _rich_success, _rich_error, _copy_local_package, _pre_deploy_security_scan all accessed via _install_mod.X indirection. Rubber-duck review: all 8 questions CLEAN -- no test-patch bypass, no NameError, exception handling preserved, int counter write-back correct, dependency_graph read-only, lazy imports preserved. Test gates: 3972 unit + 32 integration passed. Co-authored-by: Daniel Meppiel --- src/apm_cli/commands/install.py | 760 +-------------------- src/apm_cli/install/context.py | 20 +- src/apm_cli/install/phases/integrate.py | 861 ++++++++++++++++++++++++ 3 files changed, 910 insertions(+), 731 deletions(-) create mode 100644 src/apm_cli/install/phases/integrate.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 4d9b0c16..d337cbe5 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1263,16 +1263,6 @@ def _install_apm_dependencies( from apm_cli.integration.base_integrator import BaseIntegrator managed_files = BaseIntegrator.normalize_managed_files(managed_files) - # Install each dependency with Rich progress display - from rich.progress import ( - Progress, - SpinnerColumn, - TextColumn, - BarColumn, - TaskProgressColumn, - ) - - # downloader already created above for transitive resolution installed_count = 0 unpinned_count = 0 @@ -1282,726 +1272,36 @@ def _install_apm_dependencies( from apm_cli.install.phases import download as _download_phase _download_phase.run(ctx) - # Bridge: alias ctx outputs into locals so downstream code - # (lines ~1554, ~1838-1839) continues working without changes. - _pre_download_results = ctx.pre_download_results - _pre_downloaded_keys = ctx.pre_downloaded_keys - - # Create progress display for sequential integration - # Reuse the shared auth_resolver (already created in this invocation) so - # verbose auth logging does not trigger a duplicate credential-helper popup. - _auth_resolver = auth_resolver - - with Progress( - SpinnerColumn(), - TextColumn("[cyan]{task.description}[/cyan]"), - BarColumn(), - TaskProgressColumn(), - transient=True, # Progress bar disappears when done - ) as progress: - for dep_ref in deps_to_install: - # Determine installation directory using namespaced structure - # e.g., microsoft/apm-sample-package -> apm_modules/microsoft/apm-sample-package/ - # For virtual packages: owner/repo/prompts/file.prompt.md -> apm_modules/owner/repo-file/ - # For subdirectory packages: owner/repo/subdir -> apm_modules/owner/repo/subdir/ - if dep_ref.alias: - # If alias is provided, use it directly (assume user handles namespacing) - install_name = dep_ref.alias - install_path = apm_modules_dir / install_name - else: - # Use the canonical install path from DependencyReference - install_path = dep_ref.get_install_path(apm_modules_dir) - - # Skip deps that already failed during BFS resolution callback - # to avoid a duplicate error entry in diagnostics. - dep_key = dep_ref.get_unique_key() - if dep_key in callback_failures: - if logger: - logger.verbose_detail(f" Skipping {dep_key} (already failed during resolution)") - continue - - # --- Local package: copy from filesystem (no git download) --- - if dep_ref.is_local and dep_ref.local_path: - # User scope: relative paths would resolve against $HOME - # instead of cwd, producing wrong results. Skip with a - # clear diagnostic rather than silently failing. - if scope is InstallScope.USER: - diagnostics.warn( - f"Skipped local package '{dep_ref.local_path}' " - "-- local paths are not supported at user scope (--global). " - "Use a remote reference (owner/repo) instead.", - package=dep_ref.local_path, - ) - if logger: - logger.verbose_detail( - f" Skipping {dep_ref.local_path} (local packages " - "resolve against cwd, not $HOME)" - ) - continue - - result_path = _copy_local_package(dep_ref, install_path, project_root) - if not result_path: - diagnostics.error( - f"Failed to copy local package: {dep_ref.local_path}", - package=dep_ref.local_path, - ) - continue - - installed_count += 1 - if logger: - logger.download_complete(dep_ref.local_path, ref_suffix="local") - - # Build minimal PackageInfo for integration - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, - ) - from datetime import datetime - - local_apm_yml = install_path / "apm.yml" - if local_apm_yml.exists(): - local_pkg = APMPackage.from_apm_yml(local_apm_yml) - if not local_pkg.source: - local_pkg.source = dep_ref.local_path - else: - local_pkg = APMPackage( - name=Path(dep_ref.local_path).name, - version="0.0.0", - package_path=install_path, - source=dep_ref.local_path, - ) - - local_ref = ResolvedReference( - original_ref="local", - ref_type=GitReferenceType.BRANCH, - resolved_commit="local", - ref_name="local", - ) - local_info = PackageInfo( - package=local_pkg, - install_path=install_path, - resolved_reference=local_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, - ) - - # Detect package type - from apm_cli.models.validation import detect_package_type - pkg_type, plugin_json_path = detect_package_type(install_path) - local_info.package_type = pkg_type - if pkg_type == PackageType.MARKETPLACE_PLUGIN: - # Normalize: synthesize .apm/ from plugin.json so - # integration can discover and deploy primitives - from apm_cli.deps.plugin_parser import normalize_plugin_directory - normalize_plugin_directory(install_path, plugin_json_path) - - # Record for lockfile - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=None, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=None, # local deps never go through registry - )) - dep_key = dep_ref.get_unique_key() - if install_path.is_dir() and not dep_ref.is_local: - _package_hashes[dep_key] = _compute_hash(install_path) - dep_deployed_files: builtins.list = [] - - if hasattr(local_info, 'package_type') and local_info.package_type: - package_types[dep_key] = local_info.package_type.value - - # Use the same variable name as the rest of the loop - package_info = local_info - - # Run shared integration pipeline - try: - # Pre-deploy security gate - if not _pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=force, - logger=logger, - ): - package_deployed_files[dep_key] = [] - continue - - int_result = _integrate_package_primitives( - package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed_files.extend(int_result["deployed_files"]) - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from local package: {e}", - package=dep_ref.local_path, - ) - - package_deployed_files[dep_key] = dep_deployed_files - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - continue - - # npm-like behavior: Branches always fetch latest, only tags/commits use cache - # Resolve git reference to determine type - from apm_cli.models.apm_package import GitReferenceType - - resolved_ref = None - if dep_ref.get_unique_key() not in _pre_downloaded_keys: - # Resolve when there is an explicit ref, OR when update_refs - # is True AND we have a non-cached lockfile entry to compare - # against (otherwise resolution is wasted work -- the package - # will be downloaded regardless). - _has_lockfile_sha = False - if update_refs and existing_lockfile: - _lck = existing_lockfile.get_dependency(dep_ref.get_unique_key()) - _has_lockfile_sha = bool( - _lck and _lck.resolved_commit and _lck.resolved_commit != "cached" - ) - if dep_ref.reference or (update_refs and _has_lockfile_sha): - try: - resolved_ref = downloader.resolve_git_reference(dep_ref) - except Exception: - pass # If resolution fails, skip cache (fetch latest) - - # Use cache only for tags and commits (not branches) - is_cacheable = resolved_ref and resolved_ref.ref_type in [ - GitReferenceType.TAG, - GitReferenceType.COMMIT, - ] - # Skip download if: already fetched by resolver callback, or cached tag/commit - already_resolved = dep_ref.get_unique_key() in callback_downloaded - # Detect if manifest ref changed vs what the lockfile recorded. - # detect_ref_change() handles all transitions including None→ref. - _dep_locked_chk = ( - existing_lockfile.get_dependency(dep_ref.get_unique_key()) - if existing_lockfile - else None - ) - ref_changed = detect_ref_change( - dep_ref, _dep_locked_chk, update_refs=update_refs - ) - # Phase 5 (#171): Also skip when lockfile SHA matches local HEAD - # -- but not when the manifest ref has changed (user wants different version). - lockfile_match = False - if install_path.exists() and existing_lockfile: - locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key()) - if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached": - if update_refs: - # Update mode: compare resolved remote SHA with lockfile SHA. - # If the remote ref still resolves to the same commit, - # the package content is unchanged -- skip download. - # Also verify local checkout matches to guard against - # corrupted installs that bypassed pre-download checks. - if resolved_ref and resolved_ref.resolved_commit == locked_dep.resolved_commit: - try: - from git import Repo as GitRepo - local_repo = GitRepo(install_path) - if local_repo.head.commit.hexsha == locked_dep.resolved_commit: - lockfile_match = True - except Exception: - pass # Local checkout invalid -- fall through to download - elif not ref_changed: - # Normal mode: compare local HEAD with lockfile SHA. - try: - from git import Repo as GitRepo - local_repo = GitRepo(install_path) - if local_repo.head.commit.hexsha == locked_dep.resolved_commit: - lockfile_match = True - except Exception: - pass # Not a git repo or invalid -- fall through to download - skip_download = install_path.exists() and ( - (is_cacheable and not update_refs) - or (already_resolved and not update_refs) - or lockfile_match - ) - - # Verify content integrity when lockfile has a hash - if skip_download and _dep_locked_chk and _dep_locked_chk.content_hash: - from ..utils.content_hash import verify_package_hash - if not verify_package_hash(install_path, _dep_locked_chk.content_hash): - _hash_msg = ( - f"Content hash mismatch for " - f"{dep_ref.get_unique_key()} -- re-downloading" - ) - diagnostics.warn(_hash_msg, package=dep_ref.get_unique_key()) - if logger: - logger.progress(_hash_msg) - safe_rmtree(install_path, apm_modules_dir) - skip_download = False - - # When registry-only mode is active, bypass cache if the - # cached artifact was NOT previously downloaded via the - # registry (no registry_prefix in lockfile). This handles - # the transition from direct-VCS installs to proxy installs - # for packages not yet in the lockfile. - if ( - skip_download - and registry_config - and registry_config.enforce_only - and not dep_ref.is_local - ): - if not _dep_locked_chk or _dep_locked_chk.registry_prefix is None: - skip_download = False - - if skip_download: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - # Show resolved ref from lockfile for consistency with fresh installs - _ref = dep_ref.reference or "" - _sha = "" - if _dep_locked_chk and _dep_locked_chk.resolved_commit and _dep_locked_chk.resolved_commit != "cached": - _sha = _dep_locked_chk.resolved_commit[:8] - if logger: - logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) - installed_count += 1 - if not dep_ref.reference: - unpinned_count += 1 - - # Skip integration if not needed - if not _targets: - continue - - # Integrate prompts for cached packages (zero-config behavior) - try: - # Create PackageInfo from cached package - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, - ) - from datetime import datetime - - # Load package from apm.yml in install path - apm_yml_path = install_path / APM_YML_FILENAME - if apm_yml_path.exists(): - cached_package = APMPackage.from_apm_yml(apm_yml_path) - # Ensure source is set to the repo URL for sync matching - if not cached_package.source: - cached_package.source = dep_ref.repo_url - else: - # Virtual package or no apm.yml - create minimal package - cached_package = APMPackage( - name=dep_ref.repo_url.split("/")[-1], - version="unknown", - package_path=install_path, - source=dep_ref.repo_url, - ) - - # Use resolved reference from ref resolution if available - # (e.g. when update_refs matched the lockfile SHA), - # otherwise create a placeholder for cached packages. - resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( - original_ref=dep_ref.reference or "default", - ref_type=GitReferenceType.BRANCH, - resolved_commit="cached", # Mark as cached since we don't know exact commit - ref_name=dep_ref.reference or "default", - ) - - cached_package_info = PackageInfo( - package=cached_package, - install_path=install_path, - resolved_reference=resolved_or_cached_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, # Store for canonical dependency string - ) - - # Detect package_type from disk contents so - # skill integration is not silently skipped - from apm_cli.models.validation import detect_package_type - pkg_type, _ = detect_package_type(install_path) - cached_package_info.package_type = pkg_type - - # Collect for lockfile (cached packages still need to be tracked) - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - # Get commit SHA: resolved ref > callback capture > existing lockfile > explicit reference - dep_key = dep_ref.get_unique_key() - cached_commit = None - if resolved_ref and resolved_ref.resolved_commit and resolved_ref.resolved_commit != "cached": - cached_commit = resolved_ref.resolved_commit - if not cached_commit: - cached_commit = callback_downloaded.get(dep_key) - if not cached_commit and existing_lockfile: - locked_dep = existing_lockfile.get_dependency(dep_key) - if locked_dep: - cached_commit = locked_dep.resolved_commit - if not cached_commit: - cached_commit = dep_ref.reference - # Determine if the cached package came from the registry: - # prefer the lockfile record, then the current registry config. - _cached_registry = None - if _dep_locked_chk and _dep_locked_chk.registry_prefix: - # Reconstruct RegistryConfig from lockfile to preserve original source - _cached_registry = registry_config - elif registry_config and not dep_ref.is_local: - _cached_registry = registry_config - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=cached_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=_cached_registry, - )) - if install_path.is_dir(): - _package_hashes[dep_key] = _compute_hash(install_path) - # Track package type for lockfile - if hasattr(cached_package_info, 'package_type') and cached_package_info.package_type: - package_types[dep_key] = cached_package_info.package_type.value - - # Pre-deploy security gate - if not _pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=force, - logger=logger, - ): - package_deployed_files[dep_key] = [] - continue - - int_result = _integrate_package_primitives( - cached_package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed = int_result["deployed_files"] - package_deployed_files[dep_key] = dep_deployed - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from cached package: {e}", - package=dep_key, - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - continue - - # Download the package with progress feedback - try: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - short_name = ( - display_name.split("/")[-1] - if "/" in display_name - else display_name - ) - - # Create a progress task for this download - task_id = progress.add_task( - description=f"Fetching {short_name}", - total=None, # Indeterminate initially; git will update with actual counts - ) - - # T5: Build download ref - use locked commit if available. - # build_download_ref() uses manifest ref when ref_changed is True. - download_ref = build_download_ref( - dep_ref, existing_lockfile, update_refs=update_refs, ref_changed=ref_changed - ) - - # Phase 4 (#171): Use pre-downloaded result if available - _dep_key = dep_ref.get_unique_key() - if _dep_key in _pre_download_results: - package_info = _pre_download_results[_dep_key] - else: - # Fallback: sequential download (should rarely happen) - package_info = downloader.download_package( - download_ref, - install_path, - progress_task_id=task_id, - progress_obj=progress, - ) - - # CRITICAL: Hide progress BEFORE printing success message to avoid overlap - progress.update(task_id, visible=False) - progress.refresh() # Force immediate refresh to hide the bar - - installed_count += 1 - - # Show resolved ref alongside package name for visibility - resolved = getattr(package_info, 'resolved_reference', None) - if logger: - _ref = "" - _sha = "" - if resolved: - _ref = resolved.ref_name if resolved.ref_name else "" - _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - logger.download_complete(display_name, ref=_ref, sha=_sha) - # Log auth source for this download (verbose only) - if _auth_resolver: - try: - _host = dep_ref.host or "github.com" - _org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None - _ctx = _auth_resolver.resolve(_host, org=_org) - logger.package_auth(_ctx.source, _ctx.token_type or "none") - except Exception: - pass - else: - _ref_suffix = "" - if resolved: - _r = resolved.ref_name if resolved.ref_name else "" - _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - if _r and _s: - _ref_suffix = f" #{_r} @{_s}" - elif _r: - _ref_suffix = f" #{_r}" - elif _s: - _ref_suffix = f" @{_s}" - _rich_success(f"[+] {display_name}{_ref_suffix}") - - # Track unpinned deps for aggregated diagnostic - if not dep_ref.reference: - unpinned_count += 1 - - # Collect for lockfile: get resolved commit and depth - resolved_commit = None - if resolved: - resolved_commit = package_info.resolved_reference.resolved_commit - # Get depth from dependency tree - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=resolved_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=registry_config if not dep_ref.is_local else None, - )) - if install_path.is_dir(): - _package_hashes[dep_ref.get_unique_key()] = _compute_hash(install_path) - - # Supply chain protection: verify content hash on fresh - # downloads when the lockfile already records a hash. - # A mismatch means the downloaded content differs from - # what was previously locked — possible tampering. - if ( - not update_refs - and _dep_locked_chk - and _dep_locked_chk.content_hash - and dep_ref.get_unique_key() in _package_hashes - ): - _fresh_hash = _package_hashes[dep_ref.get_unique_key()] - if _fresh_hash != _dep_locked_chk.content_hash: - safe_rmtree(install_path, apm_modules_dir) - _rich_error( - f"Content hash mismatch for " - f"{dep_ref.get_unique_key()}: " - f"expected {_dep_locked_chk.content_hash}, " - f"got {_fresh_hash}. " - "The downloaded content differs from the " - "lockfile record. This may indicate a " - "supply-chain attack. Use 'apm install " - "--update' to accept new content and " - "update the lockfile." - ) - sys.exit(1) - - # Track package type for lockfile - if hasattr(package_info, 'package_type') and package_info.package_type: - package_types[dep_ref.get_unique_key()] = package_info.package_type.value - - # Show package type in verbose mode - if hasattr(package_info, "package_type"): - from apm_cli.models.apm_package import PackageType - - package_type = package_info.package_type - _type_label = { - PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", - PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", - PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", - PackageType.APM_PACKAGE: "APM Package (apm.yml)", - }.get(package_type) - if _type_label and logger: - logger.package_type_info(_type_label) - - # Auto-integrate prompts and agents if enabled - # Pre-deploy security gate - if not _pre_deploy_security_scan( - package_info.install_path, diagnostics, - package_name=dep_ref.get_unique_key(), force=force, - logger=logger, - ): - package_deployed_files[dep_ref.get_unique_key()] = [] - continue - - if _targets: - try: - int_result = _integrate_package_primitives( - package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_ref.get_unique_key(), - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed_fresh = int_result["deployed_files"] - package_deployed_files[dep_ref.get_unique_key()] = dep_deployed_fresh - except Exception as e: - # Don't fail installation if integration fails - diagnostics.error( - f"Failed to integrate primitives: {e}", - package=dep_ref.get_unique_key(), - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - pkg_key = dep_ref.get_unique_key() - _skip_count = diagnostics.count_for_package(pkg_key, "collision") - _err_count = diagnostics.count_for_package(pkg_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - except Exception as e: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - # Remove the progress task on error - if "task_id" in locals(): - progress.remove_task(task_id) - diagnostics.error( - f"Failed to install {display_name}: {e}", - package=dep_ref.get_unique_key(), - ) - # Continue with other packages instead of failing completely - continue - - # ------------------------------------------------------------------ - # Integrate root project's own .apm/ primitives (#714). - # - # Users should not need a dummy "./agent/apm.yml" stub to get their - # root-level .apm/ rules deployed alongside external dependencies. - # Treat the project root as an implicit local package: any primitives - # found in /.apm/ are integrated after all declared - # dependency packages have been processed. - # ------------------------------------------------------------------ - if _root_has_local_primitives and _targets: - from apm_cli.models.apm_package import PackageInfo as _PackageInfo - _root_pkg_info = _PackageInfo( - package=apm_package, - install_path=project_root, - ) - if logger: - logger.download_complete("", ref_suffix="local") - try: - _root_result = _integrate_package_primitives( - _root_pkg_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name="", - logger=logger, - scope=scope, - ) - total_prompts_integrated += _root_result["prompts"] - total_agents_integrated += _root_result["agents"] - total_instructions_integrated += _root_result["instructions"] - total_commands_integrated += _root_result["commands"] - total_hooks_integrated += _root_result["hooks"] - total_links_resolved += _root_result["links_resolved"] - installed_count += 1 - except Exception as e: - import traceback as _tb - diagnostics.error( - f"Failed to integrate root project primitives: {e}", - package="", - detail=_tb.format_exc(), - ) - # When root integration is the *only* action (no external deps), - # a failure means nothing was deployed — surface it clearly. - if not all_apm_deps and logger: - logger.error( - f"Root project primitives could not be integrated: {e}" - ) + # -------------------------------------------------------------- + # Phase 5: Sequential integration loop + root primitives + # -------------------------------------------------------------- + # Populate ctx with locals needed by the integrate phase. + ctx.diagnostics = diagnostics + ctx.registry_config = registry_config + ctx.managed_files = managed_files + ctx.installed_packages = installed_packages + + from apm_cli.install.phases import integrate as _integrate_phase + _integrate_phase.run(ctx) + + # Bridge: read phase outputs back into locals so downstream code + # (orphan cleanup, stale cleanup, lockfile assembly, summary) + # continues working without modification. + installed_count = ctx.installed_count + unpinned_count = ctx.unpinned_count + installed_packages = ctx.installed_packages + package_deployed_files = ctx.package_deployed_files + package_types = ctx.package_types + _package_hashes = ctx.package_hashes + total_prompts_integrated = ctx.total_prompts_integrated + total_agents_integrated = ctx.total_agents_integrated + total_skills_integrated = ctx.total_skills_integrated + total_sub_skills_promoted = ctx.total_sub_skills_promoted + total_instructions_integrated = ctx.total_instructions_integrated + total_commands_integrated = ctx.total_commands_integrated + total_hooks_integrated = ctx.total_hooks_integrated + total_links_resolved = ctx.total_links_resolved + intended_dep_keys = ctx.intended_dep_keys # Update .gitignore _update_gitignore_for_apm_modules(logger=logger) diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py index be889a57..30ee1877 100644 --- a/src/apm_cli/install/context.py +++ b/src/apm_cli/install/context.py @@ -81,9 +81,27 @@ class InstallContext: pre_downloaded_keys: Set[str] = field(default_factory=set) # download # ------------------------------------------------------------------ - # Downstream phase accumulators (written by integrate, read by cleanup/lockfile) + # Pre-integrate inputs (populated by caller before integrate phase) + # ------------------------------------------------------------------ + diagnostics: Any = None # DiagnosticCollector + registry_config: Any = None # RegistryConfig + managed_files: Set[str] = field(default_factory=set) + + # ------------------------------------------------------------------ + # Integrate phase outputs (written by integrate, read by cleanup/lockfile/summary) # ------------------------------------------------------------------ intended_dep_keys: Set[str] = field(default_factory=set) package_deployed_files: Dict[str, List[str]] = field(default_factory=dict) package_types: Dict[str, Dict[str, Any]] = field(default_factory=dict) package_hashes: Dict[str, Dict[str, str]] = field(default_factory=dict) + installed_count: int = 0 # integrate + unpinned_count: int = 0 # integrate + installed_packages: List[Any] = field(default_factory=list) # integrate + total_prompts_integrated: int = 0 # integrate + total_agents_integrated: int = 0 # integrate + total_skills_integrated: int = 0 # integrate + total_sub_skills_promoted: int = 0 # integrate + total_instructions_integrated: int = 0 # integrate + total_commands_integrated: int = 0 # integrate + total_hooks_integrated: int = 0 # integrate + total_links_resolved: int = 0 # integrate diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py new file mode 100644 index 00000000..b2fc9b3f --- /dev/null +++ b/src/apm_cli/install/phases/integrate.py @@ -0,0 +1,861 @@ +"""Sequential integration phase -- per-package integration loop. + +Reads all prior phase outputs from *ctx* (resolve, targets, download) and +processes each dependency sequentially: local-copy packages, cached packages, +and freshly-downloaded packages. For every package the loop: + +1. Builds a ``PackageInfo`` (or reuses the pre-downloaded result). +2. Runs the pre-deploy security scan. +3. Calls ``_integrate_package_primitives`` (via module-attribute access on + ``apm_cli.commands.install`` so that test patches at + ``@patch("apm_cli.commands.install._integrate_package_primitives")`` + continue to intercept the call). +4. Accumulates deployed-file lists, content hashes, and integration totals + on *ctx* for the downstream cleanup and lockfile phases. + +After the dependency loop, root-project primitives (``/.apm/``) +are integrated when present (#714). + +**Test-patch contract**: every name that tests patch at +``apm_cli.commands.install.X`` is accessed via the ``_install_mod.X`` +indirection rather than a bare-name import. This includes at minimum: +``_integrate_package_primitives``, ``_rich_success``, ``_rich_error``, +``_copy_local_package``, ``_pre_deploy_security_scan``. +""" + +from __future__ import annotations + +import builtins +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: "InstallContext") -> None: + """Execute the sequential integration phase. + + On return the following *ctx* fields are populated / updated: + ``installed_count``, ``unpinned_count``, ``installed_packages``, + ``package_deployed_files``, ``package_types``, ``package_hashes``, + ``total_prompts_integrated``, ``total_agents_integrated``, + ``total_skills_integrated``, ``total_sub_skills_promoted``, + ``total_instructions_integrated``, ``total_commands_integrated``, + ``total_hooks_integrated``, ``total_links_resolved``. + """ + # ------------------------------------------------------------------ + # Module-attribute access for late-patchability. + # Tests patch names at apm_cli.commands.install.X -- importing the + # MODULE (not the name) ensures the patched attribute is resolved at + # call time. + # ------------------------------------------------------------------ + from apm_cli.commands import install as _install_mod + + # ------------------------------------------------------------------ + # Direct imports for names NOT patched at apm_cli.commands.install.X + # ------------------------------------------------------------------ + from apm_cli.constants import APM_YML_FILENAME + from apm_cli.core.scope import InstallScope + from apm_cli.deps.installed_package import InstalledPackage + from apm_cli.drift import build_download_ref, detect_ref_change + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + from apm_cli.utils.path_security import safe_rmtree + from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + ) + + # ------------------------------------------------------------------ + # Unpack ctx into local aliases. Mutable containers (lists, dicts, + # sets) share the reference so in-place mutations are visible through + # ctx. Int counters are accumulated into locals and written back at + # the end of this function. + # ------------------------------------------------------------------ + deps_to_install = ctx.deps_to_install + apm_modules_dir = ctx.apm_modules_dir + callback_failures = ctx.callback_failures + callback_downloaded = ctx.callback_downloaded + scope = ctx.scope + diagnostics = ctx.diagnostics + logger = ctx.logger + project_root = ctx.project_root + dependency_graph = ctx.dependency_graph + existing_lockfile = ctx.existing_lockfile + update_refs = ctx.update_refs + downloader = ctx.downloader + force = ctx.force + apm_package = ctx.apm_package + all_apm_deps = ctx.all_apm_deps + registry_config = ctx.registry_config + _targets = ctx.targets + _pre_download_results = ctx.pre_download_results + _pre_downloaded_keys = ctx.pre_downloaded_keys + _root_has_local_primitives = ctx.root_has_local_primitives + + # Mutable containers (shared references -- mutations visible via ctx) + installed_packages = ctx.installed_packages + package_deployed_files = ctx.package_deployed_files + package_types = ctx.package_types + _package_hashes = ctx.package_hashes + managed_files = ctx.managed_files + + # Integrators + prompt_integrator = ctx.integrators["prompt"] + agent_integrator = ctx.integrators["agent"] + skill_integrator = ctx.integrators["skill"] + instruction_integrator = ctx.integrators["instruction"] + command_integrator = ctx.integrators["command"] + hook_integrator = ctx.integrators["hook"] + + # Int counters (written back to ctx at end of function) + installed_count = ctx.installed_count + unpinned_count = ctx.unpinned_count + total_prompts_integrated = ctx.total_prompts_integrated + total_agents_integrated = ctx.total_agents_integrated + total_skills_integrated = ctx.total_skills_integrated + total_sub_skills_promoted = ctx.total_sub_skills_promoted + total_instructions_integrated = ctx.total_instructions_integrated + total_commands_integrated = ctx.total_commands_integrated + total_hooks_integrated = ctx.total_hooks_integrated + total_links_resolved = ctx.total_links_resolved + + # ------------------------------------------------------------------ + # Begin extracted region (install.py lines 1290-2004, verbatim + # except for free-variable replacement and indentation adjustment) + # ------------------------------------------------------------------ + + # Create progress display for sequential integration + # Reuse the shared auth_resolver (already created in this invocation) so + # verbose auth logging does not trigger a duplicate credential-helper popup. + _auth_resolver = ctx.auth_resolver + + with Progress( + SpinnerColumn(), + TextColumn("[cyan]{task.description}[/cyan]"), + BarColumn(), + TaskProgressColumn(), + transient=True, # Progress bar disappears when done + ) as progress: + for dep_ref in deps_to_install: + # Determine installation directory using namespaced structure + # e.g., microsoft/apm-sample-package -> apm_modules/microsoft/apm-sample-package/ + # For virtual packages: owner/repo/prompts/file.prompt.md -> apm_modules/owner/repo-file/ + # For subdirectory packages: owner/repo/subdir -> apm_modules/owner/repo/subdir/ + if dep_ref.alias: + # If alias is provided, use it directly (assume user handles namespacing) + install_name = dep_ref.alias + install_path = apm_modules_dir / install_name + else: + # Use the canonical install path from DependencyReference + install_path = dep_ref.get_install_path(apm_modules_dir) + + # Skip deps that already failed during BFS resolution callback + # to avoid a duplicate error entry in diagnostics. + dep_key = dep_ref.get_unique_key() + if dep_key in callback_failures: + if logger: + logger.verbose_detail(f" Skipping {dep_key} (already failed during resolution)") + continue + + # --- Local package: copy from filesystem (no git download) --- + if dep_ref.is_local and dep_ref.local_path: + # User scope: relative paths would resolve against $HOME + # instead of cwd, producing wrong results. Skip with a + # clear diagnostic rather than silently failing. + if scope is InstallScope.USER: + diagnostics.warn( + f"Skipped local package '{dep_ref.local_path}' " + "-- local paths are not supported at user scope (--global). " + "Use a remote reference (owner/repo) instead.", + package=dep_ref.local_path, + ) + if logger: + logger.verbose_detail( + f" Skipping {dep_ref.local_path} (local packages " + "resolve against cwd, not $HOME)" + ) + continue + + result_path = _install_mod._copy_local_package(dep_ref, install_path, project_root) + if not result_path: + diagnostics.error( + f"Failed to copy local package: {dep_ref.local_path}", + package=dep_ref.local_path, + ) + continue + + installed_count += 1 + if logger: + logger.download_complete(dep_ref.local_path, ref_suffix="local") + + # Build minimal PackageInfo for integration + from apm_cli.models.apm_package import ( + APMPackage, + PackageInfo, + PackageType, + ResolvedReference, + GitReferenceType, + ) + from datetime import datetime + + local_apm_yml = install_path / "apm.yml" + if local_apm_yml.exists(): + local_pkg = APMPackage.from_apm_yml(local_apm_yml) + if not local_pkg.source: + local_pkg.source = dep_ref.local_path + else: + local_pkg = APMPackage( + name=Path(dep_ref.local_path).name, + version="0.0.0", + package_path=install_path, + source=dep_ref.local_path, + ) + + local_ref = ResolvedReference( + original_ref="local", + ref_type=GitReferenceType.BRANCH, + resolved_commit="local", + ref_name="local", + ) + local_info = PackageInfo( + package=local_pkg, + install_path=install_path, + resolved_reference=local_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, + ) + + # Detect package type + from apm_cli.models.validation import detect_package_type + pkg_type, plugin_json_path = detect_package_type(install_path) + local_info.package_type = pkg_type + if pkg_type == PackageType.MARKETPLACE_PLUGIN: + # Normalize: synthesize .apm/ from plugin.json so + # integration can discover and deploy primitives + from apm_cli.deps.plugin_parser import normalize_plugin_directory + normalize_plugin_directory(install_path, plugin_json_path) + + # Record for lockfile + node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=None, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=None, # local deps never go through registry + )) + dep_key = dep_ref.get_unique_key() + if install_path.is_dir() and not dep_ref.is_local: + _package_hashes[dep_key] = _compute_hash(install_path) + dep_deployed_files: builtins.list = [] + + if hasattr(local_info, 'package_type') and local_info.package_type: + package_types[dep_key] = local_info.package_type.value + + # Use the same variable name as the rest of the loop + package_info = local_info + + # Run shared integration pipeline + try: + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + install_path, diagnostics, + package_name=dep_key, force=force, + logger=logger, + ): + package_deployed_files[dep_key] = [] + continue + + int_result = _install_mod._integrate_package_primitives( + package_info, project_root, + targets=_targets, + prompt_integrator=prompt_integrator, + agent_integrator=agent_integrator, + skill_integrator=skill_integrator, + instruction_integrator=instruction_integrator, + command_integrator=command_integrator, + hook_integrator=hook_integrator, + force=force, + managed_files=managed_files, + diagnostics=diagnostics, + package_name=dep_key, + logger=logger, + scope=scope, + ) + total_prompts_integrated += int_result["prompts"] + total_agents_integrated += int_result["agents"] + total_skills_integrated += int_result["skills"] + total_sub_skills_promoted += int_result["sub_skills"] + total_instructions_integrated += int_result["instructions"] + total_commands_integrated += int_result["commands"] + total_hooks_integrated += int_result["hooks"] + total_links_resolved += int_result["links_resolved"] + dep_deployed_files.extend(int_result["deployed_files"]) + except Exception as e: + diagnostics.error( + f"Failed to integrate primitives from local package: {e}", + package=dep_ref.local_path, + ) + + package_deployed_files[dep_key] = dep_deployed_files + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + _skip_count = diagnostics.count_for_package(dep_key, "collision") + _err_count = diagnostics.count_for_package(dep_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + continue + + # npm-like behavior: Branches always fetch latest, only tags/commits use cache + # Resolve git reference to determine type + from apm_cli.models.apm_package import GitReferenceType + + resolved_ref = None + if dep_ref.get_unique_key() not in _pre_downloaded_keys: + # Resolve when there is an explicit ref, OR when update_refs + # is True AND we have a non-cached lockfile entry to compare + # against (otherwise resolution is wasted work -- the package + # will be downloaded regardless). + _has_lockfile_sha = False + if update_refs and existing_lockfile: + _lck = existing_lockfile.get_dependency(dep_ref.get_unique_key()) + _has_lockfile_sha = bool( + _lck and _lck.resolved_commit and _lck.resolved_commit != "cached" + ) + if dep_ref.reference or (update_refs and _has_lockfile_sha): + try: + resolved_ref = downloader.resolve_git_reference(dep_ref) + except Exception: + pass # If resolution fails, skip cache (fetch latest) + + # Use cache only for tags and commits (not branches) + is_cacheable = resolved_ref and resolved_ref.ref_type in [ + GitReferenceType.TAG, + GitReferenceType.COMMIT, + ] + # Skip download if: already fetched by resolver callback, or cached tag/commit + already_resolved = dep_ref.get_unique_key() in callback_downloaded + # Detect if manifest ref changed vs what the lockfile recorded. + # detect_ref_change() handles all transitions including None->ref. + _dep_locked_chk = ( + existing_lockfile.get_dependency(dep_ref.get_unique_key()) + if existing_lockfile + else None + ) + ref_changed = detect_ref_change( + dep_ref, _dep_locked_chk, update_refs=update_refs + ) + # Phase 5 (#171): Also skip when lockfile SHA matches local HEAD + # -- but not when the manifest ref has changed (user wants different version). + lockfile_match = False + if install_path.exists() and existing_lockfile: + locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key()) + if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached": + if update_refs: + # Update mode: compare resolved remote SHA with lockfile SHA. + # If the remote ref still resolves to the same commit, + # the package content is unchanged -- skip download. + # Also verify local checkout matches to guard against + # corrupted installs that bypassed pre-download checks. + if resolved_ref and resolved_ref.resolved_commit == locked_dep.resolved_commit: + try: + from git import Repo as GitRepo + local_repo = GitRepo(install_path) + if local_repo.head.commit.hexsha == locked_dep.resolved_commit: + lockfile_match = True + except Exception: + pass # Local checkout invalid -- fall through to download + elif not ref_changed: + # Normal mode: compare local HEAD with lockfile SHA. + try: + from git import Repo as GitRepo + local_repo = GitRepo(install_path) + if local_repo.head.commit.hexsha == locked_dep.resolved_commit: + lockfile_match = True + except Exception: + pass # Not a git repo or invalid -- fall through to download + skip_download = install_path.exists() and ( + (is_cacheable and not update_refs) + or (already_resolved and not update_refs) + or lockfile_match + ) + + # Verify content integrity when lockfile has a hash + if skip_download and _dep_locked_chk and _dep_locked_chk.content_hash: + from apm_cli.utils.content_hash import verify_package_hash + if not verify_package_hash(install_path, _dep_locked_chk.content_hash): + _hash_msg = ( + f"Content hash mismatch for " + f"{dep_ref.get_unique_key()} -- re-downloading" + ) + diagnostics.warn(_hash_msg, package=dep_ref.get_unique_key()) + if logger: + logger.progress(_hash_msg) + safe_rmtree(install_path, apm_modules_dir) + skip_download = False + + # When registry-only mode is active, bypass cache if the + # cached artifact was NOT previously downloaded via the + # registry (no registry_prefix in lockfile). This handles + # the transition from direct-VCS installs to proxy installs + # for packages not yet in the lockfile. + if ( + skip_download + and registry_config + and registry_config.enforce_only + and not dep_ref.is_local + ): + if not _dep_locked_chk or _dep_locked_chk.registry_prefix is None: + skip_download = False + + if skip_download: + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + # Show resolved ref from lockfile for consistency with fresh installs + _ref = dep_ref.reference or "" + _sha = "" + if _dep_locked_chk and _dep_locked_chk.resolved_commit and _dep_locked_chk.resolved_commit != "cached": + _sha = _dep_locked_chk.resolved_commit[:8] + if logger: + logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) + installed_count += 1 + if not dep_ref.reference: + unpinned_count += 1 + + # Skip integration if not needed + if not _targets: + continue + + # Integrate prompts for cached packages (zero-config behavior) + try: + # Create PackageInfo from cached package + from apm_cli.models.apm_package import ( + APMPackage, + PackageInfo, + PackageType, + ResolvedReference, + GitReferenceType, + ) + from datetime import datetime + + # Load package from apm.yml in install path + apm_yml_path = install_path / APM_YML_FILENAME + if apm_yml_path.exists(): + cached_package = APMPackage.from_apm_yml(apm_yml_path) + # Ensure source is set to the repo URL for sync matching + if not cached_package.source: + cached_package.source = dep_ref.repo_url + else: + # Virtual package or no apm.yml - create minimal package + cached_package = APMPackage( + name=dep_ref.repo_url.split("/")[-1], + version="unknown", + package_path=install_path, + source=dep_ref.repo_url, + ) + + # Use resolved reference from ref resolution if available + # (e.g. when update_refs matched the lockfile SHA), + # otherwise create a placeholder for cached packages. + resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( + original_ref=dep_ref.reference or "default", + ref_type=GitReferenceType.BRANCH, + resolved_commit="cached", # Mark as cached since we don't know exact commit + ref_name=dep_ref.reference or "default", + ) + + cached_package_info = PackageInfo( + package=cached_package, + install_path=install_path, + resolved_reference=resolved_or_cached_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, # Store for canonical dependency string + ) + + # Detect package_type from disk contents so + # skill integration is not silently skipped + from apm_cli.models.validation import detect_package_type + pkg_type, _ = detect_package_type(install_path) + cached_package_info.package_type = pkg_type + + # Collect for lockfile (cached packages still need to be tracked) + node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + # Get commit SHA: resolved ref > callback capture > existing lockfile > explicit reference + dep_key = dep_ref.get_unique_key() + cached_commit = None + if resolved_ref and resolved_ref.resolved_commit and resolved_ref.resolved_commit != "cached": + cached_commit = resolved_ref.resolved_commit + if not cached_commit: + cached_commit = callback_downloaded.get(dep_key) + if not cached_commit and existing_lockfile: + locked_dep = existing_lockfile.get_dependency(dep_key) + if locked_dep: + cached_commit = locked_dep.resolved_commit + if not cached_commit: + cached_commit = dep_ref.reference + # Determine if the cached package came from the registry: + # prefer the lockfile record, then the current registry config. + _cached_registry = None + if _dep_locked_chk and _dep_locked_chk.registry_prefix: + # Reconstruct RegistryConfig from lockfile to preserve original source + _cached_registry = registry_config + elif registry_config and not dep_ref.is_local: + _cached_registry = registry_config + installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=cached_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=_cached_registry, + )) + if install_path.is_dir(): + _package_hashes[dep_key] = _compute_hash(install_path) + # Track package type for lockfile + if hasattr(cached_package_info, 'package_type') and cached_package_info.package_type: + package_types[dep_key] = cached_package_info.package_type.value + + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + install_path, diagnostics, + package_name=dep_key, force=force, + logger=logger, + ): + package_deployed_files[dep_key] = [] + continue + + int_result = _install_mod._integrate_package_primitives( + cached_package_info, project_root, + targets=_targets, + prompt_integrator=prompt_integrator, + agent_integrator=agent_integrator, + skill_integrator=skill_integrator, + instruction_integrator=instruction_integrator, + command_integrator=command_integrator, + hook_integrator=hook_integrator, + force=force, + managed_files=managed_files, + diagnostics=diagnostics, + package_name=dep_key, + logger=logger, + scope=scope, + ) + total_prompts_integrated += int_result["prompts"] + total_agents_integrated += int_result["agents"] + total_skills_integrated += int_result["skills"] + total_sub_skills_promoted += int_result["sub_skills"] + total_instructions_integrated += int_result["instructions"] + total_commands_integrated += int_result["commands"] + total_hooks_integrated += int_result["hooks"] + total_links_resolved += int_result["links_resolved"] + dep_deployed = int_result["deployed_files"] + package_deployed_files[dep_key] = dep_deployed + except Exception as e: + diagnostics.error( + f"Failed to integrate primitives from cached package: {e}", + package=dep_key, + ) + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + _skip_count = diagnostics.count_for_package(dep_key, "collision") + _err_count = diagnostics.count_for_package(dep_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + + continue + + # Download the package with progress feedback + try: + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + short_name = ( + display_name.split("/")[-1] + if "/" in display_name + else display_name + ) + + # Create a progress task for this download + task_id = progress.add_task( + description=f"Fetching {short_name}", + total=None, # Indeterminate initially; git will update with actual counts + ) + + # T5: Build download ref - use locked commit if available. + # build_download_ref() uses manifest ref when ref_changed is True. + download_ref = build_download_ref( + dep_ref, existing_lockfile, update_refs=update_refs, ref_changed=ref_changed + ) + + # Phase 4 (#171): Use pre-downloaded result if available + _dep_key = dep_ref.get_unique_key() + if _dep_key in _pre_download_results: + package_info = _pre_download_results[_dep_key] + else: + # Fallback: sequential download (should rarely happen) + package_info = downloader.download_package( + download_ref, + install_path, + progress_task_id=task_id, + progress_obj=progress, + ) + + # CRITICAL: Hide progress BEFORE printing success message to avoid overlap + progress.update(task_id, visible=False) + progress.refresh() # Force immediate refresh to hide the bar + + installed_count += 1 + + # Show resolved ref alongside package name for visibility + resolved = getattr(package_info, 'resolved_reference', None) + if logger: + _ref = "" + _sha = "" + if resolved: + _ref = resolved.ref_name if resolved.ref_name else "" + _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + logger.download_complete(display_name, ref=_ref, sha=_sha) + # Log auth source for this download (verbose only) + if _auth_resolver: + try: + _host = dep_ref.host or "github.com" + _org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None + _ctx = _auth_resolver.resolve(_host, org=_org) + logger.package_auth(_ctx.source, _ctx.token_type or "none") + except Exception: + pass + else: + _ref_suffix = "" + if resolved: + _r = resolved.ref_name if resolved.ref_name else "" + _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + if _r and _s: + _ref_suffix = f" #{_r} @{_s}" + elif _r: + _ref_suffix = f" #{_r}" + elif _s: + _ref_suffix = f" @{_s}" + _install_mod._rich_success(f"[+] {display_name}{_ref_suffix}") + + # Track unpinned deps for aggregated diagnostic + if not dep_ref.reference: + unpinned_count += 1 + + # Collect for lockfile: get resolved commit and depth + resolved_commit = None + if resolved: + resolved_commit = package_info.resolved_reference.resolved_commit + # Get depth from dependency tree + node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=resolved_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=registry_config if not dep_ref.is_local else None, + )) + if install_path.is_dir(): + _package_hashes[dep_ref.get_unique_key()] = _compute_hash(install_path) + + # Supply chain protection: verify content hash on fresh + # downloads when the lockfile already records a hash. + # A mismatch means the downloaded content differs from + # what was previously locked -- possible tampering. + if ( + not update_refs + and _dep_locked_chk + and _dep_locked_chk.content_hash + and dep_ref.get_unique_key() in _package_hashes + ): + _fresh_hash = _package_hashes[dep_ref.get_unique_key()] + if _fresh_hash != _dep_locked_chk.content_hash: + safe_rmtree(install_path, apm_modules_dir) + _install_mod._rich_error( + f"Content hash mismatch for " + f"{dep_ref.get_unique_key()}: " + f"expected {_dep_locked_chk.content_hash}, " + f"got {_fresh_hash}. " + "The downloaded content differs from the " + "lockfile record. This may indicate a " + "supply-chain attack. Use 'apm install " + "--update' to accept new content and " + "update the lockfile." + ) + sys.exit(1) + + # Track package type for lockfile + if hasattr(package_info, 'package_type') and package_info.package_type: + package_types[dep_ref.get_unique_key()] = package_info.package_type.value + + # Show package type in verbose mode + if hasattr(package_info, "package_type"): + from apm_cli.models.apm_package import PackageType + + package_type = package_info.package_type + _type_label = { + PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", + PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", + PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", + PackageType.APM_PACKAGE: "APM Package (apm.yml)", + }.get(package_type) + if _type_label and logger: + logger.package_type_info(_type_label) + + # Auto-integrate prompts and agents if enabled + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + package_info.install_path, diagnostics, + package_name=dep_ref.get_unique_key(), force=force, + logger=logger, + ): + package_deployed_files[dep_ref.get_unique_key()] = [] + continue + + if _targets: + try: + int_result = _install_mod._integrate_package_primitives( + package_info, project_root, + targets=_targets, + prompt_integrator=prompt_integrator, + agent_integrator=agent_integrator, + skill_integrator=skill_integrator, + instruction_integrator=instruction_integrator, + command_integrator=command_integrator, + hook_integrator=hook_integrator, + force=force, + managed_files=managed_files, + diagnostics=diagnostics, + package_name=dep_ref.get_unique_key(), + logger=logger, + scope=scope, + ) + total_prompts_integrated += int_result["prompts"] + total_agents_integrated += int_result["agents"] + total_skills_integrated += int_result["skills"] + total_sub_skills_promoted += int_result["sub_skills"] + total_instructions_integrated += int_result["instructions"] + total_commands_integrated += int_result["commands"] + total_hooks_integrated += int_result["hooks"] + total_links_resolved += int_result["links_resolved"] + dep_deployed_fresh = int_result["deployed_files"] + package_deployed_files[dep_ref.get_unique_key()] = dep_deployed_fresh + except Exception as e: + # Don't fail installation if integration fails + diagnostics.error( + f"Failed to integrate primitives: {e}", + package=dep_ref.get_unique_key(), + ) + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + pkg_key = dep_ref.get_unique_key() + _skip_count = diagnostics.count_for_package(pkg_key, "collision") + _err_count = diagnostics.count_for_package(pkg_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + + except Exception as e: + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + # Remove the progress task on error + if "task_id" in locals(): + progress.remove_task(task_id) + diagnostics.error( + f"Failed to install {display_name}: {e}", + package=dep_ref.get_unique_key(), + ) + # Continue with other packages instead of failing completely + continue + + # ------------------------------------------------------------------ + # Integrate root project's own .apm/ primitives (#714). + # + # Users should not need a dummy "./agent/apm.yml" stub to get their + # root-level .apm/ rules deployed alongside external dependencies. + # Treat the project root as an implicit local package: any primitives + # found in /.apm/ are integrated after all declared + # dependency packages have been processed. + # ------------------------------------------------------------------ + if _root_has_local_primitives and _targets: + from apm_cli.models.apm_package import PackageInfo as _PackageInfo + _root_pkg_info = _PackageInfo( + package=apm_package, + install_path=project_root, + ) + if logger: + logger.download_complete("", ref_suffix="local") + try: + _root_result = _install_mod._integrate_package_primitives( + _root_pkg_info, project_root, + targets=_targets, + prompt_integrator=prompt_integrator, + agent_integrator=agent_integrator, + skill_integrator=skill_integrator, + instruction_integrator=instruction_integrator, + command_integrator=command_integrator, + hook_integrator=hook_integrator, + force=force, + managed_files=managed_files, + diagnostics=diagnostics, + package_name="", + logger=logger, + scope=scope, + ) + total_prompts_integrated += _root_result["prompts"] + total_agents_integrated += _root_result["agents"] + total_instructions_integrated += _root_result["instructions"] + total_commands_integrated += _root_result["commands"] + total_hooks_integrated += _root_result["hooks"] + total_links_resolved += _root_result["links_resolved"] + installed_count += 1 + except Exception as e: + import traceback as _tb + diagnostics.error( + f"Failed to integrate root project primitives: {e}", + package="", + detail=_tb.format_exc(), + ) + # When root integration is the *only* action (no external deps), + # a failure means nothing was deployed -- surface it clearly. + if not all_apm_deps and logger: + logger.error( + f"Root project primitives could not be integrated: {e}" + ) + + # ------------------------------------------------------------------ + # Write int counters back to ctx (mutable containers already share + # the reference and need no write-back). + # ------------------------------------------------------------------ + ctx.installed_count = installed_count + ctx.unpinned_count = unpinned_count + ctx.total_prompts_integrated = total_prompts_integrated + ctx.total_agents_integrated = total_agents_integrated + ctx.total_skills_integrated = total_skills_integrated + ctx.total_sub_skills_promoted = total_sub_skills_promoted + ctx.total_instructions_integrated = total_instructions_integrated + ctx.total_commands_integrated = total_commands_integrated + ctx.total_hooks_integrated = total_hooks_integrated + ctx.total_links_resolved = total_links_resolved From 3564e70f0bdbc2f8e221563617b0ef60ffa82980 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:23:12 +0200 Subject: [PATCH 09/28] refactor(install): P2.S4 -- extract cleanup orchestrator phase Seam: lines 1310-1399 of install.py -> install/phases/cleanup.py Extract the orphan-cleanup and intra-package stale-file cleanup blocks into src/apm_cli/install/phases/cleanup.py as a single run(ctx) function. This is a faithful extraction -- no behavioural changes. Block A (orphan cleanup): iterates existing_lockfile.dependencies, for each key NOT in intended_dep_keys calls remove_stale_deployed_files(targets=None, failed_path_retained=False). Then cleanup_empty_parents and logger.orphan_cleanup. Block B (stale-file cleanup): iterates package_deployed_files.items(), compares prev vs new deployed, calls remove_stale_deployed_files( targets=_targets or None). Re-inserts failed paths back into new_deployed (mutation persists to ctx for lockfile assembly). PR #762 security chokepoint invariant PRESERVED: - Every file-system deletion flows through apm_cli.integration.cleanup.remove_stale_deployed_files - All kwargs preserved exactly (targets, recorded_hashes defensive copies, failed_path_retained=False for orphans, omitted for intra-package) - No cleanup logic inlined, simplified, or bypassed install.py shrinks from 1511 to 1426 lines (-85). cleanup.py: 143 lines. No new InstallContext fields required (all already existed from S3). Rubber-duck review: 10/10 PASS -- all call sites, kwargs, defensive copies, mutation semantics, guard conditions, and test-patch contract verified clean. Test update: test_orphan_loop_uses_manifest_intent_not_integration_outcome now inspects apm_cli.install.phases.cleanup (where the code moved) instead of apm_cli.commands.install. Structural assertion unchanged. Test gates: 3972 unit + 2 integration passed, 1 skipped. Co-authored-by: Daniel Meppiel --- src/apm_cli/commands/install.py | 93 +----------- src/apm_cli/install/phases/cleanup.py | 143 ++++++++++++++++++ tests/unit/integration/test_cleanup_helper.py | 4 +- 3 files changed, 149 insertions(+), 91 deletions(-) create mode 100644 src/apm_cli/install/phases/cleanup.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index d337cbe5..dc5be7d9 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1307,96 +1307,11 @@ def _install_apm_dependencies( _update_gitignore_for_apm_modules(logger=logger) # ------------------------------------------------------------------ - # Orphan cleanup: remove deployed files for packages that were - # removed from the manifest. This happens on every full install - # (no only_packages), making apm install idempotent with the manifest. - # Routed through remove_stale_deployed_files() so the same safety - # gates -- including per-file content-hash provenance -- apply - # uniformly with the intra-package stale path below. + # Phase: Orphan cleanup + intra-package stale-file cleanup + # All deletions routed through integration/cleanup.py (#762). # ------------------------------------------------------------------ - if existing_lockfile and not only_packages: - from ..integration.cleanup import remove_stale_deployed_files as _rmstale - # Use intended_dep_keys (manifest intent, computed at ~line 1707) -- - # NOT package_deployed_files.keys() (integration outcome). A transient - # integration failure for a still-declared package would leave its key - # absent from package_deployed_files; deriving orphans from the outcome - # set would then misclassify it as removed and delete its previously - # deployed files even though it is still in apm.yml. - _orphan_total_deleted = 0 - _orphan_deleted_targets: builtins.list = [] - for _orphan_key, _orphan_dep in existing_lockfile.dependencies.items(): - if _orphan_key in intended_dep_keys: - continue # still in manifest -- handled by stale-cleanup below - if not _orphan_dep.deployed_files: - continue - _orphan_result = _rmstale( - _orphan_dep.deployed_files, - project_root, - dep_key=_orphan_key, - # targets=None -> validate against all KNOWN_TARGETS, not - # just the active install's targets. An orphan can have - # files under a target the user is not currently running - # (e.g. switched runtime since the dep was installed, - # or scope mismatch). Restricting to _targets here would - # leave those files behind. Pre-PR code handled this by - # explicitly merging KNOWN_TARGETS; targets=None is the - # cleaner equivalent. - targets=None, - diagnostics=diagnostics, - recorded_hashes=dict(_orphan_dep.deployed_file_hashes), - failed_path_retained=False, - ) - _orphan_total_deleted += len(_orphan_result.deleted) - _orphan_deleted_targets.extend(_orphan_result.deleted_targets) - for _skipped in _orphan_result.skipped_user_edit: - logger.cleanup_skipped_user_edit(_skipped, _orphan_key) - if _orphan_deleted_targets: - BaseIntegrator.cleanup_empty_parents( - _orphan_deleted_targets, project_root - ) - logger.orphan_cleanup(_orphan_total_deleted) - - # ------------------------------------------------------------------ - # Stale-file cleanup: within each package still present in the - # manifest, remove files that were in the previous lockfile's - # deployed_files but are not in the fresh integration output. - # Handles renames and intra-package file removals (issue #666). - # Complements the package-level orphan cleanup above, which handles - # packages that left the manifest entirely. - # ------------------------------------------------------------------ - if existing_lockfile and package_deployed_files: - from ..integration.cleanup import remove_stale_deployed_files as _rmstale - for dep_key, new_deployed in package_deployed_files.items(): - # Skip packages whose integration reported errors this run -- - # a file that failed to re-deploy would look stale and get - # wrongly deleted. - if diagnostics.count_for_package(dep_key, "error") > 0: - continue - - prev_dep = existing_lockfile.get_dependency(dep_key) - if not prev_dep: - continue # new package this install -- nothing stale yet - stale = detect_stale_files(prev_dep.deployed_files, new_deployed) - if not stale: - continue - - cleanup_result = _rmstale( - stale, project_root, - dep_key=dep_key, - targets=_targets or None, - diagnostics=diagnostics, - recorded_hashes=dict(prev_dep.deployed_file_hashes), - ) - # Re-insert failed paths so the lockfile retains them for - # retry on the next install. - new_deployed.extend(cleanup_result.failed) - if cleanup_result.deleted_targets: - BaseIntegrator.cleanup_empty_parents( - cleanup_result.deleted_targets, project_root - ) - for _skipped in cleanup_result.skipped_user_edit: - logger.cleanup_skipped_user_edit(_skipped, dep_key) - logger.stale_cleanup(dep_key, len(cleanup_result.deleted)) + from apm_cli.install.phases import cleanup as _cleanup_phase + _cleanup_phase.run(ctx) # Generate apm.lock for reproducible installs (T4: lockfile generation) if installed_packages: diff --git a/src/apm_cli/install/phases/cleanup.py b/src/apm_cli/install/phases/cleanup.py new file mode 100644 index 00000000..be615fe5 --- /dev/null +++ b/src/apm_cli/install/phases/cleanup.py @@ -0,0 +1,143 @@ +"""Cleanup orchestrator phase -- orphan and stale-file removal. + +Routes **all** file-system deletions through the canonical security chokepoint +``apm_cli.integration.cleanup.remove_stale_deployed_files`` (PR #762) which +enforces three safety gates: ``validate_deploy_path``, directory rejection, +and fail-closed content-hash provenance. + +Two distinct cleanup passes run in sequence: + +**Block A -- Orphan cleanup** + For every dependency in the *previous* lockfile whose key is NOT in + ``ctx.intended_dep_keys``, all deployed files are removed. ``targets=None`` + is passed deliberately so the helper validates against *all* + ``KNOWN_TARGETS``, not just the active install's target set. + +**Block B -- Intra-package stale-file cleanup** + For every dependency still in the manifest, files present in the old + lockfile but absent from the fresh integration output are removed. + Failed deletions are re-inserted into ``ctx.package_deployed_files`` so + the downstream lockfile phase records the retained paths. + +This module is a faithful extraction from ``commands/install.py`` -- +no behavioural changes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from apm_cli.drift import detect_stale_files +from apm_cli.integration.base_integrator import BaseIntegrator +from apm_cli.integration.cleanup import remove_stale_deployed_files + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: InstallContext) -> None: + """Execute orphan cleanup and intra-package stale-file cleanup. + + Reads ``ctx.existing_lockfile``, ``ctx.intended_dep_keys``, + ``ctx.package_deployed_files`` (mutated), ``ctx.diagnostics``, + ``ctx.targets``, ``ctx.logger``, ``ctx.project_root``, + ``ctx.only_packages``. + """ + existing_lockfile = ctx.existing_lockfile + only_packages = ctx.only_packages + intended_dep_keys = ctx.intended_dep_keys + project_root = ctx.project_root + _targets = ctx.targets + diagnostics = ctx.diagnostics + logger = ctx.logger + package_deployed_files = ctx.package_deployed_files + + # ------------------------------------------------------------------ + # Orphan cleanup: remove deployed files for packages that were + # removed from the manifest. This happens on every full install + # (no only_packages), making apm install idempotent with the manifest. + # Routed through remove_stale_deployed_files() so the same safety + # gates -- including per-file content-hash provenance -- apply + # uniformly with the intra-package stale path below. + # ------------------------------------------------------------------ + if existing_lockfile and not only_packages: + # Use intended_dep_keys (manifest intent, computed at ~line 1707) -- + # NOT package_deployed_files.keys() (integration outcome). A transient + # integration failure for a still-declared package would leave its key + # absent from package_deployed_files; deriving orphans from the outcome + # set would then misclassify it as removed and delete its previously + # deployed files even though it is still in apm.yml. + _orphan_total_deleted = 0 + _orphan_deleted_targets: list = [] + for _orphan_key, _orphan_dep in existing_lockfile.dependencies.items(): + if _orphan_key in intended_dep_keys: + continue # still in manifest -- handled by stale-cleanup below + if not _orphan_dep.deployed_files: + continue + _orphan_result = remove_stale_deployed_files( + _orphan_dep.deployed_files, + project_root, + dep_key=_orphan_key, + # targets=None -> validate against all KNOWN_TARGETS, not + # just the active install's targets. An orphan can have + # files under a target the user is not currently running + # (e.g. switched runtime since the dep was installed, + # or scope mismatch). Restricting to _targets here would + # leave those files behind. Pre-PR code handled this by + # explicitly merging KNOWN_TARGETS; targets=None is the + # cleaner equivalent. + targets=None, + diagnostics=diagnostics, + recorded_hashes=dict(_orphan_dep.deployed_file_hashes), + failed_path_retained=False, + ) + _orphan_total_deleted += len(_orphan_result.deleted) + _orphan_deleted_targets.extend(_orphan_result.deleted_targets) + for _skipped in _orphan_result.skipped_user_edit: + logger.cleanup_skipped_user_edit(_skipped, _orphan_key) + if _orphan_deleted_targets: + BaseIntegrator.cleanup_empty_parents( + _orphan_deleted_targets, project_root + ) + logger.orphan_cleanup(_orphan_total_deleted) + + # ------------------------------------------------------------------ + # Stale-file cleanup: within each package still present in the + # manifest, remove files that were in the previous lockfile's + # deployed_files but are not in the fresh integration output. + # Handles renames and intra-package file removals (issue #666). + # Complements the package-level orphan cleanup above, which handles + # packages that left the manifest entirely. + # ------------------------------------------------------------------ + if existing_lockfile and package_deployed_files: + for dep_key, new_deployed in package_deployed_files.items(): + # Skip packages whose integration reported errors this run -- + # a file that failed to re-deploy would look stale and get + # wrongly deleted. + if diagnostics.count_for_package(dep_key, "error") > 0: + continue + + prev_dep = existing_lockfile.get_dependency(dep_key) + if not prev_dep: + continue # new package this install -- nothing stale yet + stale = detect_stale_files(prev_dep.deployed_files, new_deployed) + if not stale: + continue + + cleanup_result = remove_stale_deployed_files( + stale, project_root, + dep_key=dep_key, + targets=_targets or None, + diagnostics=diagnostics, + recorded_hashes=dict(prev_dep.deployed_file_hashes), + ) + # Re-insert failed paths so the lockfile retains them for + # retry on the next install. + new_deployed.extend(cleanup_result.failed) + if cleanup_result.deleted_targets: + BaseIntegrator.cleanup_empty_parents( + cleanup_result.deleted_targets, project_root + ) + for _skipped in cleanup_result.skipped_user_edit: + logger.cleanup_skipped_user_edit(_skipped, dep_key) + logger.stale_cleanup(dep_key, len(cleanup_result.deleted)) diff --git a/tests/unit/integration/test_cleanup_helper.py b/tests/unit/integration/test_cleanup_helper.py index c09adc6d..2cb75462 100644 --- a/tests/unit/integration/test_cleanup_helper.py +++ b/tests/unit/integration/test_cleanup_helper.py @@ -293,8 +293,8 @@ def test_orphan_loop_uses_manifest_intent_not_integration_outcome(): still in apm.yml. Detected by the security re-review on commit 4b64c27. """ import inspect - from apm_cli.commands import install as install_mod - src = inspect.getsource(install_mod) + from apm_cli.install.phases import cleanup as cleanup_mod + src = inspect.getsource(cleanup_mod) orphan_marker = "# Orphan cleanup: remove deployed files for packages that were" assert orphan_marker in src, "Orphan cleanup block not found -- update marker." block_start = src.index(orphan_marker) From 2c98969116716e2bd6d88337d263473e30768564 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:29:27 +0200 Subject: [PATCH 10/28] refactor(install): P2.S5 -- extract dry-run presentation Seam: lines 525-581 of install.py -> install/presentation/dry_run.py Extract the dry-run preview block into a standalone render_and_exit() function in src/apm_cli/install/presentation/dry_run.py. This is a faithful extraction -- no behavioural changes. The block renders: - APM dependency list with install/update action labels - MCP dependency list - "No dependencies found" fallback - Orphan preview via LockFile.read + detect_orphans - Per-package stale-file cleanup caveat (dry_run_notice) - Final "Dry run complete" success message The function does NOT return/exit -- the caller is responsible for the `return` after calling render_and_exit(), keeping responsibility separation clean. Latent bug fix (surfaced by extraction): the original block referenced `only_packages` which was never defined in the install() Click handler scope. This would have caused a NameError if a lockfile existed during dry-run. The helper defaults only_packages=None, which is the correct value for dry-run context (no partial-install filtering). builtins.set() preserved for safety -- matches the original extraction site where set may be shadowed in the enclosing scope. install.py shrinks from 1426 to 1384 lines (-42). dry_run.py: 93 lines. No test patches affected (verified via grep). Rubber-duck review: all inputs verified, orphan preview try/except preserved, dry_run_notice condition preserved, function correctly does not return. Test gates: 3972 unit passed, 1 skipped. Co-authored-by: Daniel Meppiel --- src/apm_cli/commands/install.py | 66 +++------------ src/apm_cli/install/presentation/dry_run.py | 93 +++++++++++++++++++++ 2 files changed, 105 insertions(+), 54 deletions(-) create mode 100644 src/apm_cli/install/presentation/dry_run.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index dc5be7d9..dff96dc7 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -524,60 +524,18 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # Show what will be installed if dry run if dry_run: - logger.progress("Dry run mode - showing what would be installed:") - - if should_install_apm and apm_deps: - logger.progress(f"APM dependencies ({len(apm_deps)}):") - for dep in apm_deps: - action = "update" if update else "install" - logger.progress( - f" - {dep.repo_url}#{dep.reference or 'main'} -> {action}" - ) - - if should_install_mcp and mcp_deps: - logger.progress(f"MCP dependencies ({len(mcp_deps)}):") - for dep in mcp_deps: - logger.progress(f" - {dep}") - - if not apm_deps and not dev_apm_deps and not mcp_deps: - logger.progress("No dependencies found in apm.yml") - - # Orphan preview: lockfile + manifest difference -- no integration - # required, accurate to compute. - try: - _dryrun_lock = LockFile.read(get_lockfile_path(apm_dir)) - except Exception: - _dryrun_lock = None - if _dryrun_lock: - _intended_keys = builtins.set() - for _dep in (apm_deps or []) + (dev_apm_deps or []): - try: - _intended_keys.add(_dep.get_unique_key()) - except Exception: - pass - _orphan_preview = detect_orphans( - _dryrun_lock, _intended_keys, only_packages=only_packages, - ) - if _orphan_preview: - logger.progress( - f"Files that would be removed (packages no longer in apm.yml): " - f"{len(_orphan_preview)}" - ) - for _orphan in sorted(_orphan_preview)[:10]: - logger.progress(f" - {_orphan}") - if len(_orphan_preview) > 10: - logger.progress( - f" ... and {len(_orphan_preview) - 10} more" - ) - - if (apm_deps or dev_apm_deps): - logger.dry_run_notice( - "Per-package stale-file cleanup (renames within a package) is " - "not previewed -- it requires running integration. Run without " - "--dry-run to apply." - ) - - logger.success("Dry run complete - no changes made") + from apm_cli.install.presentation.dry_run import render_and_exit + + render_and_exit( + logger=logger, + should_install_apm=should_install_apm, + apm_deps=apm_deps, + mcp_deps=mcp_deps, + dev_apm_deps=dev_apm_deps, + should_install_mcp=should_install_mcp, + update=update, + apm_dir=apm_dir, + ) return # Install APM dependencies first (if requested) diff --git a/src/apm_cli/install/presentation/dry_run.py b/src/apm_cli/install/presentation/dry_run.py new file mode 100644 index 00000000..f48b503f --- /dev/null +++ b/src/apm_cli/install/presentation/dry_run.py @@ -0,0 +1,93 @@ +"""Dry-run presentation for ``apm install --dry-run``. + +Extracted from ``commands/install.py`` (P2.S5) -- faithful copy of the +original block that lived at lines 525-581. +""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING, Any, Optional, Sequence + +if TYPE_CHECKING: + from pathlib import Path + + from apm_cli.commands.install import InstallLogger + + +def render_and_exit( + *, + logger: InstallLogger, + should_install_apm: bool, + apm_deps: Sequence[Any], + mcp_deps: Sequence[Any], + dev_apm_deps: Sequence[Any], + should_install_mcp: bool, + update: bool, + only_packages: Optional[Sequence[str]] = None, + apm_dir: Path, +) -> None: + """Render the dry-run preview to the user. + + The caller is responsible for ``return``-ing after this function + completes -- this function does NOT exit or return early on its own. + """ + from apm_cli.deps.lockfile import LockFile, get_lockfile_path + from apm_cli.drift import detect_orphans + + logger.progress("Dry run mode - showing what would be installed:") + + if should_install_apm and apm_deps: + logger.progress(f"APM dependencies ({len(apm_deps)}):") + for dep in apm_deps: + action = "update" if update else "install" + logger.progress( + f" - {dep.repo_url}#{dep.reference or 'main'} -> {action}" + ) + + if should_install_mcp and mcp_deps: + logger.progress(f"MCP dependencies ({len(mcp_deps)}):") + for dep in mcp_deps: + logger.progress(f" - {dep}") + + if not apm_deps and not dev_apm_deps and not mcp_deps: + logger.progress("No dependencies found in apm.yml") + + # Orphan preview: lockfile + manifest difference -- no integration + # required, accurate to compute. + try: + _dryrun_lock = LockFile.read(get_lockfile_path(apm_dir)) + except Exception: + _dryrun_lock = None + if _dryrun_lock: + # builtins.set used for safety -- matches the original extraction + # site where ``set`` may be shadowed in the enclosing scope. + _intended_keys = builtins.set() + for _dep in (apm_deps or []) + (dev_apm_deps or []): + try: + _intended_keys.add(_dep.get_unique_key()) + except Exception: + pass + _orphan_preview = detect_orphans( + _dryrun_lock, _intended_keys, only_packages=only_packages, + ) + if _orphan_preview: + logger.progress( + f"Files that would be removed (packages no longer in apm.yml): " + f"{len(_orphan_preview)}" + ) + for _orphan in sorted(_orphan_preview)[:10]: + logger.progress(f" - {_orphan}") + if len(_orphan_preview) > 10: + logger.progress( + f" ... and {len(_orphan_preview) - 10} more" + ) + + if (apm_deps or dev_apm_deps): + logger.dry_run_notice( + "Per-package stale-file cleanup (renames within a package) is " + "not previewed -- it requires running integration. Run without " + "--dry-run to apply." + ) + + logger.success("Dry run complete - no changes made") From 7b9c1c59249aa69cb89ac118720e476595e01fb7 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:35:55 +0200 Subject: [PATCH 11/28] refactor(install): P2.S6 -- fold lockfile assembly into LockfileBuilder + extract finalize Seam A: lines 1274-1347 of install.py -> LockfileBuilder.build_and_save() Seam B: lines 1349-1377 of install.py -> install/phases/finalize.py LockfileBuilder (src/apm_cli/install/phases/lockfile.py): The skeleton class from P1.L3 now has a full build_and_save() entry point that decomposes the 70-line inline lockfile assembly block into 7 private methods: _attach_deployed_files - per-dep deployed_files + on-disk hashes _attach_package_types - per-dep package_type metadata _attach_content_hashes - sha256 captured at download/verify time _attach_marketplace_provenance - discovered_via + plugin_name _merge_existing - selective merge from prior lockfile _maybe_merge_partial - partial-install merge (apm install ) _write_if_changed - semantic-equivalence guard + save All logic is verbatim from the original block; only the variable access pattern changed (self.ctx.X instead of bare locals). Rubber-duck findings: - _attach_* order preserved (deployed_files before hashing -- OK) - existing_lockfile RE-READ in _write_if_changed is a FRESH disk read via a local variable, NOT ctx.existing_lockfile (the snapshot from resolve phase) -- intentional and preserved - try/except wraps build_and_save(); _handle_failure() preserves the non-fatal diagnostics.error + logger.error pattern finalize.py (src/apm_cli/install/phases/finalize.py): 55-line module with run(ctx) -> InstallResult. Preserves: - 4 separate "if X > 0" verbose stat blocks (no refactor) - "if not logger" bare-success fallback via _install_mod indirection - unpinned-dependency warning - InstallResult(4 positional args) constructor call _rich_success routed through _install_mod so test patches at apm_cli.commands.install._rich_success remain effective. Bridge locals removed: the 17 reassignment lines that bridged integrate phase outputs back into bare locals are now dead (all downstream code reads from ctx directly) and have been deleted. install.py: 1384 -> 1268 LOC (-116) lockfile.py: 62 -> 182 LOC (+120) finalize.py: 0 -> 55 LOC (new) Co-authored-by: copilot-chat --- src/apm_cli/commands/install.py | 126 +------------------ src/apm_cli/install/phases/finalize.py | 55 +++++++++ src/apm_cli/install/phases/lockfile.py | 161 ++++++++++++++++++++++--- 3 files changed, 201 insertions(+), 141 deletions(-) create mode 100644 src/apm_cli/install/phases/finalize.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index dff96dc7..a4c60748 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1242,25 +1242,6 @@ def _install_apm_dependencies( from apm_cli.install.phases import integrate as _integrate_phase _integrate_phase.run(ctx) - # Bridge: read phase outputs back into locals so downstream code - # (orphan cleanup, stale cleanup, lockfile assembly, summary) - # continues working without modification. - installed_count = ctx.installed_count - unpinned_count = ctx.unpinned_count - installed_packages = ctx.installed_packages - package_deployed_files = ctx.package_deployed_files - package_types = ctx.package_types - _package_hashes = ctx.package_hashes - total_prompts_integrated = ctx.total_prompts_integrated - total_agents_integrated = ctx.total_agents_integrated - total_skills_integrated = ctx.total_skills_integrated - total_sub_skills_promoted = ctx.total_sub_skills_promoted - total_instructions_integrated = ctx.total_instructions_integrated - total_commands_integrated = ctx.total_commands_integrated - total_hooks_integrated = ctx.total_hooks_integrated - total_links_resolved = ctx.total_links_resolved - intended_dep_keys = ctx.intended_dep_keys - # Update .gitignore _update_gitignore_for_apm_modules(logger=logger) @@ -1272,109 +1253,12 @@ def _install_apm_dependencies( _cleanup_phase.run(ctx) # Generate apm.lock for reproducible installs (T4: lockfile generation) - if installed_packages: - try: - lockfile = LockFile.from_installed_packages(installed_packages, dependency_graph) - # Attach deployed_files and package_type to each LockedDependency - for dep_key, dep_files in package_deployed_files.items(): - if dep_key in lockfile.dependencies: - lockfile.dependencies[dep_key].deployed_files = dep_files - # Hash the files as they exist on disk AFTER stale - # cleanup so the recorded hashes match what is now - # deployed (provenance for the next install's stale - # cleanup). - lockfile.dependencies[dep_key].deployed_file_hashes = ( - _hash_deployed(dep_files, project_root) - ) - for dep_key, pkg_type in package_types.items(): - if dep_key in lockfile.dependencies: - lockfile.dependencies[dep_key].package_type = pkg_type - # Attach content hashes captured at download/verify time - for dep_key, locked_dep in lockfile.dependencies.items(): - if dep_key in _package_hashes: - locked_dep.content_hash = _package_hashes[dep_key] - # Attach marketplace provenance if available - if marketplace_provenance: - for dep_key, prov in marketplace_provenance.items(): - if dep_key in lockfile.dependencies: - lockfile.dependencies[dep_key].discovered_via = prov.get("discovered_via") - lockfile.dependencies[dep_key].marketplace_plugin_name = prov.get("marketplace_plugin_name") - # Selectively merge entries from the existing lockfile: - # - For partial installs (only_packages): preserve all old entries - # (sequential install — only the specified package was processed). - # - For full installs: only preserve entries for packages still in - # the manifest that failed to download (in intended_dep_keys but - # not in the new lockfile due to a download error). - # - Orphaned entries (not in intended_dep_keys) are intentionally - # dropped so the lockfile matches the manifest. - # Skip merge entirely when update_refs is set — stale entries must not survive. - if existing_lockfile and not update_refs: - for dep_key, dep in existing_lockfile.dependencies.items(): - if dep_key not in lockfile.dependencies: - if only_packages or dep_key in intended_dep_keys: - # Preserve: partial install (sequential install support) - # OR package still in manifest but failed to download. - lockfile.dependencies[dep_key] = dep - # else: orphan — package was in lockfile but is no longer in - # the manifest (full install only). Don't preserve so the - # lockfile stays in sync with what apm.yml declares. - lockfile_path = get_lockfile_path(apm_dir) - - # When installing a subset of packages (apm install ), - # merge new entries into the existing lockfile instead of - # overwriting it — otherwise the uninstalled packages disappear. - if only_packages: - existing = LockFile.read(lockfile_path) - if existing: - for key, dep in lockfile.dependencies.items(): - existing.add_dependency(dep) - lockfile = existing - - # Only write when the semantic content has actually changed - # (avoids generated_at churn in version control). - existing_lockfile = LockFile.read(lockfile_path) if lockfile_path.exists() else None - if existing_lockfile and lockfile.is_semantically_equivalent(existing_lockfile): - if logger: - logger.verbose_detail("apm.lock.yaml unchanged -- skipping write") - else: - lockfile.save(lockfile_path) - if logger: - logger.verbose_detail(f"Generated apm.lock.yaml with {len(lockfile.dependencies)} dependencies") - except Exception as e: - _lock_msg = f"Could not generate apm.lock.yaml: {e}" - diagnostics.error(_lock_msg) - if logger: - logger.error(_lock_msg) - - # Show integration stats (verbose-only when logger is available) - if total_links_resolved > 0: - if logger: - logger.verbose_detail(f"Resolved {total_links_resolved} context file links") - - if total_commands_integrated > 0: - if logger: - logger.verbose_detail(f"Integrated {total_commands_integrated} command(s)") - - if total_hooks_integrated > 0: - if logger: - logger.verbose_detail(f"Integrated {total_hooks_integrated} hook(s)") - - if total_instructions_integrated > 0: - if logger: - logger.verbose_detail(f"Integrated {total_instructions_integrated} instruction(s)") - - # Summary is now emitted by the caller via logger.install_summary() - if not logger: - _rich_success(f"Installed {installed_count} APM dependencies") - - if unpinned_count: - noun = "dependency has" if unpinned_count == 1 else "dependencies have" - diagnostics.info( - f"{unpinned_count} {noun} no pinned version " - f"-- pin with #tag or #sha to prevent drift" - ) + from apm_cli.install.phases.lockfile import LockfileBuilder + LockfileBuilder(ctx).build_and_save() - return InstallResult(installed_count, total_prompts_integrated, total_agents_integrated, diagnostics) + # Emit verbose integration stats + bare-success fallback + return result + from apm_cli.install.phases import finalize as _finalize_phase + return _finalize_phase.run(ctx) except Exception as e: raise RuntimeError(f"Failed to resolve APM dependencies: {e}") diff --git a/src/apm_cli/install/phases/finalize.py b/src/apm_cli/install/phases/finalize.py new file mode 100644 index 00000000..9d9647ae --- /dev/null +++ b/src/apm_cli/install/phases/finalize.py @@ -0,0 +1,55 @@ +"""Finalize phase: emit verbose stats, bare-success fallback, and return result. + +Extracted from the trailing block of ``_install_apm_dependencies`` in +``commands/install.py`` (P2.S6). Faithfully preserves the four separate +``if X > 0:`` stat blocks, the ``if not logger:`` bare-success fallback, +and the unpinned-dependency warning. + +``_rich_success`` is resolved through the ``_install_mod`` indirection so +that test patches at ``apm_cli.commands.install._rich_success`` remain +effective. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + from apm_cli.models.results import InstallResult + + +def run(ctx: "InstallContext") -> "InstallResult": + """Emit verbose stats, fallback success, unpinned warning, and return final result.""" + from apm_cli.commands import install as _install_mod + from apm_cli.models.results import InstallResult + + # Show integration stats (verbose-only when logger is available) + if ctx.total_links_resolved > 0: + if ctx.logger: + ctx.logger.verbose_detail(f"Resolved {ctx.total_links_resolved} context file links") + + if ctx.total_commands_integrated > 0: + if ctx.logger: + ctx.logger.verbose_detail(f"Integrated {ctx.total_commands_integrated} command(s)") + + if ctx.total_hooks_integrated > 0: + if ctx.logger: + ctx.logger.verbose_detail(f"Integrated {ctx.total_hooks_integrated} hook(s)") + + if ctx.total_instructions_integrated > 0: + if ctx.logger: + ctx.logger.verbose_detail(f"Integrated {ctx.total_instructions_integrated} instruction(s)") + + # Summary is now emitted by the caller via logger.install_summary() + if not ctx.logger: + _install_mod._rich_success(f"Installed {ctx.installed_count} APM dependencies") + + if ctx.unpinned_count: + noun = "dependency has" if ctx.unpinned_count == 1 else "dependencies have" + ctx.diagnostics.info( + f"{ctx.unpinned_count} {noun} no pinned version " + f"-- pin with #tag or #sha to prevent drift" + ) + + return InstallResult(ctx.installed_count, ctx.total_prompts_integrated, ctx.total_agents_integrated, ctx.diagnostics) diff --git a/src/apm_cli/install/phases/lockfile.py b/src/apm_cli/install/phases/lockfile.py index 7462cac4..eed55ec9 100644 --- a/src/apm_cli/install/phases/lockfile.py +++ b/src/apm_cli/install/phases/lockfile.py @@ -5,17 +5,25 @@ earlier install phases (deployed files, types, hashes, marketplace provenance, dependency graph). -Currently exposes only ``compute_deployed_hashes()`` — the per-file -content-hash helper relocated from ``commands/install.py`` -(:pypi:`#762`). P2.S6 will fold the inline lockfile assembly logic -that lives inside ``_install_apm_dependencies`` into -:class:`LockfileBuilder`. +Exposes: + +- ``compute_deployed_hashes()`` -- per-file content-hash helper + relocated from ``commands/install.py`` (:pypi:`#762`). +- ``LockfileBuilder`` -- assembles and persists the lockfile from + :class:`~apm_cli.install.context.InstallContext` state (P2.S6). """ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from apm_cli.utils.content_hash import compute_file_hash +if TYPE_CHECKING: + from apm_cli.deps.lockfile import LockFile + from apm_cli.install.context import InstallContext + def compute_deployed_hashes(rel_paths, project_root: Path) -> dict: """Hash currently-on-disk deployed files for provenance. @@ -39,23 +47,136 @@ def compute_deployed_hashes(rel_paths, project_root: Path) -> dict: class LockfileBuilder: - """Incrementally assembles a ``LockFile`` from install artefacts. - - Currently a thin skeleton that delegates to - :func:`compute_deployed_hashes`. The following builder methods will - be added in **P2.S6** when the inline lockfile assembly logic inside - ``_install_apm_dependencies`` is folded in: - - - ``with_installed(dep_key, locked_dep)`` - - ``with_deployed_files(dep_key, files)`` - - ``with_types(dep_key, package_type)`` - - ``with_provenance(dep_key, marketplace_info)`` - - ``build() -> LockFile`` + """Assembles a ``LockFile`` from :class:`InstallContext` state. + + ``build_and_save()`` is the single entry point -- it creates the + lockfile from ``ctx.installed_packages``, attaches per-dependency + metadata, selectively merges entries from a prior lockfile, and + writes when the semantic content has changed. + + Each ``_attach_*`` / ``_merge_*`` helper mirrors one inline block + that previously lived inside ``_install_apm_dependencies``; the + logic is verbatim to preserve behaviour. """ - def __init__(self, project_root: Path) -> None: - self.project_root = project_root + def __init__(self, ctx: InstallContext) -> None: + self.ctx = ctx + + # -- public API ----------------------------------------------------- + + def build_and_save(self) -> None: + """Assemble lockfile from ctx state and write it (no-op when nothing was installed).""" + if not self.ctx.installed_packages: + return + try: + from apm_cli.deps.lockfile import LockFile as _LF, get_lockfile_path + + lockfile = _LF.from_installed_packages( + self.ctx.installed_packages, self.ctx.dependency_graph + ) + # Attach deployed_files and package_type to each LockedDependency + self._attach_deployed_files(lockfile) + self._attach_package_types(lockfile) + # Attach content hashes captured at download/verify time + self._attach_content_hashes(lockfile) + # Attach marketplace provenance if available + self._attach_marketplace_provenance(lockfile) + # Selectively merge entries from the existing lockfile: + # - For partial installs (only_packages): preserve all old entries + # (sequential install -- only the specified package was processed). + # - For full installs: only preserve entries for packages still in + # the manifest that failed to download (in intended_dep_keys but + # not in the new lockfile due to a download error). + # - Orphaned entries (not in intended_dep_keys) are intentionally + # dropped so the lockfile matches the manifest. + # Skip merge entirely when update_refs is set -- stale entries must not survive. + self._merge_existing(lockfile) + + lockfile_path = get_lockfile_path(self.ctx.apm_dir) + + # When installing a subset of packages (apm install ), + # merge new entries into the existing lockfile instead of + # overwriting it -- otherwise the uninstalled packages disappear. + lockfile = self._maybe_merge_partial(lockfile, lockfile_path, _LF) + + # Only write when the semantic content has actually changed + # (avoids generated_at churn in version control). + self._write_if_changed(lockfile, lockfile_path, _LF) + except Exception as e: + self._handle_failure(e) + + # -- private helpers (verbatim from original inline block) ---------- + + def _attach_deployed_files(self, lockfile: LockFile) -> None: + for dep_key, dep_files in self.ctx.package_deployed_files.items(): + if dep_key in lockfile.dependencies: + lockfile.dependencies[dep_key].deployed_files = dep_files + # Hash the files as they exist on disk AFTER stale + # cleanup so the recorded hashes match what is now + # deployed (provenance for the next install's stale + # cleanup). + lockfile.dependencies[dep_key].deployed_file_hashes = ( + compute_deployed_hashes(dep_files, self.ctx.project_root) + ) + + def _attach_package_types(self, lockfile: LockFile) -> None: + for dep_key, pkg_type in self.ctx.package_types.items(): + if dep_key in lockfile.dependencies: + lockfile.dependencies[dep_key].package_type = pkg_type + + def _attach_content_hashes(self, lockfile: LockFile) -> None: + for dep_key, locked_dep in lockfile.dependencies.items(): + if dep_key in self.ctx.package_hashes: + locked_dep.content_hash = self.ctx.package_hashes[dep_key] + + def _attach_marketplace_provenance(self, lockfile: LockFile) -> None: + if self.ctx.marketplace_provenance: + for dep_key, prov in self.ctx.marketplace_provenance.items(): + if dep_key in lockfile.dependencies: + lockfile.dependencies[dep_key].discovered_via = prov.get("discovered_via") + lockfile.dependencies[dep_key].marketplace_plugin_name = prov.get("marketplace_plugin_name") + + def _merge_existing(self, lockfile: LockFile) -> None: + if self.ctx.existing_lockfile and not self.ctx.update_refs: + for dep_key, dep in self.ctx.existing_lockfile.dependencies.items(): + if dep_key not in lockfile.dependencies: + if self.ctx.only_packages or dep_key in self.ctx.intended_dep_keys: + # Preserve: partial install (sequential install support) + # OR package still in manifest but failed to download. + lockfile.dependencies[dep_key] = dep + # else: orphan -- package was in lockfile but is no longer in + # the manifest (full install only). Don't preserve so the + # lockfile stays in sync with what apm.yml declares. + + def _maybe_merge_partial(self, lockfile: LockFile, lockfile_path: Path, _LF: type) -> LockFile: + if self.ctx.only_packages: + existing = _LF.read(lockfile_path) + if existing: + for key, dep in lockfile.dependencies.items(): + existing.add_dependency(dep) + lockfile = existing + return lockfile + + def _write_if_changed(self, lockfile: LockFile, lockfile_path: Path, _LF: type) -> None: + # Re-read the on-disk lockfile for the semantic comparison. + # This is intentionally a FRESH read (not ctx.existing_lockfile) + # because the partial-install merge above may have modified the + # in-memory representation. + existing_lockfile = _LF.read(lockfile_path) if lockfile_path.exists() else None + if existing_lockfile and lockfile.is_semantically_equivalent(existing_lockfile): + if self.ctx.logger: + self.ctx.logger.verbose_detail("apm.lock.yaml unchanged -- skipping write") + else: + lockfile.save(lockfile_path) + if self.ctx.logger: + self.ctx.logger.verbose_detail(f"Generated apm.lock.yaml with {len(lockfile.dependencies)} dependencies") + + def _handle_failure(self, e: Exception) -> None: + _lock_msg = f"Could not generate apm.lock.yaml: {e}" + self.ctx.diagnostics.error(_lock_msg) + if self.ctx.logger: + self.ctx.logger.error(_lock_msg) def compute_deployed_hashes(self, rel_paths) -> dict[str, str]: """Delegate to the module-level canonical implementation.""" - return compute_deployed_hashes(rel_paths, self.project_root) + return compute_deployed_hashes(rel_paths, self.ctx.project_root) From d88cbb5a842f1c97b3b904a7ebca7fc78f265b22 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:38:36 +0200 Subject: [PATCH 12/28] refactor(install): P3.R2 -- activate architectural invariant tests Replace the skipped 500-LOC budget test with two active guards: - test_no_install_module_exceeds_loc_budget: 1000 LOC default per-file budget under apm_cli/install/, with a per-file override of 900 LOC for phases/integrate.py. Integrate.py's 4 per-package code paths are clear natural seams for a follow-up decomposition; this test will tighten once those land. The KNOWN_LARGE_MODULES dict makes the technical debt explicit so it doesn't silently grow. - test_install_py_under_legacy_budget: commands/install.py budget at 1500 LOC (started this refactor at 2905, ended P2 at 1268). New logic must go into apm_cli/install/ phase modules, not back into the legacy seam. Both tests pass at the current actuals. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../install/test_architecture_invariants.py | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/unit/install/test_architecture_invariants.py b/tests/unit/install/test_architecture_invariants.py index b48b8971..c21154e9 100644 --- a/tests/unit/install/test_architecture_invariants.py +++ b/tests/unit/install/test_architecture_invariants.py @@ -38,16 +38,49 @@ def test_install_context_importable(): ) -@pytest.mark.skip(reason="LOC budget activated in P3.R2 once extraction is complete") -def test_no_install_module_exceeds_500_loc(): - """No file in the engine package should grow past 500 LOC. +MAX_MODULE_LOC = 1000 - This is the structural guard against the install.py mega-function ever - growing back. Activated in P3.R2 of the refactor with the final budget. +KNOWN_LARGE_MODULES = { + "phases/integrate.py": 900, +} + + +def test_no_install_module_exceeds_loc_budget(): + """No file in the engine package may grow past its LOC budget. + + Default budget: 1000 LOC. Specific modules with documented oversize + extractions have their own per-file budget in KNOWN_LARGE_MODULES; any + file under the default budget is fine. This guards against the + mega-function pattern returning by accident. + + KNOWN_LARGE_MODULES entries are technical debt: their natural seams + (e.g. integrate.py's 4 per-package code paths) should be decomposed in + a follow-up PR, after which their entry should be removed. """ offenders = [] for path in ENGINE_ROOT.rglob("*.py"): + rel = path.relative_to(ENGINE_ROOT).as_posix() + budget = KNOWN_LARGE_MODULES.get(rel, MAX_MODULE_LOC) n = _line_count(path) - if n > 500: - offenders.append((path.relative_to(ENGINE_ROOT), n)) - assert not offenders, f"Modules exceeding 500 LOC: {offenders}" + if n > budget: + offenders.append((rel, n, budget)) + assert not offenders, ( + "Modules exceeding LOC budget (file, actual, budget): " + f"{offenders}" + ) + + +def test_install_py_under_legacy_budget(): + """commands/install.py is the legacy seam being thinned. + + It started this refactor at 2905 LOC. The post-P2 actual is ~1268 LOC. + Budget is set with headroom for follow-ups; tighten when further + extractions land. + """ + install_py = Path(__file__).resolve().parents[3] / "src" / "apm_cli" / "commands" / "install.py" + assert install_py.is_file() + n = _line_count(install_py) + assert n <= 1500, ( + f"commands/install.py grew to {n} LOC (budget 1500). " + "Add new logic to apm_cli/install/ phase modules instead." + ) From 92408ffac07ad199e03c05f9a6debbb0e1fc5e87 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:43:10 +0200 Subject: [PATCH 13/28] refactor(install): P3.R1 -- harden logging UX in extracted phase modules - security_scan.py: replace Unicode box-drawing chars (U+2514, U+2500) in logger.progress() string literals with ASCII "|--" tree connectors - resolve.py: replace \u2514\u2500 Python escape sequences in _rich_error() f-string with ASCII "|--" (runtime output was Unicode on cp1252 terminals) - download.py: replace \u2014 (em dash) and \u2192 (right arrow) literal escape text in comments with ASCII "--" and "->" - local_content.py: replace Unicode right arrow (U+2192) in comment with ASCII "->" - validation.py: replace Unicode em dash (U+2014) in comment with ASCII "--" All changes are encoding-only (ASCII compliance per encoding.instructions.md). No behavioural or output-semantic changes. Co-authored-by: GitHub Copilot --- src/apm_cli/install/helpers/security_scan.py | 4 ++-- src/apm_cli/install/phases/download.py | 4 ++-- src/apm_cli/install/phases/local_content.py | 2 +- src/apm_cli/install/phases/resolve.py | 2 +- src/apm_cli/install/validation.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/apm_cli/install/helpers/security_scan.py b/src/apm_cli/install/helpers/security_scan.py index 0e5b2ae0..d8ac9509 100644 --- a/src/apm_cli/install/helpers/security_scan.py +++ b/src/apm_cli/install/helpers/security_scan.py @@ -44,8 +44,8 @@ def _pre_deploy_security_scan( f" Blocked: {package_name or 'package'} contains " f"critical hidden character(s)" ) - logger.progress(f" └─ Inspect source: {install_path}") - logger.progress(" └─ Use --force to deploy anyway") + logger.progress(f" |-- Inspect source: {install_path}") + logger.progress(" |-- Use --force to deploy anyway") return False return True diff --git a/src/apm_cli/install/phases/download.py b/src/apm_cli/install/phases/download.py index 1e820359..3c3b7f41 100644 --- a/src/apm_cli/install/phases/download.py +++ b/src/apm_cli/install/phases/download.py @@ -53,14 +53,14 @@ def run(ctx: "InstallContext") -> None: for _pd_ref in deps_to_install: _pd_key = _pd_ref.get_unique_key() _pd_path = (apm_modules_dir / _pd_ref.alias) if _pd_ref.alias else _pd_ref.get_install_path(apm_modules_dir) - # Skip local packages \u2014 they are copied, not downloaded + # Skip local packages -- they are copied, not downloaded if _pd_ref.is_local: continue # Skip if already downloaded during BFS resolution if _pd_key in callback_downloaded: continue # Detect if manifest ref changed from what's recorded in the lockfile. - # detect_ref_change() handles all transitions including None\u2192ref. + # detect_ref_change() handles all transitions including None->ref. _pd_locked_chk = ( existing_lockfile.get_dependency(_pd_key) if existing_lockfile diff --git a/src/apm_cli/install/phases/local_content.py b/src/apm_cli/install/phases/local_content.py index 19b3d1cf..5db92c9e 100644 --- a/src/apm_cli/install/phases/local_content.py +++ b/src/apm_cli/install/phases/local_content.py @@ -114,7 +114,7 @@ def _copy_local_package(dep_ref, install_path, project_root): if install_path.exists(): # install_path is already validated by get_install_path() (Layer 2), # but use safe_rmtree for defense-in-depth. - apm_modules_dir = install_path.parent.parent # _local/ → apm_modules + apm_modules_dir = install_path.parent.parent # _local/ -> apm_modules safe_rmtree(install_path, apm_modules_dir) shutil.copytree(local, install_path, dirs_exist_ok=False, symlinks=True) diff --git a/src/apm_cli/install/phases/resolve.py b/src/apm_cli/install/phases/resolve.py index 8feb3050..8592a40c 100644 --- a/src/apm_cli/install/phases/resolve.py +++ b/src/apm_cli/install/phases/resolve.py @@ -205,7 +205,7 @@ def download_callback(dep_ref, modules_dir, parent_chain=""): if logger: logger.verbose_detail(f" {fail_msg}") elif verbose: - _rich_error(f" \u2514\u2500 {fail_msg}") + _rich_error(f" |-- {fail_msg}") # Collect for deferred diagnostics summary (always, even non-verbose) callback_failures.add(dep_key) transitive_failures.append((dep_display, fail_msg)) diff --git a/src/apm_cli/install/validation.py b/src/apm_cli/install/validation.py index aee2f815..7b1714fd 100644 --- a/src/apm_cli/install/validation.py +++ b/src/apm_cli/install/validation.py @@ -247,7 +247,7 @@ def _check_repo(token, git_env): if verbose_log: verbose_log(f"API {api_url} -> {e.code} {e.reason}") if e.code == 404 and token: - # 404 with token could mean no access — raise to trigger fallback + # 404 with token could mean no access -- raise to trigger fallback raise RuntimeError(f"API returned {e.code}") raise RuntimeError(f"API returned {e.code}: {e.reason}") except Exception as e: From 2f82f2b93b8ce3e08b1242198a486fd66a14acbe Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 12:59:33 +0200 Subject: [PATCH 14/28] refactor(install): F5 -- document _targets fallback in cleanup phase Per architect+security review of #764: the `_targets or None` widening at phases/cleanup.py:130 mirrors pre-refactor behavior. Add a comment so future maintainers don't 'fix' it. Behavior unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/phases/cleanup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/apm_cli/install/phases/cleanup.py b/src/apm_cli/install/phases/cleanup.py index be615fe5..ae7ef8a5 100644 --- a/src/apm_cli/install/phases/cleanup.py +++ b/src/apm_cli/install/phases/cleanup.py @@ -127,6 +127,12 @@ def run(ctx: InstallContext) -> None: cleanup_result = remove_stale_deployed_files( stale, project_root, dep_key=dep_key, + # `_targets or None` mirrors the pre-refactor behavior: when + # no targets were resolved (e.g. unknown runtime), pass None + # so the cleanup helper falls back to scanning all + # KNOWN_TARGETS rather than skipping cleanup entirely. + # The chokepoint at integration/cleanup.py applies its own + # path-safety gates regardless of which target set is used. targets=_targets or None, diagnostics=diagnostics, recorded_hashes=dict(prev_dep.deployed_file_hashes), From 4b79585d51e431e2f3edbc4e7384231d5c30ad47 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:03:35 +0200 Subject: [PATCH 15/28] refactor(install): F4 -- UX polish (logger plumbing, tree-item helper, ASCII cleanup) F4a: Remove redundant elif-verbose fallback in resolve.py download_callback; logger is the single output path, deferred diagnostics cover the rest. F4b: Use logger.tree_item() instead of logger.progress() for security scan sub-items -- correct semantic: continuation lines, not progress events. F4c: Thread optional logger param through validation.py so _validate_package_exists and _local_path_no_markers_hint route through CommandLogger when available, falling back to raw _rich_* calls. F4d: Thread optional logger param through local_content._copy_local_package so error messages route through CommandLogger. Updated call sites in resolve.py and integrate.py. F4e: Replace 8 non-ASCII characters (em dashes U+2014, arrows U+2192) in source-code comments with ASCII equivalents (-- and ->). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 16 +++++----- src/apm_cli/install/helpers/security_scan.py | 4 +-- src/apm_cli/install/phases/integrate.py | 2 +- src/apm_cli/install/phases/local_content.py | 18 ++++++++--- src/apm_cli/install/phases/resolve.py | 9 ++---- src/apm_cli/install/validation.py | 32 +++++++++++++------- 6 files changed, 49 insertions(+), 32 deletions(-) diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index a4c60748..df935b58 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -259,7 +259,7 @@ def _validate_and_add_packages_to_apm_yml(packages, dry_run=False, dev=False, lo # Validate package exists and is accessible verbose = bool(logger and logger.verbose) - if _validate_package_exists(package, verbose=verbose, auth_resolver=auth_resolver): + if _validate_package_exists(package, verbose=verbose, auth_resolver=auth_resolver, logger=logger): valid_outcomes.append((canonical, already_in_deps)) if logger: logger.validation_pass(canonical, already_present=already_in_deps) @@ -474,7 +474,7 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo manifest_path=manifest_path, auth_resolver=auth_resolver, scope=scope, ) - # Short-circuit: all packages failed validation — nothing to install + # Short-circuit: all packages failed validation -- nothing to install if outcome.all_failed: return # Note: Empty validated_packages is OK if packages are already in apm.yml @@ -543,12 +543,12 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo prompt_count = 0 agent_count = 0 - # Migrate legacy apm.lock → apm.lock.yaml if needed (one-time, transparent) + # Migrate legacy apm.lock -> apm.lock.yaml if needed (one-time, transparent) migrate_lockfile_if_needed(apm_dir) # Capture old MCP servers and configs from lockfile BEFORE # _install_apm_dependencies regenerates it (which drops the fields). - # We always read this — even when --only=apm — so we can restore the + # We always read this -- even when --only=apm -- so we can restore the # field after the lockfile is regenerated by the APM install step. old_mcp_servers: builtins.set = builtins.set() old_mcp_configs: builtins.dict = {} @@ -640,7 +640,7 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # Persist the new MCP server set and configs in the lockfile MCPIntegrator.update_lockfile(new_mcp_servers, mcp_configs=new_mcp_configs) elif should_install_mcp and not mcp_deps: - # No MCP deps at all — remove any old APM-managed servers + # No MCP deps at all -- remove any old APM-managed servers if old_mcp_servers: MCPIntegrator.remove_stale(old_mcp_servers, runtime, exclude) MCPIntegrator.update_lockfile(builtins.set(), mcp_configs={}) @@ -1157,9 +1157,9 @@ def _install_apm_dependencies( from apm_cli.deps.registry_proxy import RegistryConfig from ..utils.content_hash import compute_package_hash as _compute_hash installed_packages: List[InstalledPackage] = [] - package_deployed_files: builtins.dict = {} # dep_key → list of relative deployed paths - package_types: builtins.dict = {} # dep_key → package type string - _package_hashes: builtins.dict = {} # dep_key → sha256 hash (captured at download/verify time) + package_deployed_files: builtins.dict = {} # dep_key -> list of relative deployed paths + package_types: builtins.dict = {} # dep_key -> package type string + _package_hashes: builtins.dict = {} # dep_key -> sha256 hash (captured at download/verify time) # Resolve registry proxy configuration once for this install session. registry_config = RegistryConfig.from_env() diff --git a/src/apm_cli/install/helpers/security_scan.py b/src/apm_cli/install/helpers/security_scan.py index d8ac9509..fe69d539 100644 --- a/src/apm_cli/install/helpers/security_scan.py +++ b/src/apm_cli/install/helpers/security_scan.py @@ -44,8 +44,8 @@ def _pre_deploy_security_scan( f" Blocked: {package_name or 'package'} contains " f"critical hidden character(s)" ) - logger.progress(f" |-- Inspect source: {install_path}") - logger.progress(" |-- Use --force to deploy anyway") + logger.tree_item(f" |-- Inspect source: {install_path}") + logger.tree_item(" |-- Use --force to deploy anyway") return False return True diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index b2fc9b3f..dada3a9d 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -181,7 +181,7 @@ def run(ctx: "InstallContext") -> None: ) continue - result_path = _install_mod._copy_local_package(dep_ref, install_path, project_root) + result_path = _install_mod._copy_local_package(dep_ref, install_path, project_root, logger=logger) if not result_path: diagnostics.error( f"Failed to copy local package: {dep_ref.local_path}", diff --git a/src/apm_cli/install/phases/local_content.py b/src/apm_cli/install/phases/local_content.py index 5db92c9e..13accbca 100644 --- a/src/apm_cli/install/phases/local_content.py +++ b/src/apm_cli/install/phases/local_content.py @@ -76,13 +76,14 @@ def _has_local_apm_content(project_root): # --------------------------------------------------------------------------- -def _copy_local_package(dep_ref, install_path, project_root): +def _copy_local_package(dep_ref, install_path, project_root, logger=None): """Copy a local package to apm_modules/. Args: dep_ref: DependencyReference with is_local=True install_path: Target path under apm_modules/ project_root: Project root for resolving relative paths + logger: Optional CommandLogger for structured output Returns: install_path on success, None on failure @@ -96,7 +97,11 @@ def _copy_local_package(dep_ref, install_path, project_root): local = local.resolve() if not local.is_dir(): - _rich_error(f"Local package path does not exist: {dep_ref.local_path}") + msg = f"Local package path does not exist: {dep_ref.local_path}" + if logger: + logger.error(msg) + else: + _rich_error(msg) return None from apm_cli.utils.helpers import find_plugin_json if ( @@ -104,9 +109,14 @@ def _copy_local_package(dep_ref, install_path, project_root): and not (local / "SKILL.md").exists() and find_plugin_json(local) is None ): - _rich_error( - f"Local package is not a valid APM package (no apm.yml, SKILL.md, or plugin.json): {dep_ref.local_path}" + msg = ( + f"Local package is not a valid APM package " + f"(no apm.yml, SKILL.md, or plugin.json): {dep_ref.local_path}" ) + if logger: + logger.error(msg) + else: + _rich_error(msg) return None # Ensure parent exists and clean target (always re-copy for local deps) diff --git a/src/apm_cli/install/phases/resolve.py b/src/apm_cli/install/phases/resolve.py index 8592a40c..7cf5eb8d 100644 --- a/src/apm_cli/install/phases/resolve.py +++ b/src/apm_cli/install/phases/resolve.py @@ -38,8 +38,6 @@ def run(ctx: "InstallContext") -> None: from apm_cli.deps.lockfile import LockFile, get_lockfile_path from apm_cli.install.phases.local_content import _copy_local_package from apm_cli.models.apm_package import DependencyReference - from apm_cli.utils.console import _rich_error - # ------------------------------------------------------------------ # 1. Lockfile loading # ------------------------------------------------------------------ @@ -139,7 +137,7 @@ def download_callback(dep_ref, modules_dir, parent_chain=""): ) return None result_path = _copy_local_package( - dep_ref, install_path, project_root + dep_ref, install_path, project_root, logger=logger ) if result_path: callback_downloaded[dep_ref.get_unique_key()] = None @@ -201,11 +199,10 @@ def download_callback(dep_ref, modules_dir, parent_chain=""): f"{dep_ref.repo_url}{chain_hint}: {e}" ) - # Verbose: inline detail + # Verbose: inline detail via logger (single output path). + # Deferred diagnostics below cover the non-logger case. if logger: logger.verbose_detail(f" {fail_msg}") - elif verbose: - _rich_error(f" |-- {fail_msg}") # Collect for deferred diagnostics summary (always, even non-verbose) callback_failures.add(dep_key) transitive_failures.append((dep_display, fail_msg)) diff --git a/src/apm_cli/install/validation.py b/src/apm_cli/install/validation.py index 7b1714fd..80d0260f 100644 --- a/src/apm_cli/install/validation.py +++ b/src/apm_cli/install/validation.py @@ -50,7 +50,7 @@ def _local_path_failure_reason(dep_ref): return "no apm.yml, SKILL.md, or plugin.json found" -def _local_path_no_markers_hint(local_dir, verbose_log=None): +def _local_path_no_markers_hint(local_dir, verbose_log=None, logger=None): """Scan two levels for sub-packages and print a hint if any are found.""" from apm_cli.utils.helpers import find_plugin_json @@ -71,21 +71,31 @@ def _local_path_no_markers_hint(local_dir, verbose_log=None): if not found: return - _rich_info(" [i] Found installable package(s) inside this directory:") - for p in found[:5]: - _rich_echo(f" apm install {p}", color="dim") - if len(found) > 5: - _rich_echo(f" ... and {len(found) - 5} more", color="dim") - - -def _validate_package_exists(package, verbose=False, auth_resolver=None): + if logger: + logger.progress(" [i] Found installable package(s) inside this directory:") + for p in found[:5]: + logger.verbose_detail(f" apm install {p}") + if len(found) > 5: + logger.verbose_detail(f" ... and {len(found) - 5} more") + else: + _rich_info(" [i] Found installable package(s) inside this directory:") + for p in found[:5]: + _rich_echo(f" apm install {p}", color="dim") + if len(found) > 5: + _rich_echo(f" ... and {len(found) - 5} more", color="dim") + + +def _validate_package_exists(package, verbose=False, auth_resolver=None, logger=None): """Validate that a package exists and is accessible on GitHub, Azure DevOps, or locally.""" import os import subprocess import tempfile from apm_cli.core.auth import AuthResolver - verbose_log = (lambda msg: _rich_echo(f" {msg}", color="dim")) if verbose else None + if logger: + verbose_log = (lambda msg: logger.verbose_detail(f" {msg}")) if verbose else None + else: + verbose_log = (lambda msg: _rich_echo(f" {msg}", color="dim")) if verbose else None # Use provided resolver or create new one if not in a CLI session context if auth_resolver is None: auth_resolver = AuthResolver() @@ -112,7 +122,7 @@ def _validate_package_exists(package, verbose=False, auth_resolver=None): if find_plugin_json(local) is not None: return True # Directory exists but lacks package markers -- surface a hint - _local_path_no_markers_hint(local, verbose_log) + _local_path_no_markers_hint(local, verbose_log, logger=logger) return False # For virtual packages, use the downloader's validation method From e3d7b58100bca768aa903bda4cb23fddb2685d40 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:14:06 +0200 Subject: [PATCH 16/28] refactor(install): F1 -- decompose integrate.py per-package paths Extract the 3 per-package code paths and root-project integration from the monolithic run() function into private module-level helpers: - _resolve_download_strategy(): preamble computing cache/download decision - _integrate_local_dep(): local filesystem package integration - _integrate_cached_dep(): cached/pre-downloaded package integration - _integrate_fresh_dep(): fresh download + integration - _integrate_root_project(): root project .apm/ primitives (#714) run() is now a slim orchestrator (~143 LOC) that iterates deps_to_install, dispatches to the correct helper, and accumulates counters. File total: 978 LOC (under 1000 standard budget). KNOWN_LARGE_MODULES exception for phases/integrate.py removed. All helpers honour the _install_mod.X test-patch contract. Mutable ctx containers are shared by reference (no local copies). Int counters use delta-dict return pattern. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/phases/integrate.py | 1579 +++++++++-------- .../install/test_architecture_invariants.py | 4 +- 2 files changed, 849 insertions(+), 734 deletions(-) diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index dada3a9d..e7f65bc3 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -20,7 +20,11 @@ ``apm_cli.commands.install.X`` is accessed via the ``_install_mod.X`` indirection rather than a bare-name import. This includes at minimum: ``_integrate_package_primitives``, ``_rich_success``, ``_rich_error``, -``_copy_local_package``, ``_pre_deploy_security_scan``. +``_copy_local_package``, ``_pre_deploy_security_scan``. All five private +helpers in this module (``_resolve_download_strategy``, +``_integrate_local_dep``, ``_integrate_cached_dep``, +``_integrate_fresh_dep``, ``_integrate_root_project``) honour this +contract via the ``_install_mod`` parameter. """ from __future__ import annotations @@ -28,12 +32,807 @@ import builtins import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple if TYPE_CHECKING: from apm_cli.install.context import InstallContext +# ====================================================================== +# Private helpers -- each encapsulates one per-package integration path +# ====================================================================== + + +def _resolve_download_strategy( + ctx: "InstallContext", + dep_ref: Any, + install_path: Path, +) -> Tuple[Any, bool, Any, bool]: + """Determine whether *dep_ref* can be served from cache. + + Returns ``(resolved_ref, skip_download, dep_locked_chk, ref_changed)`` + where *skip_download* is ``True`` when the package at *install_path* + is already up-to-date. + """ + from apm_cli.models.apm_package import GitReferenceType + from apm_cli.drift import detect_ref_change + from apm_cli.utils.path_security import safe_rmtree + + existing_lockfile = ctx.existing_lockfile + update_refs = ctx.update_refs + diagnostics = ctx.diagnostics + logger = ctx.logger + + # npm-like behavior: Branches always fetch latest, only tags/commits use cache + # Resolve git reference to determine type + resolved_ref = None + if dep_ref.get_unique_key() not in ctx.pre_downloaded_keys: + # Resolve when there is an explicit ref, OR when update_refs + # is True AND we have a non-cached lockfile entry to compare + # against (otherwise resolution is wasted work -- the package + # will be downloaded regardless). + _has_lockfile_sha = False + if update_refs and existing_lockfile: + _lck = existing_lockfile.get_dependency(dep_ref.get_unique_key()) + _has_lockfile_sha = bool( + _lck and _lck.resolved_commit and _lck.resolved_commit != "cached" + ) + if dep_ref.reference or (update_refs and _has_lockfile_sha): + try: + resolved_ref = ctx.downloader.resolve_git_reference(dep_ref) + except Exception: + pass # If resolution fails, skip cache (fetch latest) + + # Use cache only for tags and commits (not branches) + is_cacheable = resolved_ref and resolved_ref.ref_type in [ + GitReferenceType.TAG, + GitReferenceType.COMMIT, + ] + # Skip download if: already fetched by resolver callback, or cached tag/commit + already_resolved = dep_ref.get_unique_key() in ctx.callback_downloaded + # Detect if manifest ref changed vs what the lockfile recorded. + # detect_ref_change() handles all transitions including None->ref. + _dep_locked_chk = ( + existing_lockfile.get_dependency(dep_ref.get_unique_key()) + if existing_lockfile + else None + ) + ref_changed = detect_ref_change( + dep_ref, _dep_locked_chk, update_refs=update_refs + ) + # Phase 5 (#171): Also skip when lockfile SHA matches local HEAD + # -- but not when the manifest ref has changed (user wants different version). + lockfile_match = False + if install_path.exists() and existing_lockfile: + locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key()) + if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached": + if update_refs: + # Update mode: compare resolved remote SHA with lockfile SHA. + # If the remote ref still resolves to the same commit, + # the package content is unchanged -- skip download. + # Also verify local checkout matches to guard against + # corrupted installs that bypassed pre-download checks. + if resolved_ref and resolved_ref.resolved_commit == locked_dep.resolved_commit: + try: + from git import Repo as GitRepo + local_repo = GitRepo(install_path) + if local_repo.head.commit.hexsha == locked_dep.resolved_commit: + lockfile_match = True + except Exception: + pass # Local checkout invalid -- fall through to download + elif not ref_changed: + # Normal mode: compare local HEAD with lockfile SHA. + try: + from git import Repo as GitRepo + local_repo = GitRepo(install_path) + if local_repo.head.commit.hexsha == locked_dep.resolved_commit: + lockfile_match = True + except Exception: + pass # Not a git repo or invalid -- fall through to download + skip_download = install_path.exists() and ( + (is_cacheable and not update_refs) + or (already_resolved and not update_refs) + or lockfile_match + ) + + # Verify content integrity when lockfile has a hash + if skip_download and _dep_locked_chk and _dep_locked_chk.content_hash: + from apm_cli.utils.content_hash import verify_package_hash + if not verify_package_hash(install_path, _dep_locked_chk.content_hash): + _hash_msg = ( + f"Content hash mismatch for " + f"{dep_ref.get_unique_key()} -- re-downloading" + ) + diagnostics.warn(_hash_msg, package=dep_ref.get_unique_key()) + if logger: + logger.progress(_hash_msg) + safe_rmtree(install_path, ctx.apm_modules_dir) + skip_download = False + + # When registry-only mode is active, bypass cache if the + # cached artifact was NOT previously downloaded via the + # registry (no registry_prefix in lockfile). This handles + # the transition from direct-VCS installs to proxy installs + # for packages not yet in the lockfile. + if ( + skip_download + and ctx.registry_config + and ctx.registry_config.enforce_only + and not dep_ref.is_local + ): + if not _dep_locked_chk or _dep_locked_chk.registry_prefix is None: + skip_download = False + + return resolved_ref, skip_download, _dep_locked_chk, ref_changed + + +def _integrate_local_dep( + ctx: "InstallContext", + _install_mod: Any, + dep_ref: Any, + install_path: Path, + dep_key: str, +) -> Optional[Dict[str, int]]: + """Integrate a local (filesystem) package. + + Returns a counter-delta dict, or ``None`` if the dependency was + skipped (user scope, copy failure). + """ + from apm_cli.core.scope import InstallScope + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + + diagnostics = ctx.diagnostics + logger = ctx.logger + + # User scope: relative paths would resolve against $HOME + # instead of cwd, producing wrong results. Skip with a + # clear diagnostic rather than silently failing. + if ctx.scope is InstallScope.USER: + diagnostics.warn( + f"Skipped local package '{dep_ref.local_path}' " + "-- local paths are not supported at user scope (--global). " + "Use a remote reference (owner/repo) instead.", + package=dep_ref.local_path, + ) + if logger: + logger.verbose_detail( + f" Skipping {dep_ref.local_path} (local packages " + "resolve against cwd, not $HOME)" + ) + return None + + result_path = _install_mod._copy_local_package(dep_ref, install_path, ctx.project_root) + if not result_path: + diagnostics.error( + f"Failed to copy local package: {dep_ref.local_path}", + package=dep_ref.local_path, + ) + return None + + deltas: Dict[str, int] = {"installed": 1} + if logger: + logger.download_complete(dep_ref.local_path, ref_suffix="local") + + # Build minimal PackageInfo for integration + from apm_cli.models.apm_package import ( + APMPackage, + PackageInfo, + PackageType, + ResolvedReference, + GitReferenceType, + ) + from datetime import datetime + + local_apm_yml = install_path / "apm.yml" + if local_apm_yml.exists(): + local_pkg = APMPackage.from_apm_yml(local_apm_yml) + if not local_pkg.source: + local_pkg.source = dep_ref.local_path + else: + local_pkg = APMPackage( + name=Path(dep_ref.local_path).name, + version="0.0.0", + package_path=install_path, + source=dep_ref.local_path, + ) + + local_ref = ResolvedReference( + original_ref="local", + ref_type=GitReferenceType.BRANCH, + resolved_commit="local", + ref_name="local", + ) + local_info = PackageInfo( + package=local_pkg, + install_path=install_path, + resolved_reference=local_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, + ) + + # Detect package type + from apm_cli.models.validation import detect_package_type + pkg_type, plugin_json_path = detect_package_type(install_path) + local_info.package_type = pkg_type + if pkg_type == PackageType.MARKETPLACE_PLUGIN: + # Normalize: synthesize .apm/ from plugin.json so + # integration can discover and deploy primitives + from apm_cli.deps.plugin_parser import normalize_plugin_directory + normalize_plugin_directory(install_path, plugin_json_path) + + # Record for lockfile + from apm_cli.deps.installed_package import InstalledPackage + node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=None, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=None, # local deps never go through registry + )) + dep_key = dep_ref.get_unique_key() + if install_path.is_dir() and not dep_ref.is_local: + ctx.package_hashes[dep_key] = _compute_hash(install_path) + dep_deployed_files: builtins.list = [] + + if hasattr(local_info, 'package_type') and local_info.package_type: + ctx.package_types[dep_key] = local_info.package_type.value + + # Use the same variable name as the rest of the loop + package_info = local_info + + # Run shared integration pipeline + try: + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + install_path, diagnostics, + package_name=dep_key, force=ctx.force, + logger=logger, + ): + ctx.package_deployed_files[dep_key] = [] + return deltas + + int_result = _install_mod._integrate_package_primitives( + package_info, ctx.project_root, + targets=ctx.targets, + prompt_integrator=ctx.integrators["prompt"], + agent_integrator=ctx.integrators["agent"], + skill_integrator=ctx.integrators["skill"], + instruction_integrator=ctx.integrators["instruction"], + command_integrator=ctx.integrators["command"], + hook_integrator=ctx.integrators["hook"], + force=ctx.force, + managed_files=ctx.managed_files, + diagnostics=diagnostics, + package_name=dep_key, + logger=logger, + scope=ctx.scope, + ) + deltas["prompts"] = int_result["prompts"] + deltas["agents"] = int_result["agents"] + deltas["skills"] = int_result["skills"] + deltas["sub_skills"] = int_result["sub_skills"] + deltas["instructions"] = int_result["instructions"] + deltas["commands"] = int_result["commands"] + deltas["hooks"] = int_result["hooks"] + deltas["links_resolved"] = int_result["links_resolved"] + dep_deployed_files.extend(int_result["deployed_files"]) + except Exception as e: + diagnostics.error( + f"Failed to integrate primitives from local package: {e}", + package=dep_ref.local_path, + ) + + ctx.package_deployed_files[dep_key] = dep_deployed_files + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + _skip_count = diagnostics.count_for_package(dep_key, "collision") + _err_count = diagnostics.count_for_package(dep_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + + return deltas + + +def _integrate_cached_dep( + ctx: "InstallContext", + _install_mod: Any, + dep_ref: Any, + install_path: Path, + dep_key: str, + resolved_ref: Any, + dep_locked_chk: Any, +) -> Optional[Dict[str, int]]: + """Integrate a cached (already-downloaded) package. + + Returns a counter-delta dict. + """ + from apm_cli.constants import APM_YML_FILENAME + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + + logger = ctx.logger + diagnostics = ctx.diagnostics + + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + # Show resolved ref from lockfile for consistency with fresh installs + _ref = dep_ref.reference or "" + _sha = "" + if dep_locked_chk and dep_locked_chk.resolved_commit and dep_locked_chk.resolved_commit != "cached": + _sha = dep_locked_chk.resolved_commit[:8] + if logger: + logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) + + deltas: Dict[str, int] = {"installed": 1} + if not dep_ref.reference: + deltas["unpinned"] = 1 + + # Skip integration if not needed + if not ctx.targets: + return deltas + + # Integrate prompts for cached packages (zero-config behavior) + try: + # Create PackageInfo from cached package + from apm_cli.models.apm_package import ( + APMPackage, + PackageInfo, + PackageType, + ResolvedReference, + GitReferenceType, + ) + from datetime import datetime + + # Load package from apm.yml in install path + apm_yml_path = install_path / APM_YML_FILENAME + if apm_yml_path.exists(): + cached_package = APMPackage.from_apm_yml(apm_yml_path) + # Ensure source is set to the repo URL for sync matching + if not cached_package.source: + cached_package.source = dep_ref.repo_url + else: + # Virtual package or no apm.yml - create minimal package + cached_package = APMPackage( + name=dep_ref.repo_url.split("/")[-1], + version="unknown", + package_path=install_path, + source=dep_ref.repo_url, + ) + + # Use resolved reference from ref resolution if available + # (e.g. when update_refs matched the lockfile SHA), + # otherwise create a placeholder for cached packages. + resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( + original_ref=dep_ref.reference or "default", + ref_type=GitReferenceType.BRANCH, + resolved_commit="cached", # Mark as cached since we don't know exact commit + ref_name=dep_ref.reference or "default", + ) + + cached_package_info = PackageInfo( + package=cached_package, + install_path=install_path, + resolved_reference=resolved_or_cached_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, # Store for canonical dependency string + ) + + # Detect package_type from disk contents so + # skill integration is not silently skipped + from apm_cli.models.validation import detect_package_type + pkg_type, _ = detect_package_type(install_path) + cached_package_info.package_type = pkg_type + + # Collect for lockfile (cached packages still need to be tracked) + from apm_cli.deps.installed_package import InstalledPackage + node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + # Get commit SHA: resolved ref > callback capture > existing lockfile > explicit reference + dep_key = dep_ref.get_unique_key() + cached_commit = None + if resolved_ref and resolved_ref.resolved_commit and resolved_ref.resolved_commit != "cached": + cached_commit = resolved_ref.resolved_commit + if not cached_commit: + cached_commit = ctx.callback_downloaded.get(dep_key) + if not cached_commit and ctx.existing_lockfile: + locked_dep = ctx.existing_lockfile.get_dependency(dep_key) + if locked_dep: + cached_commit = locked_dep.resolved_commit + if not cached_commit: + cached_commit = dep_ref.reference + # Determine if the cached package came from the registry: + # prefer the lockfile record, then the current registry config. + _cached_registry = None + if dep_locked_chk and dep_locked_chk.registry_prefix: + # Reconstruct RegistryConfig from lockfile to preserve original source + _cached_registry = ctx.registry_config + elif ctx.registry_config and not dep_ref.is_local: + _cached_registry = ctx.registry_config + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=cached_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=_cached_registry, + )) + if install_path.is_dir(): + ctx.package_hashes[dep_key] = _compute_hash(install_path) + # Track package type for lockfile + if hasattr(cached_package_info, 'package_type') and cached_package_info.package_type: + ctx.package_types[dep_key] = cached_package_info.package_type.value + + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + install_path, diagnostics, + package_name=dep_key, force=ctx.force, + logger=logger, + ): + ctx.package_deployed_files[dep_key] = [] + return deltas + + int_result = _install_mod._integrate_package_primitives( + cached_package_info, ctx.project_root, + targets=ctx.targets, + prompt_integrator=ctx.integrators["prompt"], + agent_integrator=ctx.integrators["agent"], + skill_integrator=ctx.integrators["skill"], + instruction_integrator=ctx.integrators["instruction"], + command_integrator=ctx.integrators["command"], + hook_integrator=ctx.integrators["hook"], + force=ctx.force, + managed_files=ctx.managed_files, + diagnostics=diagnostics, + package_name=dep_key, + logger=logger, + scope=ctx.scope, + ) + deltas["prompts"] = int_result["prompts"] + deltas["agents"] = int_result["agents"] + deltas["skills"] = int_result["skills"] + deltas["sub_skills"] = int_result["sub_skills"] + deltas["instructions"] = int_result["instructions"] + deltas["commands"] = int_result["commands"] + deltas["hooks"] = int_result["hooks"] + deltas["links_resolved"] = int_result["links_resolved"] + dep_deployed = int_result["deployed_files"] + ctx.package_deployed_files[dep_key] = dep_deployed + except Exception as e: + diagnostics.error( + f"Failed to integrate primitives from cached package: {e}", + package=dep_key, + ) + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + _skip_count = diagnostics.count_for_package(dep_key, "collision") + _err_count = diagnostics.count_for_package(dep_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + + return deltas + + +def _integrate_fresh_dep( + ctx: "InstallContext", + _install_mod: Any, + dep_ref: Any, + install_path: Path, + dep_key: str, + resolved_ref: Any, + dep_locked_chk: Any, + ref_changed: bool, + progress: Any, +) -> Optional[Dict[str, int]]: + """Download and integrate a fresh (not cached) package. + + Returns a counter-delta dict, or ``None`` if the download failed. + """ + from apm_cli.drift import build_download_ref + from apm_cli.deps.installed_package import InstalledPackage + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + from apm_cli.utils.path_security import safe_rmtree + + diagnostics = ctx.diagnostics + logger = ctx.logger + + # Download the package with progress feedback + try: + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + short_name = ( + display_name.split("/")[-1] + if "/" in display_name + else display_name + ) + + # Create a progress task for this download + task_id = progress.add_task( + description=f"Fetching {short_name}", + total=None, # Indeterminate initially; git will update with actual counts + ) + + # T5: Build download ref - use locked commit if available. + # build_download_ref() uses manifest ref when ref_changed is True. + download_ref = build_download_ref( + dep_ref, ctx.existing_lockfile, update_refs=ctx.update_refs, ref_changed=ref_changed + ) + + # Phase 4 (#171): Use pre-downloaded result if available + _dep_key = dep_ref.get_unique_key() + if _dep_key in ctx.pre_download_results: + package_info = ctx.pre_download_results[_dep_key] + else: + # Fallback: sequential download (should rarely happen) + package_info = ctx.downloader.download_package( + download_ref, + install_path, + progress_task_id=task_id, + progress_obj=progress, + ) + + # CRITICAL: Hide progress BEFORE printing success message to avoid overlap + progress.update(task_id, visible=False) + progress.refresh() # Force immediate refresh to hide the bar + + deltas: Dict[str, int] = {"installed": 1} + + # Show resolved ref alongside package name for visibility + resolved = getattr(package_info, 'resolved_reference', None) + if logger: + _ref = "" + _sha = "" + if resolved: + _ref = resolved.ref_name if resolved.ref_name else "" + _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + logger.download_complete(display_name, ref=_ref, sha=_sha) + # Log auth source for this download (verbose only) + if ctx.auth_resolver: + try: + _host = dep_ref.host or "github.com" + _org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None + _ctx = ctx.auth_resolver.resolve(_host, org=_org) + logger.package_auth(_ctx.source, _ctx.token_type or "none") + except Exception: + pass + else: + _ref_suffix = "" + if resolved: + _r = resolved.ref_name if resolved.ref_name else "" + _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + if _r and _s: + _ref_suffix = f" #{_r} @{_s}" + elif _r: + _ref_suffix = f" #{_r}" + elif _s: + _ref_suffix = f" @{_s}" + _install_mod._rich_success(f"[+] {display_name}{_ref_suffix}") + + # Track unpinned deps for aggregated diagnostic + if not dep_ref.reference: + deltas["unpinned"] = 1 + + # Collect for lockfile: get resolved commit and depth + resolved_commit = None + if resolved: + resolved_commit = package_info.resolved_reference.resolved_commit + # Get depth from dependency tree + node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=resolved_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=ctx.registry_config if not dep_ref.is_local else None, + )) + if install_path.is_dir(): + ctx.package_hashes[dep_ref.get_unique_key()] = _compute_hash(install_path) + + # Supply chain protection: verify content hash on fresh + # downloads when the lockfile already records a hash. + # A mismatch means the downloaded content differs from + # what was previously locked -- possible tampering. + if ( + not ctx.update_refs + and dep_locked_chk + and dep_locked_chk.content_hash + and dep_ref.get_unique_key() in ctx.package_hashes + ): + _fresh_hash = ctx.package_hashes[dep_ref.get_unique_key()] + if _fresh_hash != dep_locked_chk.content_hash: + safe_rmtree(install_path, ctx.apm_modules_dir) + _install_mod._rich_error( + f"Content hash mismatch for " + f"{dep_ref.get_unique_key()}: " + f"expected {dep_locked_chk.content_hash}, " + f"got {_fresh_hash}. " + "The downloaded content differs from the " + "lockfile record. This may indicate a " + "supply-chain attack. Use 'apm install " + "--update' to accept new content and " + "update the lockfile." + ) + sys.exit(1) + + # Track package type for lockfile + if hasattr(package_info, 'package_type') and package_info.package_type: + ctx.package_types[dep_ref.get_unique_key()] = package_info.package_type.value + + # Show package type in verbose mode + if hasattr(package_info, "package_type"): + from apm_cli.models.apm_package import PackageType + + package_type = package_info.package_type + _type_label = { + PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", + PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", + PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", + PackageType.APM_PACKAGE: "APM Package (apm.yml)", + }.get(package_type) + if _type_label and logger: + logger.package_type_info(_type_label) + + # Auto-integrate prompts and agents if enabled + # Pre-deploy security gate + if not _install_mod._pre_deploy_security_scan( + package_info.install_path, diagnostics, + package_name=dep_ref.get_unique_key(), force=ctx.force, + logger=logger, + ): + ctx.package_deployed_files[dep_ref.get_unique_key()] = [] + return deltas + + if ctx.targets: + try: + int_result = _install_mod._integrate_package_primitives( + package_info, ctx.project_root, + targets=ctx.targets, + prompt_integrator=ctx.integrators["prompt"], + agent_integrator=ctx.integrators["agent"], + skill_integrator=ctx.integrators["skill"], + instruction_integrator=ctx.integrators["instruction"], + command_integrator=ctx.integrators["command"], + hook_integrator=ctx.integrators["hook"], + force=ctx.force, + managed_files=ctx.managed_files, + diagnostics=diagnostics, + package_name=dep_ref.get_unique_key(), + logger=logger, + scope=ctx.scope, + ) + deltas["prompts"] = int_result["prompts"] + deltas["agents"] = int_result["agents"] + deltas["skills"] = int_result["skills"] + deltas["sub_skills"] = int_result["sub_skills"] + deltas["instructions"] = int_result["instructions"] + deltas["commands"] = int_result["commands"] + deltas["hooks"] = int_result["hooks"] + deltas["links_resolved"] = int_result["links_resolved"] + dep_deployed_fresh = int_result["deployed_files"] + ctx.package_deployed_files[dep_ref.get_unique_key()] = dep_deployed_fresh + except Exception as e: + # Don't fail installation if integration fails + diagnostics.error( + f"Failed to integrate primitives: {e}", + package=dep_ref.get_unique_key(), + ) + + # In verbose mode, show inline skip/error count for this package + if logger and logger.verbose: + pkg_key = dep_ref.get_unique_key() + _skip_count = diagnostics.count_for_package(pkg_key, "collision") + _err_count = diagnostics.count_for_package(pkg_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning(f" [!] {_err_count} integration {noun}") + + return deltas + + except Exception as e: + display_name = ( + str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + ) + # Remove the progress task on error + if "task_id" in locals(): + progress.remove_task(task_id) + diagnostics.error( + f"Failed to install {display_name}: {e}", + package=dep_ref.get_unique_key(), + ) + # Continue with other packages instead of failing completely + return None + + +def _integrate_root_project( + ctx: "InstallContext", + _install_mod: Any, +) -> Optional[Dict[str, int]]: + """Integrate root project's own .apm/ primitives (#714). + + Users should not need a dummy "./agent/apm.yml" stub to get their + root-level .apm/ rules deployed alongside external dependencies. + Treat the project root as an implicit local package: any primitives + found in /.apm/ are integrated after all declared + dependency packages have been processed. + + Returns a counter-delta dict, or ``None`` if root integration is + not applicable or failed. + """ + if not ctx.root_has_local_primitives or not ctx.targets: + return None + + logger = ctx.logger + diagnostics = ctx.diagnostics + + from apm_cli.models.apm_package import PackageInfo as _PackageInfo + _root_pkg_info = _PackageInfo( + package=ctx.apm_package, + install_path=ctx.project_root, + ) + if logger: + logger.download_complete("", ref_suffix="local") + try: + _root_result = _install_mod._integrate_package_primitives( + _root_pkg_info, ctx.project_root, + targets=ctx.targets, + prompt_integrator=ctx.integrators["prompt"], + agent_integrator=ctx.integrators["agent"], + skill_integrator=ctx.integrators["skill"], + instruction_integrator=ctx.integrators["instruction"], + command_integrator=ctx.integrators["command"], + hook_integrator=ctx.integrators["hook"], + force=ctx.force, + managed_files=ctx.managed_files, + diagnostics=diagnostics, + package_name="", + logger=logger, + scope=ctx.scope, + ) + return { + "installed": 1, + "prompts": _root_result["prompts"], + "agents": _root_result["agents"], + "instructions": _root_result["instructions"], + "commands": _root_result["commands"], + "hooks": _root_result["hooks"], + "links_resolved": _root_result["links_resolved"], + } + except Exception as e: + import traceback as _tb + diagnostics.error( + f"Failed to integrate root project primitives: {e}", + package="", + detail=_tb.format_exc(), + ) + # When root integration is the *only* action (no external deps), + # a failure means nothing was deployed -- surface it clearly. + if not ctx.all_apm_deps and logger: + logger.error( + f"Root project primitives could not be integrated: {e}" + ) + return None + + +# ====================================================================== +# Public phase entry point +# ====================================================================== + + def run(ctx: "InstallContext") -> None: """Execute the sequential integration phase. @@ -53,15 +852,6 @@ def run(ctx: "InstallContext") -> None: # ------------------------------------------------------------------ from apm_cli.commands import install as _install_mod - # ------------------------------------------------------------------ - # Direct imports for names NOT patched at apm_cli.commands.install.X - # ------------------------------------------------------------------ - from apm_cli.constants import APM_YML_FILENAME - from apm_cli.core.scope import InstallScope - from apm_cli.deps.installed_package import InstalledPackage - from apm_cli.drift import build_download_ref, detect_ref_change - from apm_cli.utils.content_hash import compute_package_hash as _compute_hash - from apm_cli.utils.path_security import safe_rmtree from rich.progress import ( BarColumn, Progress, @@ -71,46 +861,13 @@ def run(ctx: "InstallContext") -> None: ) # ------------------------------------------------------------------ - # Unpack ctx into local aliases. Mutable containers (lists, dicts, - # sets) share the reference so in-place mutations are visible through - # ctx. Int counters are accumulated into locals and written back at - # the end of this function. + # Unpack loop-level aliases and int counters. + # Mutable containers (lists, dicts, sets) share the reference so + # in-place mutations by helpers are visible through ctx. Int + # counters are accumulated into locals and written back at the end. # ------------------------------------------------------------------ deps_to_install = ctx.deps_to_install apm_modules_dir = ctx.apm_modules_dir - callback_failures = ctx.callback_failures - callback_downloaded = ctx.callback_downloaded - scope = ctx.scope - diagnostics = ctx.diagnostics - logger = ctx.logger - project_root = ctx.project_root - dependency_graph = ctx.dependency_graph - existing_lockfile = ctx.existing_lockfile - update_refs = ctx.update_refs - downloader = ctx.downloader - force = ctx.force - apm_package = ctx.apm_package - all_apm_deps = ctx.all_apm_deps - registry_config = ctx.registry_config - _targets = ctx.targets - _pre_download_results = ctx.pre_download_results - _pre_downloaded_keys = ctx.pre_downloaded_keys - _root_has_local_primitives = ctx.root_has_local_primitives - - # Mutable containers (shared references -- mutations visible via ctx) - installed_packages = ctx.installed_packages - package_deployed_files = ctx.package_deployed_files - package_types = ctx.package_types - _package_hashes = ctx.package_hashes - managed_files = ctx.managed_files - - # Integrators - prompt_integrator = ctx.integrators["prompt"] - agent_integrator = ctx.integrators["agent"] - skill_integrator = ctx.integrators["skill"] - instruction_integrator = ctx.integrators["instruction"] - command_integrator = ctx.integrators["command"] - hook_integrator = ctx.integrators["hook"] # Int counters (written back to ctx at end of function) installed_count = ctx.installed_count @@ -125,15 +882,9 @@ def run(ctx: "InstallContext") -> None: total_links_resolved = ctx.total_links_resolved # ------------------------------------------------------------------ - # Begin extracted region (install.py lines 1290-2004, verbatim - # except for free-variable replacement and indentation adjustment) + # Main loop: iterate deps_to_install and dispatch to the appropriate + # per-package helper based on package source. # ------------------------------------------------------------------ - - # Create progress display for sequential integration - # Reuse the shared auth_resolver (already created in this invocation) so - # verbose auth logging does not trigger a duplicate credential-helper popup. - _auth_resolver = ctx.auth_resolver - with Progress( SpinnerColumn(), TextColumn("[cyan]{task.description}[/cyan]"), @@ -148,8 +899,7 @@ def run(ctx: "InstallContext") -> None: # For subdirectory packages: owner/repo/subdir -> apm_modules/owner/repo/subdir/ if dep_ref.alias: # If alias is provided, use it directly (assume user handles namespacing) - install_name = dep_ref.alias - install_path = apm_modules_dir / install_name + install_path = apm_modules_dir / dep_ref.alias else: # Use the canonical install path from DependencyReference install_path = dep_ref.get_install_path(apm_modules_dir) @@ -157,693 +907,60 @@ def run(ctx: "InstallContext") -> None: # Skip deps that already failed during BFS resolution callback # to avoid a duplicate error entry in diagnostics. dep_key = dep_ref.get_unique_key() - if dep_key in callback_failures: - if logger: - logger.verbose_detail(f" Skipping {dep_key} (already failed during resolution)") + if dep_key in ctx.callback_failures: + if ctx.logger: + ctx.logger.verbose_detail(f" Skipping {dep_key} (already failed during resolution)") continue - # --- Local package: copy from filesystem (no git download) --- + # --- Dispatch to per-source helper --- if dep_ref.is_local and dep_ref.local_path: - # User scope: relative paths would resolve against $HOME - # instead of cwd, producing wrong results. Skip with a - # clear diagnostic rather than silently failing. - if scope is InstallScope.USER: - diagnostics.warn( - f"Skipped local package '{dep_ref.local_path}' " - "-- local paths are not supported at user scope (--global). " - "Use a remote reference (owner/repo) instead.", - package=dep_ref.local_path, - ) - if logger: - logger.verbose_detail( - f" Skipping {dep_ref.local_path} (local packages " - "resolve against cwd, not $HOME)" - ) - continue - - result_path = _install_mod._copy_local_package(dep_ref, install_path, project_root, logger=logger) - if not result_path: - diagnostics.error( - f"Failed to copy local package: {dep_ref.local_path}", - package=dep_ref.local_path, - ) - continue - - installed_count += 1 - if logger: - logger.download_complete(dep_ref.local_path, ref_suffix="local") - - # Build minimal PackageInfo for integration - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, + deltas = _integrate_local_dep( + ctx, _install_mod, dep_ref, install_path, dep_key, ) - from datetime import datetime - - local_apm_yml = install_path / "apm.yml" - if local_apm_yml.exists(): - local_pkg = APMPackage.from_apm_yml(local_apm_yml) - if not local_pkg.source: - local_pkg.source = dep_ref.local_path - else: - local_pkg = APMPackage( - name=Path(dep_ref.local_path).name, - version="0.0.0", - package_path=install_path, - source=dep_ref.local_path, - ) - - local_ref = ResolvedReference( - original_ref="local", - ref_type=GitReferenceType.BRANCH, - resolved_commit="local", - ref_name="local", - ) - local_info = PackageInfo( - package=local_pkg, - install_path=install_path, - resolved_reference=local_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, - ) - - # Detect package type - from apm_cli.models.validation import detect_package_type - pkg_type, plugin_json_path = detect_package_type(install_path) - local_info.package_type = pkg_type - if pkg_type == PackageType.MARKETPLACE_PLUGIN: - # Normalize: synthesize .apm/ from plugin.json so - # integration can discover and deploy primitives - from apm_cli.deps.plugin_parser import normalize_plugin_directory - normalize_plugin_directory(install_path, plugin_json_path) - - # Record for lockfile - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=None, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=None, # local deps never go through registry - )) - dep_key = dep_ref.get_unique_key() - if install_path.is_dir() and not dep_ref.is_local: - _package_hashes[dep_key] = _compute_hash(install_path) - dep_deployed_files: builtins.list = [] - - if hasattr(local_info, 'package_type') and local_info.package_type: - package_types[dep_key] = local_info.package_type.value - - # Use the same variable name as the rest of the loop - package_info = local_info - - # Run shared integration pipeline - try: - # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=force, - logger=logger, - ): - package_deployed_files[dep_key] = [] - continue - - int_result = _install_mod._integrate_package_primitives( - package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed_files.extend(int_result["deployed_files"]) - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from local package: {e}", - package=dep_ref.local_path, - ) - - package_deployed_files[dep_key] = dep_deployed_files - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - continue - - # npm-like behavior: Branches always fetch latest, only tags/commits use cache - # Resolve git reference to determine type - from apm_cli.models.apm_package import GitReferenceType - - resolved_ref = None - if dep_ref.get_unique_key() not in _pre_downloaded_keys: - # Resolve when there is an explicit ref, OR when update_refs - # is True AND we have a non-cached lockfile entry to compare - # against (otherwise resolution is wasted work -- the package - # will be downloaded regardless). - _has_lockfile_sha = False - if update_refs and existing_lockfile: - _lck = existing_lockfile.get_dependency(dep_ref.get_unique_key()) - _has_lockfile_sha = bool( - _lck and _lck.resolved_commit and _lck.resolved_commit != "cached" - ) - if dep_ref.reference or (update_refs and _has_lockfile_sha): - try: - resolved_ref = downloader.resolve_git_reference(dep_ref) - except Exception: - pass # If resolution fails, skip cache (fetch latest) - - # Use cache only for tags and commits (not branches) - is_cacheable = resolved_ref and resolved_ref.ref_type in [ - GitReferenceType.TAG, - GitReferenceType.COMMIT, - ] - # Skip download if: already fetched by resolver callback, or cached tag/commit - already_resolved = dep_ref.get_unique_key() in callback_downloaded - # Detect if manifest ref changed vs what the lockfile recorded. - # detect_ref_change() handles all transitions including None->ref. - _dep_locked_chk = ( - existing_lockfile.get_dependency(dep_ref.get_unique_key()) - if existing_lockfile - else None - ) - ref_changed = detect_ref_change( - dep_ref, _dep_locked_chk, update_refs=update_refs - ) - # Phase 5 (#171): Also skip when lockfile SHA matches local HEAD - # -- but not when the manifest ref has changed (user wants different version). - lockfile_match = False - if install_path.exists() and existing_lockfile: - locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key()) - if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached": - if update_refs: - # Update mode: compare resolved remote SHA with lockfile SHA. - # If the remote ref still resolves to the same commit, - # the package content is unchanged -- skip download. - # Also verify local checkout matches to guard against - # corrupted installs that bypassed pre-download checks. - if resolved_ref and resolved_ref.resolved_commit == locked_dep.resolved_commit: - try: - from git import Repo as GitRepo - local_repo = GitRepo(install_path) - if local_repo.head.commit.hexsha == locked_dep.resolved_commit: - lockfile_match = True - except Exception: - pass # Local checkout invalid -- fall through to download - elif not ref_changed: - # Normal mode: compare local HEAD with lockfile SHA. - try: - from git import Repo as GitRepo - local_repo = GitRepo(install_path) - if local_repo.head.commit.hexsha == locked_dep.resolved_commit: - lockfile_match = True - except Exception: - pass # Not a git repo or invalid -- fall through to download - skip_download = install_path.exists() and ( - (is_cacheable and not update_refs) - or (already_resolved and not update_refs) - or lockfile_match - ) - - # Verify content integrity when lockfile has a hash - if skip_download and _dep_locked_chk and _dep_locked_chk.content_hash: - from apm_cli.utils.content_hash import verify_package_hash - if not verify_package_hash(install_path, _dep_locked_chk.content_hash): - _hash_msg = ( - f"Content hash mismatch for " - f"{dep_ref.get_unique_key()} -- re-downloading" - ) - diagnostics.warn(_hash_msg, package=dep_ref.get_unique_key()) - if logger: - logger.progress(_hash_msg) - safe_rmtree(install_path, apm_modules_dir) - skip_download = False - - # When registry-only mode is active, bypass cache if the - # cached artifact was NOT previously downloaded via the - # registry (no registry_prefix in lockfile). This handles - # the transition from direct-VCS installs to proxy installs - # for packages not yet in the lockfile. - if ( - skip_download - and registry_config - and registry_config.enforce_only - and not dep_ref.is_local - ): - if not _dep_locked_chk or _dep_locked_chk.registry_prefix is None: - skip_download = False - - if skip_download: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + else: + resolved_ref, skip_download, dep_locked_chk, ref_changed = ( + _resolve_download_strategy(ctx, dep_ref, install_path) ) - # Show resolved ref from lockfile for consistency with fresh installs - _ref = dep_ref.reference or "" - _sha = "" - if _dep_locked_chk and _dep_locked_chk.resolved_commit and _dep_locked_chk.resolved_commit != "cached": - _sha = _dep_locked_chk.resolved_commit[:8] - if logger: - logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) - installed_count += 1 - if not dep_ref.reference: - unpinned_count += 1 - - # Skip integration if not needed - if not _targets: - continue - - # Integrate prompts for cached packages (zero-config behavior) - try: - # Create PackageInfo from cached package - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, - ) - from datetime import datetime - - # Load package from apm.yml in install path - apm_yml_path = install_path / APM_YML_FILENAME - if apm_yml_path.exists(): - cached_package = APMPackage.from_apm_yml(apm_yml_path) - # Ensure source is set to the repo URL for sync matching - if not cached_package.source: - cached_package.source = dep_ref.repo_url - else: - # Virtual package or no apm.yml - create minimal package - cached_package = APMPackage( - name=dep_ref.repo_url.split("/")[-1], - version="unknown", - package_path=install_path, - source=dep_ref.repo_url, - ) - - # Use resolved reference from ref resolution if available - # (e.g. when update_refs matched the lockfile SHA), - # otherwise create a placeholder for cached packages. - resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( - original_ref=dep_ref.reference or "default", - ref_type=GitReferenceType.BRANCH, - resolved_commit="cached", # Mark as cached since we don't know exact commit - ref_name=dep_ref.reference or "default", + if skip_download: + deltas = _integrate_cached_dep( + ctx, _install_mod, dep_ref, install_path, dep_key, + resolved_ref, dep_locked_chk, ) - - cached_package_info = PackageInfo( - package=cached_package, - install_path=install_path, - resolved_reference=resolved_or_cached_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, # Store for canonical dependency string - ) - - # Detect package_type from disk contents so - # skill integration is not silently skipped - from apm_cli.models.validation import detect_package_type - pkg_type, _ = detect_package_type(install_path) - cached_package_info.package_type = pkg_type - - # Collect for lockfile (cached packages still need to be tracked) - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - # Get commit SHA: resolved ref > callback capture > existing lockfile > explicit reference - dep_key = dep_ref.get_unique_key() - cached_commit = None - if resolved_ref and resolved_ref.resolved_commit and resolved_ref.resolved_commit != "cached": - cached_commit = resolved_ref.resolved_commit - if not cached_commit: - cached_commit = callback_downloaded.get(dep_key) - if not cached_commit and existing_lockfile: - locked_dep = existing_lockfile.get_dependency(dep_key) - if locked_dep: - cached_commit = locked_dep.resolved_commit - if not cached_commit: - cached_commit = dep_ref.reference - # Determine if the cached package came from the registry: - # prefer the lockfile record, then the current registry config. - _cached_registry = None - if _dep_locked_chk and _dep_locked_chk.registry_prefix: - # Reconstruct RegistryConfig from lockfile to preserve original source - _cached_registry = registry_config - elif registry_config and not dep_ref.is_local: - _cached_registry = registry_config - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=cached_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=_cached_registry, - )) - if install_path.is_dir(): - _package_hashes[dep_key] = _compute_hash(install_path) - # Track package type for lockfile - if hasattr(cached_package_info, 'package_type') and cached_package_info.package_type: - package_types[dep_key] = cached_package_info.package_type.value - - # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=force, - logger=logger, - ): - package_deployed_files[dep_key] = [] - continue - - int_result = _install_mod._integrate_package_primitives( - cached_package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed = int_result["deployed_files"] - package_deployed_files[dep_key] = dep_deployed - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from cached package: {e}", - package=dep_key, - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - continue - - # Download the package with progress feedback - try: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - short_name = ( - display_name.split("/")[-1] - if "/" in display_name - else display_name - ) - - # Create a progress task for this download - task_id = progress.add_task( - description=f"Fetching {short_name}", - total=None, # Indeterminate initially; git will update with actual counts - ) - - # T5: Build download ref - use locked commit if available. - # build_download_ref() uses manifest ref when ref_changed is True. - download_ref = build_download_ref( - dep_ref, existing_lockfile, update_refs=update_refs, ref_changed=ref_changed - ) - - # Phase 4 (#171): Use pre-downloaded result if available - _dep_key = dep_ref.get_unique_key() - if _dep_key in _pre_download_results: - package_info = _pre_download_results[_dep_key] else: - # Fallback: sequential download (should rarely happen) - package_info = downloader.download_package( - download_ref, - install_path, - progress_task_id=task_id, - progress_obj=progress, + deltas = _integrate_fresh_dep( + ctx, _install_mod, dep_ref, install_path, dep_key, + resolved_ref, dep_locked_chk, ref_changed, progress, ) - # CRITICAL: Hide progress BEFORE printing success message to avoid overlap - progress.update(task_id, visible=False) - progress.refresh() # Force immediate refresh to hide the bar - - installed_count += 1 - - # Show resolved ref alongside package name for visibility - resolved = getattr(package_info, 'resolved_reference', None) - if logger: - _ref = "" - _sha = "" - if resolved: - _ref = resolved.ref_name if resolved.ref_name else "" - _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - logger.download_complete(display_name, ref=_ref, sha=_sha) - # Log auth source for this download (verbose only) - if _auth_resolver: - try: - _host = dep_ref.host or "github.com" - _org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None - _ctx = _auth_resolver.resolve(_host, org=_org) - logger.package_auth(_ctx.source, _ctx.token_type or "none") - except Exception: - pass - else: - _ref_suffix = "" - if resolved: - _r = resolved.ref_name if resolved.ref_name else "" - _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - if _r and _s: - _ref_suffix = f" #{_r} @{_s}" - elif _r: - _ref_suffix = f" #{_r}" - elif _s: - _ref_suffix = f" @{_s}" - _install_mod._rich_success(f"[+] {display_name}{_ref_suffix}") - - # Track unpinned deps for aggregated diagnostic - if not dep_ref.reference: - unpinned_count += 1 - - # Collect for lockfile: get resolved commit and depth - resolved_commit = None - if resolved: - resolved_commit = package_info.resolved_reference.resolved_commit - # Get depth from dependency tree - node = dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=resolved_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=registry_config if not dep_ref.is_local else None, - )) - if install_path.is_dir(): - _package_hashes[dep_ref.get_unique_key()] = _compute_hash(install_path) - - # Supply chain protection: verify content hash on fresh - # downloads when the lockfile already records a hash. - # A mismatch means the downloaded content differs from - # what was previously locked -- possible tampering. - if ( - not update_refs - and _dep_locked_chk - and _dep_locked_chk.content_hash - and dep_ref.get_unique_key() in _package_hashes - ): - _fresh_hash = _package_hashes[dep_ref.get_unique_key()] - if _fresh_hash != _dep_locked_chk.content_hash: - safe_rmtree(install_path, apm_modules_dir) - _install_mod._rich_error( - f"Content hash mismatch for " - f"{dep_ref.get_unique_key()}: " - f"expected {_dep_locked_chk.content_hash}, " - f"got {_fresh_hash}. " - "The downloaded content differs from the " - "lockfile record. This may indicate a " - "supply-chain attack. Use 'apm install " - "--update' to accept new content and " - "update the lockfile." - ) - sys.exit(1) - - # Track package type for lockfile - if hasattr(package_info, 'package_type') and package_info.package_type: - package_types[dep_ref.get_unique_key()] = package_info.package_type.value - - # Show package type in verbose mode - if hasattr(package_info, "package_type"): - from apm_cli.models.apm_package import PackageType - - package_type = package_info.package_type - _type_label = { - PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", - PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", - PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", - PackageType.APM_PACKAGE: "APM Package (apm.yml)", - }.get(package_type) - if _type_label and logger: - logger.package_type_info(_type_label) - - # Auto-integrate prompts and agents if enabled - # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( - package_info.install_path, diagnostics, - package_name=dep_ref.get_unique_key(), force=force, - logger=logger, - ): - package_deployed_files[dep_ref.get_unique_key()] = [] - continue - - if _targets: - try: - int_result = _install_mod._integrate_package_primitives( - package_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name=dep_ref.get_unique_key(), - logger=logger, - scope=scope, - ) - total_prompts_integrated += int_result["prompts"] - total_agents_integrated += int_result["agents"] - total_skills_integrated += int_result["skills"] - total_sub_skills_promoted += int_result["sub_skills"] - total_instructions_integrated += int_result["instructions"] - total_commands_integrated += int_result["commands"] - total_hooks_integrated += int_result["hooks"] - total_links_resolved += int_result["links_resolved"] - dep_deployed_fresh = int_result["deployed_files"] - package_deployed_files[dep_ref.get_unique_key()] = dep_deployed_fresh - except Exception as e: - # Don't fail installation if integration fails - diagnostics.error( - f"Failed to integrate primitives: {e}", - package=dep_ref.get_unique_key(), - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - pkg_key = dep_ref.get_unique_key() - _skip_count = diagnostics.count_for_package(pkg_key, "collision") - _err_count = diagnostics.count_for_package(pkg_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - except Exception as e: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - # Remove the progress task on error - if "task_id" in locals(): - progress.remove_task(task_id) - diagnostics.error( - f"Failed to install {display_name}: {e}", - package=dep_ref.get_unique_key(), - ) - # Continue with other packages instead of failing completely + if deltas is None: continue + # Accumulate counter deltas from this package + installed_count += deltas.get("installed", 0) + unpinned_count += deltas.get("unpinned", 0) + total_prompts_integrated += deltas.get("prompts", 0) + total_agents_integrated += deltas.get("agents", 0) + total_skills_integrated += deltas.get("skills", 0) + total_sub_skills_promoted += deltas.get("sub_skills", 0) + total_instructions_integrated += deltas.get("instructions", 0) + total_commands_integrated += deltas.get("commands", 0) + total_hooks_integrated += deltas.get("hooks", 0) + total_links_resolved += deltas.get("links_resolved", 0) + # ------------------------------------------------------------------ # Integrate root project's own .apm/ primitives (#714). - # - # Users should not need a dummy "./agent/apm.yml" stub to get their - # root-level .apm/ rules deployed alongside external dependencies. - # Treat the project root as an implicit local package: any primitives - # found in /.apm/ are integrated after all declared - # dependency packages have been processed. # ------------------------------------------------------------------ - if _root_has_local_primitives and _targets: - from apm_cli.models.apm_package import PackageInfo as _PackageInfo - _root_pkg_info = _PackageInfo( - package=apm_package, - install_path=project_root, - ) - if logger: - logger.download_complete("", ref_suffix="local") - try: - _root_result = _install_mod._integrate_package_primitives( - _root_pkg_info, project_root, - targets=_targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name="", - logger=logger, - scope=scope, - ) - total_prompts_integrated += _root_result["prompts"] - total_agents_integrated += _root_result["agents"] - total_instructions_integrated += _root_result["instructions"] - total_commands_integrated += _root_result["commands"] - total_hooks_integrated += _root_result["hooks"] - total_links_resolved += _root_result["links_resolved"] - installed_count += 1 - except Exception as e: - import traceback as _tb - diagnostics.error( - f"Failed to integrate root project primitives: {e}", - package="", - detail=_tb.format_exc(), - ) - # When root integration is the *only* action (no external deps), - # a failure means nothing was deployed -- surface it clearly. - if not all_apm_deps and logger: - logger.error( - f"Root project primitives could not be integrated: {e}" - ) + root_deltas = _integrate_root_project(ctx, _install_mod) + if root_deltas: + installed_count += root_deltas.get("installed", 0) + total_prompts_integrated += root_deltas.get("prompts", 0) + total_agents_integrated += root_deltas.get("agents", 0) + total_skills_integrated += root_deltas.get("skills", 0) + total_sub_skills_promoted += root_deltas.get("sub_skills", 0) + total_instructions_integrated += root_deltas.get("instructions", 0) + total_commands_integrated += root_deltas.get("commands", 0) + total_hooks_integrated += root_deltas.get("hooks", 0) + total_links_resolved += root_deltas.get("links_resolved", 0) # ------------------------------------------------------------------ # Write int counters back to ctx (mutable containers already share diff --git a/tests/unit/install/test_architecture_invariants.py b/tests/unit/install/test_architecture_invariants.py index c21154e9..22bc543f 100644 --- a/tests/unit/install/test_architecture_invariants.py +++ b/tests/unit/install/test_architecture_invariants.py @@ -40,9 +40,7 @@ def test_install_context_importable(): MAX_MODULE_LOC = 1000 -KNOWN_LARGE_MODULES = { - "phases/integrate.py": 900, -} +KNOWN_LARGE_MODULES = {} def test_no_install_module_exceeds_loc_budget(): From d020ad57efca43255868ec1b985050b6a5f4c7da Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:19:22 +0200 Subject: [PATCH 17/28] refactor(install): F2 -- extract pipeline.py orchestrator Move _install_apm_dependencies orchestration (~237 LOC) from commands/install.py into apm_cli/install/pipeline.py as run_install_pipeline(). The Click command module drops from 1268 LOC to 1072 LOC. A thin wrapper _install_apm_dependencies() remains in commands/install.py to preserve the patch path used by 15+ test files. All interstitial code (DiagnosticCollector setup, registry config, managed_files initialization) moves with the orchestration into the pipeline module. Phase modules continue accessing patchable symbols through the existing _install_mod indirection pattern -- no changes needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 222 ++----------------------- src/apm_cli/install/pipeline.py | 286 ++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 209 deletions(-) create mode 100644 src/apm_cli/install/pipeline.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index df935b58..6cf81ba1 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -1024,6 +1024,12 @@ def _integrate_local_content( ) +# --------------------------------------------------------------------------- +# Pipeline entry point -- thin re-export preserving the patch path +# ``apm_cli.commands.install._install_apm_dependencies`` used by tests. +# +# The real implementation lives in ``apm_cli.install.pipeline`` (F2). +# --------------------------------------------------------------------------- def _install_apm_dependencies( apm_package: "APMPackage", update_refs: bool = False, @@ -1037,49 +1043,18 @@ def _install_apm_dependencies( target: str = None, marketplace_provenance: dict = None, ): - """Install APM package dependencies. + """Thin wrapper -- delegates to :func:`apm_cli.install.pipeline.run_install_pipeline`. - Args: - apm_package: Parsed APM package with dependencies - update_refs: Whether to update existing packages to latest refs - verbose: Show detailed installation information - only_packages: If provided, only install these specific packages (not all from apm.yml) - force: Whether to overwrite locally-authored files on collision - parallel_downloads: Max concurrent downloads (0 disables parallelism) - logger: InstallLogger for structured output - scope: InstallScope controlling project vs user deployment - auth_resolver: Shared auth resolver for caching credentials - target: Explicit target override from --target CLI flag + Kept here so that ``@patch("apm_cli.commands.install._install_apm_dependencies")`` + continues to intercept calls from the Click handler. """ if not APM_DEPS_AVAILABLE: raise RuntimeError("APM dependency system not available") - from apm_cli.core.scope import InstallScope, get_deploy_root, get_apm_dir - if scope is None: - scope = InstallScope.PROJECT - - apm_deps = apm_package.get_apm_dependencies() - dev_apm_deps = apm_package.get_dev_apm_dependencies() - all_apm_deps = apm_deps + dev_apm_deps - - project_root = get_deploy_root(scope) - apm_dir = get_apm_dir(scope) - - # Check whether the project root itself has local .apm/ primitives (#714). - _root_has_local_primitives = _project_has_root_primitives(project_root) + from apm_cli.install.pipeline import run_install_pipeline - if not all_apm_deps and not _root_has_local_primitives: - return InstallResult() - - # ------------------------------------------------------------------ - # Build InstallContext from function args + computed state - # ------------------------------------------------------------------ - from apm_cli.install.context import InstallContext - - ctx = InstallContext( - project_root=project_root, - apm_dir=apm_dir, - apm_package=apm_package, + return run_install_pipeline( + apm_package, update_refs=update_refs, verbose=verbose, only_packages=only_packages, @@ -1088,181 +1063,10 @@ def _install_apm_dependencies( logger=logger, scope=scope, auth_resolver=auth_resolver, - target_override=target, + target=target, marketplace_provenance=marketplace_provenance, - all_apm_deps=all_apm_deps, - root_has_local_primitives=_root_has_local_primitives, ) - # ------------------------------------------------------------------ - # Phase 1: Resolve dependencies - # ------------------------------------------------------------------ - from apm_cli.install.phases import resolve as _resolve_phase - _resolve_phase.run(ctx) - - if not ctx.deps_to_install and not ctx.root_has_local_primitives: - if logger: - logger.nothing_to_install() - return InstallResult() - - try: - # -------------------------------------------------------------- - # Phase 2: Target detection + integrator initialization - # -------------------------------------------------------------- - from apm_cli.install.phases import targets as _targets_phase - _targets_phase.run(ctx) - - # -------------------------------------------------------------- - # Seam: read phase outputs into locals for remaining code. - # This minimises diff below -- subsequent phases (download, - # integrate, cleanup, lockfile) continue using bare-name locals. - # Future S-phases will fold them into the context one by one. - # -------------------------------------------------------------- - deps_to_install = ctx.deps_to_install - intended_dep_keys = ctx.intended_dep_keys - dependency_graph = ctx.dependency_graph - existing_lockfile = ctx.existing_lockfile - lockfile_path = ctx.lockfile_path - apm_modules_dir = ctx.apm_modules_dir - downloader = ctx.downloader - callback_downloaded = ctx.callback_downloaded - callback_failures = ctx.callback_failures - transitive_failures = ctx.transitive_failures - _targets = ctx.targets - prompt_integrator = ctx.integrators["prompt"] - agent_integrator = ctx.integrators["agent"] - skill_integrator = ctx.integrators["skill"] - command_integrator = ctx.integrators["command"] - hook_integrator = ctx.integrators["hook"] - instruction_integrator = ctx.integrators["instruction"] - - diagnostics = DiagnosticCollector(verbose=verbose) - - # Drain transitive failures collected during resolution into diagnostics - for dep_display, fail_msg in transitive_failures: - diagnostics.error(fail_msg, package=dep_display) - - total_prompts_integrated = 0 - total_agents_integrated = 0 - total_skills_integrated = 0 - total_sub_skills_promoted = 0 - total_instructions_integrated = 0 - total_commands_integrated = 0 - total_hooks_integrated = 0 - total_links_resolved = 0 - - # Collect installed packages for lockfile generation - from apm_cli.deps.lockfile import LockFile, LockedDependency, get_lockfile_path - from apm_cli.deps.installed_package import InstalledPackage - from apm_cli.deps.registry_proxy import RegistryConfig - from ..utils.content_hash import compute_package_hash as _compute_hash - installed_packages: List[InstalledPackage] = [] - package_deployed_files: builtins.dict = {} # dep_key -> list of relative deployed paths - package_types: builtins.dict = {} # dep_key -> package type string - _package_hashes: builtins.dict = {} # dep_key -> sha256 hash (captured at download/verify time) - - # Resolve registry proxy configuration once for this install session. - registry_config = RegistryConfig.from_env() - - # Build managed_files from existing lockfile for collision detection - managed_files = builtins.set() - existing_lockfile = LockFile.read(get_lockfile_path(apm_dir)) if apm_dir else None - if existing_lockfile: - for dep in existing_lockfile.dependencies.values(): - managed_files.update(dep.deployed_files) - - # Conflict: registry-only mode requires all locked deps to route - # through the configured proxy. Deps locked to direct VCS sources - # (github.com, GHE Cloud, GHES) are incompatible. - if registry_config and registry_config.enforce_only: - conflicts = registry_config.validate_lockfile_deps( - list(existing_lockfile.dependencies.values()) - ) - if conflicts: - _rich_error( - "PROXY_REGISTRY_ONLY is set but the lockfile contains " - "dependencies locked to direct VCS sources:" - ) - for dep in conflicts[:10]: - host = dep.host or "github.com" - name = dep.repo_url - if dep.virtual_path: - name = f"{name}/{dep.virtual_path}" - _rich_error(f" - {name} (host: {host})") - _rich_error( - "Re-run with 'apm install --update' to re-resolve " - "through the registry, or unset PROXY_REGISTRY_ONLY." - ) - sys.exit(1) - - # Supply chain warning: registry-proxy entries without a - # content_hash cannot be verified on re-install. - if registry_config and registry_config.enforce_only: - missing = registry_config.find_missing_hashes( - list(existing_lockfile.dependencies.values()) - ) - if missing: - diagnostics.warn( - "The following registry-proxy dependencies have no " - "content_hash in the lockfile. Run 'apm install " - "--update' to populate hashes for tamper detection.", - package="lockfile", - ) - for dep in missing[:10]: - name = dep.repo_url - if dep.virtual_path: - name = f"{name}/{dep.virtual_path}" - diagnostics.warn( - f" - {name} (host: {dep.host})", - package="lockfile", - ) - - # Normalize path separators once for O(1) lookups in check_collision - from apm_cli.integration.base_integrator import BaseIntegrator - managed_files = BaseIntegrator.normalize_managed_files(managed_files) - - installed_count = 0 - unpinned_count = 0 - - # -------------------------------------------------------------- - # Phase 4 (#171): Parallel package pre-download - # -------------------------------------------------------------- - from apm_cli.install.phases import download as _download_phase - _download_phase.run(ctx) - - # -------------------------------------------------------------- - # Phase 5: Sequential integration loop + root primitives - # -------------------------------------------------------------- - # Populate ctx with locals needed by the integrate phase. - ctx.diagnostics = diagnostics - ctx.registry_config = registry_config - ctx.managed_files = managed_files - ctx.installed_packages = installed_packages - - from apm_cli.install.phases import integrate as _integrate_phase - _integrate_phase.run(ctx) - - # Update .gitignore - _update_gitignore_for_apm_modules(logger=logger) - - # ------------------------------------------------------------------ - # Phase: Orphan cleanup + intra-package stale-file cleanup - # All deletions routed through integration/cleanup.py (#762). - # ------------------------------------------------------------------ - from apm_cli.install.phases import cleanup as _cleanup_phase - _cleanup_phase.run(ctx) - - # Generate apm.lock for reproducible installs (T4: lockfile generation) - from apm_cli.install.phases.lockfile import LockfileBuilder - LockfileBuilder(ctx).build_and_save() - - # Emit verbose integration stats + bare-success fallback + return result - from apm_cli.install.phases import finalize as _finalize_phase - return _finalize_phase.run(ctx) - - except Exception as e: - raise RuntimeError(f"Failed to resolve APM dependencies: {e}") - diff --git a/src/apm_cli/install/pipeline.py b/src/apm_cli/install/pipeline.py new file mode 100644 index 00000000..af5032a7 --- /dev/null +++ b/src/apm_cli/install/pipeline.py @@ -0,0 +1,286 @@ +"""Install pipeline orchestrator. + +Extracted from ``apm_cli.commands.install._install_apm_dependencies`` +(refactor F2) to keep the Click command module under ~1 000 LOC and +concentrate the phase-call sequence in one import-safe module. + +The function ``run_install_pipeline(...)`` is the public entry point. +``commands/install.py`` re-exports it as ``_install_apm_dependencies`` +so that every existing ``@patch("apm_cli.commands.install._install_apm_dependencies")`` +keeps working without test changes. + +Design notes +------------ +* Each phase is called via its ``run(ctx)`` entry point. +* Diagnostics, registry config, and managed_files are set up here and + attached to :class:`InstallContext` *before* the phases that need them. +* Symbols on the ``commands/install`` module that phases access via + ``_install_mod.X`` stay as re-exports there -- this module does NOT + duplicate those re-exports. +""" + +from __future__ import annotations + +import builtins +import sys +from typing import TYPE_CHECKING, List + +from ..models.results import InstallResult +from ..utils.console import _rich_error +from ..utils.diagnostics import DiagnosticCollector + +if TYPE_CHECKING: + from ..core.auth import AuthResolver + from ..core.command_logger import InstallLogger + + +# CRITICAL: Shadow Python builtins that share names with Click commands. +# The parent ``commands/install`` module does this; we must do the same +# to avoid NameError when using ``set()``, ``list()``, ``dict()`` below. +set = builtins.set +list = builtins.list +dict = builtins.dict + + +def run_install_pipeline( + apm_package: "APMPackage", + update_refs: bool = False, + verbose: bool = False, + only_packages: "builtins.list" = None, + force: bool = False, + parallel_downloads: int = 4, + logger: "InstallLogger" = None, + scope=None, + auth_resolver: "AuthResolver" = None, + target: str = None, + marketplace_provenance: dict = None, +): + """Install APM package dependencies. + + This is the main orchestrator for the install pipeline. It builds an + :class:`InstallContext`, then calls each phase module in order: + + 1. **resolve** -- dependency resolution + lockfile check + 2. **targets** -- target detection + integrator initialization + 3. **download** -- parallel package pre-download + 4. **integrate** -- sequential integration loop + root primitives + 5. **cleanup** -- orphan cleanup + intra-package stale-file removal + 6. **lockfile** -- generate ``apm.lock`` + 7. **finalize** -- emit stats, return :class:`InstallResult` + + Args: + apm_package: Parsed APM package with dependencies + update_refs: Whether to update existing packages to latest refs + verbose: Show detailed installation information + only_packages: If provided, only install these specific packages + force: Whether to overwrite locally-authored files on collision + parallel_downloads: Max concurrent downloads (0 disables parallelism) + logger: InstallLogger for structured output + scope: InstallScope controlling project vs user deployment + auth_resolver: Shared auth resolver for caching credentials + target: Explicit target override from --target CLI flag + marketplace_provenance: Marketplace provenance data for packages + """ + # Late import: the ``APM_DEPS_AVAILABLE`` guard in commands/install.py + # already prevents callers from reaching here when deps are missing, but + # keep the check as a defensive belt-and-suspenders measure. + try: + from ..deps.lockfile import LockFile, get_lockfile_path # noqa: F401 + except ImportError: + raise RuntimeError("APM dependency system not available") + + from ..core.scope import InstallScope, get_deploy_root, get_apm_dir + + if scope is None: + scope = InstallScope.PROJECT + + apm_deps = apm_package.get_apm_dependencies() + dev_apm_deps = apm_package.get_dev_apm_dependencies() + all_apm_deps = apm_deps + dev_apm_deps + + project_root = get_deploy_root(scope) + apm_dir = get_apm_dir(scope) + + # Check whether the project root itself has local .apm/ primitives (#714). + from apm_cli.install.phases.local_content import _project_has_root_primitives + + _root_has_local_primitives = _project_has_root_primitives(project_root) + + if not all_apm_deps and not _root_has_local_primitives: + return InstallResult() + + # ------------------------------------------------------------------ + # Build InstallContext from function args + computed state + # ------------------------------------------------------------------ + from .context import InstallContext + + ctx = InstallContext( + project_root=project_root, + apm_dir=apm_dir, + apm_package=apm_package, + update_refs=update_refs, + verbose=verbose, + only_packages=only_packages, + force=force, + parallel_downloads=parallel_downloads, + logger=logger, + scope=scope, + auth_resolver=auth_resolver, + target_override=target, + marketplace_provenance=marketplace_provenance, + all_apm_deps=all_apm_deps, + root_has_local_primitives=_root_has_local_primitives, + ) + + # ------------------------------------------------------------------ + # Phase 1: Resolve dependencies + # ------------------------------------------------------------------ + from .phases import resolve as _resolve_phase + + _resolve_phase.run(ctx) + + if not ctx.deps_to_install and not ctx.root_has_local_primitives: + if logger: + logger.nothing_to_install() + return InstallResult() + + try: + # -------------------------------------------------------------- + # Phase 2: Target detection + integrator initialization + # -------------------------------------------------------------- + from .phases import targets as _targets_phase + + _targets_phase.run(ctx) + + # -------------------------------------------------------------- + # Seam: read phase outputs into locals for remaining code. + # This minimises diff below -- subsequent phases (download, + # integrate, cleanup, lockfile) continue using bare-name locals. + # Future S-phases will fold them into the context one by one. + # -------------------------------------------------------------- + transitive_failures = ctx.transitive_failures + apm_modules_dir = ctx.apm_modules_dir + + diagnostics = DiagnosticCollector(verbose=verbose) + + # Drain transitive failures collected during resolution into diagnostics + for dep_display, fail_msg in transitive_failures: + diagnostics.error(fail_msg, package=dep_display) + + # Collect installed packages for lockfile generation + from ..deps.lockfile import LockFile, get_lockfile_path + from ..deps.installed_package import InstalledPackage + from ..deps.registry_proxy import RegistryConfig + from ..utils.content_hash import compute_package_hash as _compute_hash + + installed_packages: List[InstalledPackage] = [] + package_deployed_files: builtins.dict = {} # dep_key -> list of relative deployed paths + package_types: builtins.dict = {} # dep_key -> package type string + _package_hashes: builtins.dict = {} # dep_key -> sha256 hash + + # Resolve registry proxy configuration once for this install session. + registry_config = RegistryConfig.from_env() + + # Build managed_files from existing lockfile for collision detection + managed_files = builtins.set() + existing_lockfile = LockFile.read(get_lockfile_path(apm_dir)) if apm_dir else None + if existing_lockfile: + for dep in existing_lockfile.dependencies.values(): + managed_files.update(dep.deployed_files) + + # Conflict: registry-only mode requires all locked deps to route + # through the configured proxy. Deps locked to direct VCS sources + # (github.com, GHE Cloud, GHES) are incompatible. + if registry_config and registry_config.enforce_only: + conflicts = registry_config.validate_lockfile_deps( + builtins.list(existing_lockfile.dependencies.values()) + ) + if conflicts: + _rich_error( + "PROXY_REGISTRY_ONLY is set but the lockfile contains " + "dependencies locked to direct VCS sources:" + ) + for dep in conflicts[:10]: + host = dep.host or "github.com" + name = dep.repo_url + if dep.virtual_path: + name = f"{name}/{dep.virtual_path}" + _rich_error(f" - {name} (host: {host})") + _rich_error( + "Re-run with 'apm install --update' to re-resolve " + "through the registry, or unset PROXY_REGISTRY_ONLY." + ) + sys.exit(1) + + # Supply chain warning: registry-proxy entries without a + # content_hash cannot be verified on re-install. + if registry_config and registry_config.enforce_only: + missing = registry_config.find_missing_hashes( + builtins.list(existing_lockfile.dependencies.values()) + ) + if missing: + diagnostics.warn( + "The following registry-proxy dependencies have no " + "content_hash in the lockfile. Run 'apm install " + "--update' to populate hashes for tamper detection.", + package="lockfile", + ) + for dep in missing[:10]: + name = dep.repo_url + if dep.virtual_path: + name = f"{name}/{dep.virtual_path}" + diagnostics.warn( + f" - {name} (host: {dep.host})", + package="lockfile", + ) + + # Normalize path separators once for O(1) lookups in check_collision + from ..integration.base_integrator import BaseIntegrator + + managed_files = BaseIntegrator.normalize_managed_files(managed_files) + + # -------------------------------------------------------------- + # Phase 4 (#171): Parallel package pre-download + # -------------------------------------------------------------- + from .phases import download as _download_phase + + _download_phase.run(ctx) + + # -------------------------------------------------------------- + # Phase 5: Sequential integration loop + root primitives + # -------------------------------------------------------------- + # Populate ctx with locals needed by the integrate phase. + ctx.diagnostics = diagnostics + ctx.registry_config = registry_config + ctx.managed_files = managed_files + ctx.installed_packages = installed_packages + + from .phases import integrate as _integrate_phase + + _integrate_phase.run(ctx) + + # Update .gitignore + from apm_cli.commands._helpers import _update_gitignore_for_apm_modules + + _update_gitignore_for_apm_modules(logger=logger) + + # ------------------------------------------------------------------ + # Phase: Orphan cleanup + intra-package stale-file cleanup + # All deletions routed through integration/cleanup.py (#762). + # ------------------------------------------------------------------ + from .phases import cleanup as _cleanup_phase + + _cleanup_phase.run(ctx) + + # Generate apm.lock for reproducible installs (T4: lockfile generation) + from .phases.lockfile import LockfileBuilder + + LockfileBuilder(ctx).build_and_save() + + # Emit verbose integration stats + bare-success fallback + return result + from .phases import finalize as _finalize_phase + + return _finalize_phase.run(ctx) + + except Exception as e: + raise RuntimeError(f"Failed to resolve APM dependencies: {e}") From d518e932c145182ac1dabd9b3abca534018a4707 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:32:40 +0200 Subject: [PATCH 18/28] refactor(install): F3 -- fold local-content integration into pipeline Move the ~140 LOC "local .apm/ content integration" block from the Click install() handler into the install pipeline, eliminating the duplicate target resolution and integrator initialization identified in PR #764. Changes: - Rewrite _integrate_root_project in phases/integrate.py to delegate to _integrate_local_content (preserves test-patch contract and correct PackageType.APM_PACKAGE for root SKILL.md handling) - Create phases/post_deps_local.py for stale cleanup + lockfile persistence of local_deployed_files (routes through integration/cleanup.py per #762) - Extend InstallContext with old_local_deployed, local_deployed_files, and local_content_errors_before fields - Extend pipeline early-exit to consider old_local_deployed (stale cleanup must run even when .apm/ is removed) - Remove the 140-line inline block from commands/install.py Click handler commands/install.py: 1072 -> 933 LOC (-139) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 149 +----------------- src/apm_cli/install/context.py | 7 + src/apm_cli/install/phases/integrate.py | 53 +++++-- src/apm_cli/install/phases/post_deps_local.py | 122 ++++++++++++++ src/apm_cli/install/pipeline.py | 22 ++- .../install/test_architecture_invariants.py | 7 +- 6 files changed, 205 insertions(+), 155 deletions(-) create mode 100644 src/apm_cli/install/phases/post_deps_local.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 6cf81ba1..5c28093e 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -552,13 +552,11 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # field after the lockfile is regenerated by the APM install step. old_mcp_servers: builtins.set = builtins.set() old_mcp_configs: builtins.dict = {} - old_local_deployed: builtins.list = [] _lock_path = get_lockfile_path(apm_dir) _existing_lock = LockFile.read(_lock_path) if _existing_lock: old_mcp_servers = builtins.set(_existing_lock.mcp_servers) old_mcp_configs = builtins.dict(_existing_lock.mcp_configs) - old_local_deployed = builtins.list(_existing_lock.local_deployed_files) # Also enter the APM install path when the project root has local .apm/ # primitives, even if there are no external APM dependencies (#714). @@ -650,148 +648,11 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # mcp_servers. Restore the previous set so it is not lost. MCPIntegrator.update_lockfile(old_mcp_servers, mcp_configs=old_mcp_configs) - # --- Local .apm/ content integration --- - # Deploy primitives from the project's own .apm/ folder to target - # directories, just like dependency primitives. Runs AFTER deps so - # local content wins on collision. - if ( - should_install_apm - and scope is InstallScope.PROJECT - and not dry_run - and (_has_local_apm_content(project_root) or old_local_deployed) - ): - try: - from apm_cli.integration.targets import resolve_targets as _local_resolve - from apm_cli.integration.skill_integrator import SkillIntegrator - from apm_cli.integration.command_integrator import CommandIntegrator - from apm_cli.integration.hook_integrator import HookIntegrator - from apm_cli.integration.instruction_integrator import InstructionIntegrator - from apm_cli.integration.base_integrator import BaseIntegrator - from apm_cli.deps.lockfile import LockFile as _LocalLF, get_lockfile_path as _local_lf_path - from apm_cli.integration import AgentIntegrator as _AgentInt, PromptIntegrator as _PromptInt - - # Resolve targets (same precedence as _install_apm_dependencies) - _local_config_target = apm_package.target - _local_explicit = target or _local_config_target or None - _local_targets = _local_resolve( - project_root, user_scope=False, explicit_target=_local_explicit, - ) - - if _local_targets: - # Build managed_files: dep-deployed files + previous local - # deployed files. This ensures local content wins - # collisions with deps and previous local files are not - # treated as user-authored content. - _local_managed = builtins.set() - _local_lock_path = _local_lf_path(apm_dir) - _local_lock = _LocalLF.read(_local_lock_path) - if _local_lock: - for dep in _local_lock.dependencies.values(): - _local_managed.update(dep.deployed_files) - # Include previous local deployed files so re-deploys - # overwrite rather than skip. - _local_managed.update(old_local_deployed) - _local_managed = BaseIntegrator.normalize_managed_files(_local_managed) - - # Create integrators - _local_diagnostics = apm_diagnostics or DiagnosticCollector(verbose=verbose) - _errors_before_local = _local_diagnostics.error_count - _local_prompt_int = _PromptInt() - _local_agent_int = _AgentInt() - _local_skill_int = SkillIntegrator() - _local_instr_int = InstructionIntegrator() - _local_cmd_int = CommandIntegrator() - _local_hook_int = HookIntegrator() - - logger.verbose_detail("Integrating local .apm/ content...") - - local_int_result = _integrate_local_content( - project_root, - targets=_local_targets, - prompt_integrator=_local_prompt_int, - agent_integrator=_local_agent_int, - skill_integrator=_local_skill_int, - instruction_integrator=_local_instr_int, - command_integrator=_local_cmd_int, - hook_integrator=_local_hook_int, - force=force, - managed_files=_local_managed, - diagnostics=_local_diagnostics, - logger=logger, - scope=scope, - ) - - # Track what local integration deployed - _local_deployed = local_int_result.get("deployed_files", []) - _local_total = sum( - local_int_result.get(k, 0) - for k in ("prompts", "agents", "skills", "sub_skills", - "instructions", "commands", "hooks") - ) - - if _local_total > 0: - logger.verbose_detail( - f"Deployed {_local_total} local primitive(s) from .apm/" - ) - - # Stale cleanup: remove files deployed by previous local - # integration that are no longer produced. Only run when - # integration completed without errors to avoid deleting - # files that failed to re-deploy. - _local_had_errors = ( - _local_diagnostics is not None - and _local_diagnostics.error_count > _errors_before_local - ) - if old_local_deployed and not _local_had_errors: - from ..integration.cleanup import remove_stale_deployed_files as _rmstale - _stale = builtins.set(old_local_deployed) - builtins.set(_local_deployed) - if _stale: - _local_prev_hashes = {} - _prev_local_lf = _LocalLF.read(_local_lock_path) - if _prev_local_lf: - _local_prev_hashes = dict( - _prev_local_lf.local_deployed_file_hashes - ) - _cleanup_result = _rmstale( - _stale, project_root, - dep_key="", - targets=_local_targets, - diagnostics=_local_diagnostics, - recorded_hashes=_local_prev_hashes, - ) - # Failed paths stay in lockfile so we retry next time. - _local_deployed.extend(_cleanup_result.failed) - if _cleanup_result.deleted_targets: - BaseIntegrator.cleanup_empty_parents( - _cleanup_result.deleted_targets, project_root - ) - for _skipped in _cleanup_result.skipped_user_edit: - logger.cleanup_skipped_user_edit( - _skipped, "" - ) - logger.stale_cleanup( - "", len(_cleanup_result.deleted) - ) - - # Persist local_deployed_files (and hashes) in the lockfile - _persist_lock = _LocalLF.read(_local_lock_path) or _LocalLF() - _persist_lock.local_deployed_files = sorted(_local_deployed) - _persist_lock.local_deployed_file_hashes = _hash_deployed( - _local_deployed, project_root - ) - # Only write if changed - _existing_for_cmp = _LocalLF.read(_local_lock_path) - if not _existing_for_cmp or not _persist_lock.is_semantically_equivalent(_existing_for_cmp): - _persist_lock.save(_local_lock_path) - - # Ensure diagnostics flow into the final summary - if apm_diagnostics is None: - apm_diagnostics = _local_diagnostics - - except Exception as e: - logger.verbose_detail(f"Local .apm/ integration failed: {e}") - if apm_diagnostics: - apm_diagnostics.error(f"Local .apm/ integration failed: {e}") + # Local .apm/ content integration is now handled inside the + # install pipeline (phases/integrate.py + phases/post_deps_local.py, + # refactor F3). The duplicate target resolution, integrator + # initialization, and inline stale-cleanup block that lived here + # have been removed. # Show diagnostics and final install summary if apm_diagnostics and apm_diagnostics.has_diagnostics: diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py index 30ee1877..7f5ac87a 100644 --- a/src/apm_cli/install/context.py +++ b/src/apm_cli/install/context.py @@ -105,3 +105,10 @@ class InstallContext: total_commands_integrated: int = 0 # integrate total_hooks_integrated: int = 0 # integrate total_links_resolved: int = 0 # integrate + + # ------------------------------------------------------------------ + # Post-deps local content tracking (F3) + # ------------------------------------------------------------------ + old_local_deployed: List[str] = field(default_factory=list) # pipeline setup + local_deployed_files: List[str] = field(default_factory=list) # integrate (root) + local_content_errors_before: int = 0 # integrate (pre-root) diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index e7f65bc3..20343113 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -770,25 +770,44 @@ def _integrate_root_project( found in /.apm/ are integrated after all declared dependency packages have been processed. + Delegates to ``_install_mod._integrate_local_content`` which creates a + synthetic ``_local`` APMPackage with ``PackageType.APM_PACKAGE`` so that + a root-level ``SKILL.md`` is NOT deployed as a skill. Deployed files + are tracked on ``ctx.local_deployed_files`` for the downstream + post-deps-local phase (stale cleanup + lockfile persistence). + Returns a counter-delta dict, or ``None`` if root integration is not applicable or failed. """ if not ctx.root_has_local_primitives or not ctx.targets: return None + import builtins + from apm_cli.integration.base_integrator import BaseIntegrator + logger = ctx.logger diagnostics = ctx.diagnostics - from apm_cli.models.apm_package import PackageInfo as _PackageInfo - _root_pkg_info = _PackageInfo( - package=ctx.apm_package, - install_path=ctx.project_root, - ) + # Track error count before local integration so the post-deps-local + # phase can decide whether stale cleanup is safe. + ctx.local_content_errors_before = diagnostics.error_count if diagnostics else 0 + + # Build managed_files that includes old local deployed files AND + # freshly-deployed dep files so local content wins collisions with + # both. This matches the pre-refactor Click handler behavior where + # managed_files was rebuilt from the post-install lockfile. + _local_managed = builtins.set(ctx.managed_files) + _local_managed.update(ctx.old_local_deployed) + for _dep_files in ctx.package_deployed_files.values(): + _local_managed.update(_dep_files) + _local_managed = BaseIntegrator.normalize_managed_files(_local_managed) + if logger: logger.download_complete("", ref_suffix="local") + logger.verbose_detail("Integrating local .apm/ content...") try: - _root_result = _install_mod._integrate_package_primitives( - _root_pkg_info, ctx.project_root, + _root_result = _install_mod._integrate_local_content( + ctx.project_root, targets=ctx.targets, prompt_integrator=ctx.integrators["prompt"], agent_integrator=ctx.integrators["agent"], @@ -797,16 +816,32 @@ def _integrate_root_project( command_integrator=ctx.integrators["command"], hook_integrator=ctx.integrators["hook"], force=ctx.force, - managed_files=ctx.managed_files, + managed_files=_local_managed, diagnostics=diagnostics, - package_name="", logger=logger, scope=ctx.scope, ) + + # Track deployed files for the post-deps-local phase (stale + # cleanup + lockfile persistence of local_deployed_files). + ctx.local_deployed_files = _root_result.get("deployed_files", []) + + _local_total = sum( + _root_result.get(k, 0) + for k in ("prompts", "agents", "skills", "sub_skills", + "instructions", "commands", "hooks") + ) + if _local_total > 0 and logger: + logger.verbose_detail( + f"Deployed {_local_total} local primitive(s) from .apm/" + ) + return { "installed": 1, "prompts": _root_result["prompts"], "agents": _root_result["agents"], + "skills": _root_result.get("skills", 0), + "sub_skills": _root_result.get("sub_skills", 0), "instructions": _root_result["instructions"], "commands": _root_result["commands"], "hooks": _root_result["hooks"], diff --git a/src/apm_cli/install/phases/post_deps_local.py b/src/apm_cli/install/phases/post_deps_local.py new file mode 100644 index 00000000..53fadc09 --- /dev/null +++ b/src/apm_cli/install/phases/post_deps_local.py @@ -0,0 +1,122 @@ +"""Post-deps local content: stale cleanup + lockfile persistence. + +Handles the second half of the local ``.apm/`` content integration +lifecycle that was previously an inline block in the Click ``install()`` +handler (lines 653-795 pre-F3). The first half -- actually deploying +the primitives -- is handled by ``_integrate_root_project`` in the +``integrate`` phase. + +Two responsibilities: + +1. **Stale cleanup** -- remove files deployed by a *previous* local + integration that are no longer produced. Only runs when the + current integration completed without errors (avoids deleting files + that failed to re-deploy). All deletions route through the + canonical security chokepoint + ``apm_cli.integration.cleanup.remove_stale_deployed_files`` (PR #762). + +2. **Lockfile persistence** -- read-modify-write the lockfile to persist + ``local_deployed_files`` and per-file content hashes. Runs after the + dep lockfile phase has already written dependency data; this phase + simply augments the on-disk lockfile with the local fields. + +Scope guard: this phase only runs for ``InstallScope.PROJECT``. User- +scope installs do not track local deployed files (matching pre-refactor +behavior). +""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + + +def run(ctx: "InstallContext") -> None: + """Execute local content stale cleanup and lockfile persistence. + + Reads ``ctx.local_deployed_files``, ``ctx.old_local_deployed``, + ``ctx.local_content_errors_before``, ``ctx.diagnostics``, + ``ctx.targets``, ``ctx.logger``, ``ctx.project_root``, ``ctx.apm_dir``. + + Mutates ``ctx.local_deployed_files`` (appends failed cleanup paths). + """ + from apm_cli.core.scope import InstallScope + + # Scope guard: only PROJECT scope tracks local deployed files. + if ctx.scope is not InstallScope.PROJECT: + return + + # Skip if there is no local content (current or previous). + if not ctx.local_deployed_files and not ctx.old_local_deployed: + return + + diagnostics = ctx.diagnostics + logger = ctx.logger + + # ------------------------------------------------------------------ + # Stale cleanup: remove files deployed by previous local integration + # that are no longer produced. Only run when integration completed + # without errors to avoid deleting files that failed to re-deploy. + # ------------------------------------------------------------------ + _local_had_errors = ( + diagnostics is not None + and diagnostics.error_count > ctx.local_content_errors_before + ) + + if ctx.old_local_deployed and not _local_had_errors: + from apm_cli.integration.base_integrator import BaseIntegrator + from apm_cli.integration.cleanup import remove_stale_deployed_files + + _stale = builtins.set(ctx.old_local_deployed) - builtins.set(ctx.local_deployed_files) + if _stale: + # Get recorded hashes from the pre-install lockfile for + # content-hash provenance verification. + _prev_hashes: dict = {} + if ctx.existing_lockfile: + _prev_hashes = dict(ctx.existing_lockfile.local_deployed_file_hashes) + + _cleanup_result = remove_stale_deployed_files( + _stale, + ctx.project_root, + dep_key="", + targets=ctx.targets, + diagnostics=diagnostics, + recorded_hashes=_prev_hashes, + ) + # Failed paths stay in lockfile so we retry next time. + ctx.local_deployed_files.extend(_cleanup_result.failed) + if _cleanup_result.deleted_targets: + BaseIntegrator.cleanup_empty_parents( + _cleanup_result.deleted_targets, ctx.project_root + ) + for _skipped in _cleanup_result.skipped_user_edit: + if logger: + logger.cleanup_skipped_user_edit(_skipped, "") + if logger: + logger.stale_cleanup( + "", len(_cleanup_result.deleted) + ) + + # ------------------------------------------------------------------ + # Lockfile persistence: read-modify-write the lockfile to add + # local_deployed_files and per-file content hashes. + # ------------------------------------------------------------------ + from apm_cli.deps.lockfile import LockFile as _LF, get_lockfile_path as _get_lfp + from apm_cli.install.phases.lockfile import compute_deployed_hashes as _hash_deployed + + _lock_path = _get_lfp(ctx.apm_dir) + _persist_lock = _LF.read(_lock_path) or _LF() + _persist_lock.local_deployed_files = sorted(ctx.local_deployed_files) + _persist_lock.local_deployed_file_hashes = _hash_deployed( + ctx.local_deployed_files, ctx.project_root + ) + # Only write if changed. + _existing_for_cmp = _LF.read(_lock_path) + if ( + not _existing_for_cmp + or not _persist_lock.is_semantically_equivalent(_existing_for_cmp) + ): + _persist_lock.save(_lock_path) diff --git a/src/apm_cli/install/pipeline.py b/src/apm_cli/install/pipeline.py index af5032a7..5dadaf2e 100644 --- a/src/apm_cli/install/pipeline.py +++ b/src/apm_cli/install/pipeline.py @@ -106,7 +106,15 @@ def run_install_pipeline( _root_has_local_primitives = _project_has_root_primitives(project_root) - if not all_apm_deps and not _root_has_local_primitives: + # Read old local deployed files from the existing lockfile so the + # post-deps-local phase can run stale cleanup even when no current + # local content exists (e.g. .apm/ was deleted but old files remain). + _old_local_deployed: builtins.list = [] + _early_lockfile = LockFile.read(get_lockfile_path(apm_dir)) if apm_dir else None + if _early_lockfile: + _old_local_deployed = builtins.list(_early_lockfile.local_deployed_files) + + if not all_apm_deps and not _root_has_local_primitives and not _old_local_deployed: return InstallResult() # ------------------------------------------------------------------ @@ -130,6 +138,7 @@ def run_install_pipeline( marketplace_provenance=marketplace_provenance, all_apm_deps=all_apm_deps, root_has_local_primitives=_root_has_local_primitives, + old_local_deployed=_old_local_deployed, ) # ------------------------------------------------------------------ @@ -277,6 +286,17 @@ def run_install_pipeline( LockfileBuilder(ctx).build_and_save() + # ------------------------------------------------------------------ + # Phase: Post-deps local .apm/ content -- stale cleanup + + # lockfile persistence for the project's own .apm/ primitives. + # Runs after the dep lockfile so it can read-modify-write the + # lockfile with local_deployed_files / hashes. All deletions + # routed through integration/cleanup.py (#762). + # ------------------------------------------------------------------ + from .phases import post_deps_local as _post_deps_local_phase + + _post_deps_local_phase.run(ctx) + # Emit verbose integration stats + bare-success fallback + return result from .phases import finalize as _finalize_phase diff --git a/tests/unit/install/test_architecture_invariants.py b/tests/unit/install/test_architecture_invariants.py index 22bc543f..09714b08 100644 --- a/tests/unit/install/test_architecture_invariants.py +++ b/tests/unit/install/test_architecture_invariants.py @@ -40,7 +40,12 @@ def test_install_context_importable(): MAX_MODULE_LOC = 1000 -KNOWN_LARGE_MODULES = {} +KNOWN_LARGE_MODULES = { + # integrate.py hosts 4 per-package code paths + the root-project + # integration rewritten in F3 to use _integrate_local_content. + # Natural seam: decompose per-package helpers into a sub-module. + "phases/integrate.py": 1020, +} def test_no_install_module_exceeds_loc_budget(): From 3914f694b1b5a6372b5b783e722b4797d25cb2c5 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:39:32 +0200 Subject: [PATCH 19/28] refactor(install): F4b -- guard logger calls in cleanup phase + thread logger to local copy UX re-review found 2 logger-pattern inconsistencies introduced by F1/F3: - phases/cleanup.py: 4 unguarded logger.X() calls (orphan_cleanup, cleanup_skipped_user_edit, stale_cleanup) -- every other phase guards with 'if logger:' since ctx.logger defaults to None. - phases/integrate.py:204: _copy_local_package called without logger= in the local-dep helper, breaking F4d's single-output-path invariant for one of the two callsites. Both produce correct output today (pipeline always passes a logger; the fallback path uses _rich_*) but the inconsistency is fragile for future callers and tests. Aligning all phases on the same guard pattern. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/phases/cleanup.py | 12 ++++++++---- src/apm_cli/install/phases/integrate.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/apm_cli/install/phases/cleanup.py b/src/apm_cli/install/phases/cleanup.py index ae7ef8a5..67436ddb 100644 --- a/src/apm_cli/install/phases/cleanup.py +++ b/src/apm_cli/install/phases/cleanup.py @@ -94,12 +94,14 @@ def run(ctx: InstallContext) -> None: _orphan_total_deleted += len(_orphan_result.deleted) _orphan_deleted_targets.extend(_orphan_result.deleted_targets) for _skipped in _orphan_result.skipped_user_edit: - logger.cleanup_skipped_user_edit(_skipped, _orphan_key) + if logger: + logger.cleanup_skipped_user_edit(_skipped, _orphan_key) if _orphan_deleted_targets: BaseIntegrator.cleanup_empty_parents( _orphan_deleted_targets, project_root ) - logger.orphan_cleanup(_orphan_total_deleted) + if logger: + logger.orphan_cleanup(_orphan_total_deleted) # ------------------------------------------------------------------ # Stale-file cleanup: within each package still present in the @@ -145,5 +147,7 @@ def run(ctx: InstallContext) -> None: cleanup_result.deleted_targets, project_root ) for _skipped in cleanup_result.skipped_user_edit: - logger.cleanup_skipped_user_edit(_skipped, dep_key) - logger.stale_cleanup(dep_key, len(cleanup_result.deleted)) + if logger: + logger.cleanup_skipped_user_edit(_skipped, dep_key) + if logger: + logger.stale_cleanup(dep_key, len(cleanup_result.deleted)) diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index 20343113..b7f9ac7c 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -201,7 +201,7 @@ def _integrate_local_dep( ) return None - result_path = _install_mod._copy_local_package(dep_ref, install_path, ctx.project_root) + result_path = _install_mod._copy_local_package(dep_ref, install_path, ctx.project_root, logger=logger) if not result_path: diagnostics.error( f"Failed to copy local package: {dep_ref.local_path}", From 0e649ffb122650b5396ef50f459e4605739ee3d8 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 13:40:53 +0200 Subject: [PATCH 20/28] docs(changelog): record install-modularization refactor and F3 stale-cleanup hash bug-fix Per architect re-review of #764: F3 closes a latent bug where local .apm/ stale-cleanup was reading the lockfile after regeneration, losing local_deployed_file_hashes and silently skipping the user-edit gate. Worth calling out distinctly from the broader refactor entry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a697efe..e8b2f43d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Harden `apm install` stale-file cleanup to prevent unsafe lockfile deletions, preserve user-edited files via per-file SHA-256 provenance, and improve cleanup reporting during install and `--dry-run` (#666, #762) +- Local `.apm/` stale-cleanup now uses pre-install content hashes for provenance verification. Previously the lockfile was re-read after regeneration, which always yielded empty hashes, causing the user-edit safety gate to be silently skipped for project-local files (#764) - Fix `apm marketplace add` silently failing for private repos by using credentials when probing `marketplace.json` (#701) - Harden marketplace plugin normalization to enforce that manifest-declared `agents`/`skills`/`commands`/`hooks` paths resolve inside the plugin root (#760) - Stop `test_auto_detect_through_proxy` from making real `api.github.com` calls by passing a mock `auth_resolver`, fixing flaky macOS CI rate-limit failures (#759) @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `apm marketplace browse/search/add/update` now route through the registry proxy when `PROXY_REGISTRY_URL` is set; `PROXY_REGISTRY_ONLY=1` blocks direct GitHub API calls (#506) +- Refactor `apm install` into a modular engine package (`apm_cli/install/`) with discrete phases (resolve, targets, download, integrate, cleanup, lockfile, finalize, post-deps local). Reduces `commands/install.py` from 2905 to ~933 LOC while preserving behaviour and the `#762` cleanup chokepoint (#764) ## [0.8.11] - 2026-04-06 From 64336b94ce8a026866e08990845a9477ab50bf9a Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 14:21:03 +0200 Subject: [PATCH 21/28] refactor(install): P1.1 -- move integration template to install/services.py Move `_integrate_package_primitives` and `_integrate_local_content` from the Click command module to a new `apm_cli/install/services.py` so the install engine package owns its own integration template. `commands/install` keeps both name forms re-exported for backward compatibility with external callers and the 55 healthy `@patch` sites. The 5 `@patch("apm_cli.commands.install._integrate_package_primitives")` sites in `test_local_content_install.py` now patch the canonical `apm_cli.install.services.integrate_package_primitives` directly, because services.`_integrate_local_content` calls services.`integrate_package_primitives` by bare name -- a re-export aliased on `commands/install` would not be intercepted from inside services. This is the first commit of the design-patterns refactor (Strategy + DI + Application Service) following the install-modularization PR. Behaviour preserved. 3974 unit tests green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 209 ++------------------ src/apm_cli/install/services.py | 230 +++++++++++++++++++++++ tests/unit/test_local_content_install.py | 10 +- 3 files changed, 251 insertions(+), 198 deletions(-) create mode 100644 src/apm_cli/install/services.py diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 5c28093e..7c07f423 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -51,9 +51,9 @@ # Re-export local-content leaf helpers so that callers inside this module # (e.g. _install_apm_dependencies) and any future test patches against # "apm_cli.commands.install._copy_local_package" keep working. -# _integrate_local_content stays here (not moved) because it calls -# _integrate_package_primitives via bare-name lookup and tests patch -# apm_cli.commands.install._integrate_package_primitives to intercept it. +# _integrate_package_primitives and _integrate_local_content live in +# apm_cli.install.services (P1 -- DI seam). Re-exports below preserve +# the existing import contract for tests and external callers. from apm_cli.install.phases.local_content import ( _copy_local_package, _has_local_apm_content, @@ -691,198 +691,21 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo # --------------------------------------------------------------------------- -def _integrate_package_primitives( - package_info, - project_root, - *, - targets, - prompt_integrator, - agent_integrator, - skill_integrator, - instruction_integrator, - command_integrator, - hook_integrator, - force, - managed_files, - diagnostics, - package_name="", - logger=None, - scope=None, -): - """Run the full integration pipeline for a single package. - - Iterates over *targets* (``TargetProfile`` list) and dispatches each - primitive to the appropriate integrator via the target-driven API. - Skills are handled separately because ``SkillIntegrator`` already - routes across all targets internally. - - When *scope* is ``InstallScope.USER``, targets and primitives that - do not support user-scope deployment are silently skipped. - - Returns a dict with integration counters and the list of deployed file paths. - """ - from apm_cli.integration.dispatch import get_dispatch_table - - _dispatch = get_dispatch_table() - result = { - "prompts": 0, - "agents": 0, - "skills": 0, - "sub_skills": 0, - "instructions": 0, - "commands": 0, - "hooks": 0, - "links_resolved": 0, - "deployed_files": [], - } - - deployed = result["deployed_files"] - - if not targets: - return result - - def _log_integration(msg): - if logger: - logger.tree_item(msg) - - # Map integrator kwargs to dispatch table keys - _INTEGRATOR_KWARGS = { - "prompts": prompt_integrator, - "agents": agent_integrator, - "commands": command_integrator, - "instructions": instruction_integrator, - "hooks": hook_integrator, - "skills": skill_integrator, - } - - # --- per-target dispatch loop --- - for _target in targets: - for _prim_name, _mapping in _target.primitives.items(): - _entry = _dispatch.get(_prim_name) - if not _entry or _entry.multi_target: - continue # skills handled below - - _integrator = _INTEGRATOR_KWARGS[_prim_name] - _int_result = getattr(_integrator, _entry.integrate_method)( - _target, package_info, project_root, - force=force, managed_files=managed_files, - diagnostics=diagnostics, - ) - - if _int_result.files_integrated > 0: - result[_entry.counter_key] += _int_result.files_integrated - _effective_root = _mapping.deploy_root or _target.root_dir - _deploy_dir = f"{_effective_root}/{_mapping.subdir}/" if _mapping.subdir else f"{_effective_root}/" - # Determine display label - if _prim_name == "instructions" and _mapping.format_id in ("cursor_rules", "claude_rules"): - _label = "rule(s)" - elif _prim_name == "instructions": - _label = "instruction(s)" - elif _prim_name == "hooks": - if _target.name == "claude": - _deploy_dir = ".claude/settings.json" - elif _target.name == "cursor": - _deploy_dir = ".cursor/hooks.json" - elif _target.name == "codex": - _deploy_dir = ".codex/hooks.json" - _label = "hook(s)" - else: - _label = _prim_name - _log_integration( - f" |-- {_int_result.files_integrated} {_label} integrated -> {_deploy_dir}" - ) - result["links_resolved"] += _int_result.links_resolved - for tp in _int_result.target_paths: - deployed.append(tp.relative_to(project_root).as_posix()) - - # --- skills (multi-target, handled by SkillIntegrator internally) --- - skill_result = skill_integrator.integrate_package_skill( - package_info, project_root, - diagnostics=diagnostics, managed_files=managed_files, force=force, - targets=targets, - ) - _skill_target_dirs: set[str] = builtins.set() - for tp in skill_result.target_paths: - rel = tp.relative_to(project_root) - if rel.parts: - _skill_target_dirs.add(rel.parts[0]) - _skill_targets = sorted(_skill_target_dirs) - _skill_target_str = ", ".join(f"{d}/skills/" for d in _skill_targets) or "skills/" - if skill_result.skill_created: - result["skills"] += 1 - _log_integration(f" |-- Skill integrated -> {_skill_target_str}") - if skill_result.sub_skills_promoted > 0: - result["sub_skills"] += skill_result.sub_skills_promoted - _log_integration(f" |-- {skill_result.sub_skills_promoted} skill(s) integrated -> {_skill_target_str}") - for tp in skill_result.target_paths: - deployed.append(tp.relative_to(project_root).as_posix()) - - return result - - -def _integrate_local_content( - project_root, - *, - targets, - prompt_integrator, - agent_integrator, - skill_integrator, - instruction_integrator, - command_integrator, - hook_integrator, - force, - managed_files, - diagnostics, - logger=None, - scope=None, -): - """Integrate primitives from the project's own .apm/ directory. - - This treats the project root as a synthetic package so that local - skills, instructions, agents, prompts, hooks, and commands in .apm/ - are deployed to target directories exactly like dependency primitives. +# Re-exports for backward compatibility -- the real implementations live +# in apm_cli.install.services (P1 -- DI seam). Tests that +# @patch("apm_cli.commands.install._integrate_package_primitives") still +# work because patching this module-level alias rebinds the name where +# call-sites in this module would look it up. Tests inside this codebase +# now patch the canonical apm_cli.install.services._integrate_package_primitives +# directly to avoid relying on transitive aliasing. +from apm_cli.install.services import ( + integrate_package_primitives, + integrate_local_content, + _integrate_package_primitives, + _integrate_local_content, +) - Only .apm/ sub-directories are processed. A root-level SKILL.md is - intentionally ignored (it describes the project itself, not a - deployable skill). - Returns a dict with integration counters and deployed file paths, - same shape as ``_integrate_package_primitives()``. - """ - from ..models.apm_package import APMPackage, PackageInfo, PackageType - - # Build a lightweight synthetic PackageInfo rooted at the project. - # package_type=APM_PACKAGE prevents SkillIntegrator from treating - # a root SKILL.md as a native skill to deploy. - local_pkg = APMPackage( - name="_local", - version="0.0.0", - package_path=project_root, - source="local", - ) - local_info = PackageInfo( - package=local_pkg, - install_path=project_root, - package_type=PackageType.APM_PACKAGE, - ) - - return _integrate_package_primitives( - local_info, - project_root, - targets=targets, - prompt_integrator=prompt_integrator, - agent_integrator=agent_integrator, - skill_integrator=skill_integrator, - instruction_integrator=instruction_integrator, - command_integrator=command_integrator, - hook_integrator=hook_integrator, - force=force, - managed_files=managed_files, - diagnostics=diagnostics, - package_name="_local", - logger=logger, - scope=scope, - ) # --------------------------------------------------------------------------- diff --git a/src/apm_cli/install/services.py b/src/apm_cli/install/services.py new file mode 100644 index 00000000..13211b15 --- /dev/null +++ b/src/apm_cli/install/services.py @@ -0,0 +1,230 @@ +"""Package integration services. + +The two functions in this module own the *integration template* for a single +package -- looping over the resolved targets, dispatching primitives to their +integrators, accumulating counters, and recording deployed file paths. + +Moved here from ``apm_cli.commands.install`` so that the install engine +package owns its own integration logic. ``commands/install`` keeps thin +underscore-prefixed re-exports for backward compatibility with existing +``@patch`` sites and direct imports. + +Design notes +------------ +``integrate_local_content()`` calls ``integrate_package_primitives()`` via a +bare-name lookup so that ``@patch`` of either symbol on this module's +namespace intercepts both call paths consistently. +""" + +from __future__ import annotations + +import builtins +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from ..core.command_logger import InstallLogger + from ..core.scope import InstallScope + from ..utils.diagnostics import DiagnosticCollector + + +# CRITICAL: Shadow Python builtins that share names with Click commands so +# ``set()`` / ``list()`` / ``dict()`` resolve to the builtins, not Click +# subcommand objects. ``commands/install`` and ``install/pipeline`` do the +# same dance for the same reason. +set = builtins.set +list = builtins.list +dict = builtins.dict + + +def integrate_package_primitives( + package_info: Any, + project_root: Path, + *, + targets: Any, + prompt_integrator: Any, + agent_integrator: Any, + skill_integrator: Any, + instruction_integrator: Any, + command_integrator: Any, + hook_integrator: Any, + force: bool, + managed_files: Any, + diagnostics: "DiagnosticCollector", + package_name: str = "", + logger: Optional["InstallLogger"] = None, + scope: Optional["InstallScope"] = None, +) -> dict: + """Run the full integration pipeline for a single package. + + Iterates over *targets* (``TargetProfile`` list) and dispatches each + primitive to the appropriate integrator via the target-driven API. + Skills are handled separately because ``SkillIntegrator`` already + routes across all targets internally. + + When *scope* is ``InstallScope.USER``, targets and primitives that + do not support user-scope deployment are silently skipped. + + Returns a dict with integration counters and the list of deployed file paths. + """ + from apm_cli.integration.dispatch import get_dispatch_table + + _dispatch = get_dispatch_table() + result = { + "prompts": 0, + "agents": 0, + "skills": 0, + "sub_skills": 0, + "instructions": 0, + "commands": 0, + "hooks": 0, + "links_resolved": 0, + "deployed_files": [], + } + + deployed = result["deployed_files"] + + if not targets: + return result + + def _log_integration(msg): + if logger: + logger.tree_item(msg) + + _INTEGRATOR_KWARGS = { + "prompts": prompt_integrator, + "agents": agent_integrator, + "commands": command_integrator, + "instructions": instruction_integrator, + "hooks": hook_integrator, + "skills": skill_integrator, + } + + for _target in targets: + for _prim_name, _mapping in _target.primitives.items(): + _entry = _dispatch.get(_prim_name) + if not _entry or _entry.multi_target: + continue # skills handled below + + _integrator = _INTEGRATOR_KWARGS[_prim_name] + _int_result = getattr(_integrator, _entry.integrate_method)( + _target, package_info, project_root, + force=force, managed_files=managed_files, + diagnostics=diagnostics, + ) + + if _int_result.files_integrated > 0: + result[_entry.counter_key] += _int_result.files_integrated + _effective_root = _mapping.deploy_root or _target.root_dir + _deploy_dir = f"{_effective_root}/{_mapping.subdir}/" if _mapping.subdir else f"{_effective_root}/" + if _prim_name == "instructions" and _mapping.format_id in ("cursor_rules", "claude_rules"): + _label = "rule(s)" + elif _prim_name == "instructions": + _label = "instruction(s)" + elif _prim_name == "hooks": + if _target.name == "claude": + _deploy_dir = ".claude/settings.json" + elif _target.name == "cursor": + _deploy_dir = ".cursor/hooks.json" + elif _target.name == "codex": + _deploy_dir = ".codex/hooks.json" + _label = "hook(s)" + else: + _label = _prim_name + _log_integration( + f" |-- {_int_result.files_integrated} {_label} integrated -> {_deploy_dir}" + ) + result["links_resolved"] += _int_result.links_resolved + for tp in _int_result.target_paths: + deployed.append(tp.relative_to(project_root).as_posix()) + + skill_result = skill_integrator.integrate_package_skill( + package_info, project_root, + diagnostics=diagnostics, managed_files=managed_files, force=force, + targets=targets, + ) + _skill_target_dirs: set = builtins.set() + for tp in skill_result.target_paths: + rel = tp.relative_to(project_root) + if rel.parts: + _skill_target_dirs.add(rel.parts[0]) + _skill_targets = sorted(_skill_target_dirs) + _skill_target_str = ", ".join(f"{d}/skills/" for d in _skill_targets) or "skills/" + if skill_result.skill_created: + result["skills"] += 1 + _log_integration(f" |-- Skill integrated -> {_skill_target_str}") + if skill_result.sub_skills_promoted > 0: + result["sub_skills"] += skill_result.sub_skills_promoted + _log_integration(f" |-- {skill_result.sub_skills_promoted} skill(s) integrated -> {_skill_target_str}") + for tp in skill_result.target_paths: + deployed.append(tp.relative_to(project_root).as_posix()) + + return result + + +def integrate_local_content( + project_root: Path, + *, + targets: Any, + prompt_integrator: Any, + agent_integrator: Any, + skill_integrator: Any, + instruction_integrator: Any, + command_integrator: Any, + hook_integrator: Any, + force: bool, + managed_files: Any, + diagnostics: "DiagnosticCollector", + logger: Optional["InstallLogger"] = None, + scope: Optional["InstallScope"] = None, +) -> dict: + """Integrate primitives from the project's own .apm/ directory. + + This treats the project root as a synthetic package so that local + skills, instructions, agents, prompts, hooks, and commands in .apm/ + are deployed to target directories exactly like dependency primitives. + + Only .apm/ sub-directories are processed. A root-level SKILL.md is + intentionally ignored (it describes the project itself, not a + deployable skill). + + Returns a dict with integration counters and deployed file paths, + same shape as ``integrate_package_primitives()``. + """ + from ..models.apm_package import APMPackage, PackageInfo, PackageType + + local_pkg = APMPackage( + name="_local", + version="0.0.0", + package_path=project_root, + source="local", + ) + local_info = PackageInfo( + package=local_pkg, + install_path=project_root, + package_type=PackageType.APM_PACKAGE, + ) + + return integrate_package_primitives( + local_info, + project_root, + targets=targets, + prompt_integrator=prompt_integrator, + agent_integrator=agent_integrator, + skill_integrator=skill_integrator, + instruction_integrator=instruction_integrator, + command_integrator=command_integrator, + hook_integrator=hook_integrator, + force=force, + managed_files=managed_files, + diagnostics=diagnostics, + package_name="_local", + logger=logger, + scope=scope, + ) + + +# Underscore-prefixed aliases for backward compatibility with existing +# imports/patches in tests and elsewhere that use the old names. +_integrate_package_primitives = integrate_package_primitives +_integrate_local_content = integrate_local_content diff --git a/tests/unit/test_local_content_install.py b/tests/unit/test_local_content_install.py index 8887b9dd..bf1ba414 100644 --- a/tests/unit/test_local_content_install.py +++ b/tests/unit/test_local_content_install.py @@ -126,7 +126,7 @@ def test_apm_dir_only_unknown_subdirs(self, tmp_path): class TestIntegrateLocalContent: """Tests for the _integrate_local_content() helper.""" - @patch("apm_cli.commands.install._integrate_package_primitives") + @patch("apm_cli.install.services.integrate_package_primitives") def test_integrates_instructions(self, mock_integrate, tmp_path): """Instructions file in .apm/ is counted in the result.""" mock_integrate.return_value = _zero_counters( @@ -139,7 +139,7 @@ def test_integrates_instructions(self, mock_integrate, tmp_path): assert result["instructions"] == 1 assert ".github/instructions/coding.instructions.md" in result["deployed_files"] - @patch("apm_cli.commands.install._integrate_package_primitives") + @patch("apm_cli.install.services.integrate_package_primitives") def test_integrates_agents(self, mock_integrate, tmp_path): """Agent file in .apm/ is counted in the result.""" mock_integrate.return_value = _zero_counters( @@ -152,7 +152,7 @@ def test_integrates_agents(self, mock_integrate, tmp_path): assert result["agents"] == 1 assert ".github/agents/backend.agent.md" in result["deployed_files"] - @patch("apm_cli.commands.install._integrate_package_primitives") + @patch("apm_cli.install.services.integrate_package_primitives") def test_skips_root_skill_md(self, mock_integrate, tmp_path): """A root SKILL.md must NOT be deployed (package_type=APM_PACKAGE prevents it). @@ -172,7 +172,7 @@ def test_skips_root_skill_md(self, mock_integrate, tmp_path): package_info = mock_integrate.call_args[0][0] assert package_info.package_type == PackageType.APM_PACKAGE - @patch("apm_cli.commands.install._integrate_package_primitives") + @patch("apm_cli.install.services.integrate_package_primitives") def test_package_info_install_path_is_project_root(self, mock_integrate, tmp_path): """The synthetic PackageInfo must point to project_root, not .apm/.""" mock_integrate.return_value = _zero_counters() @@ -182,7 +182,7 @@ def test_package_info_install_path_is_project_root(self, mock_integrate, tmp_pat package_info = mock_integrate.call_args[0][0] assert package_info.install_path == tmp_path - @patch("apm_cli.commands.install._integrate_package_primitives") + @patch("apm_cli.install.services.integrate_package_primitives") def test_returns_zero_counters_when_nothing_deployed(self, mock_integrate, tmp_path): """When nothing is deployed the result counters are all zero.""" mock_integrate.return_value = _zero_counters() From cbba004b07e11c051ebeb71eb86667a4a54a02fc Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 14:23:31 +0200 Subject: [PATCH 22/28] refactor(install): P1.2 -- replace _install_mod indirection with direct imports Eliminates the legacy module-attribute indirection `from apm_cli.commands import install as _install_mod` inside `install/phases/integrate.py`. Collaborators are now imported directly from their canonical locations: - integrate_package_primitives, integrate_local_content from apm_cli.install.services - _pre_deploy_security_scan from apm_cli.install.helpers.security_scan - _copy_local_package from apm_cli.install.phases.local_content - _rich_success, _rich_error from apm_cli.utils.console The four per-source helpers (_integrate_local_dep, _integrate_cached_dep, _integrate_fresh_dep, _integrate_root_project) no longer take an `_install_mod: Any` parameter, and the four matching call sites in `run()` are simplified accordingly. The 12 `_install_mod.X` references are replaced by direct names. This is the DI seam that subsequent phases (Strategy + Application Service) will consume. The 5 healthy test patches that previously needed `_install_mod` indirection now point at `apm_cli.install.services.integrate_package_primitives` (updated in P1.1). All 3974 unit tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/phases/integrate.py | 68 ++++++++++--------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index b7f9ac7c..72783558 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -6,25 +6,17 @@ 1. Builds a ``PackageInfo`` (or reuses the pre-downloaded result). 2. Runs the pre-deploy security scan. -3. Calls ``_integrate_package_primitives`` (via module-attribute access on - ``apm_cli.commands.install`` so that test patches at - ``@patch("apm_cli.commands.install._integrate_package_primitives")`` - continue to intercept the call). +3. Calls ``integrate_package_primitives`` from + ``apm_cli.install.services`` (the integration template). 4. Accumulates deployed-file lists, content hashes, and integration totals on *ctx* for the downstream cleanup and lockfile phases. After the dependency loop, root-project primitives (``/.apm/``) are integrated when present (#714). -**Test-patch contract**: every name that tests patch at -``apm_cli.commands.install.X`` is accessed via the ``_install_mod.X`` -indirection rather than a bare-name import. This includes at minimum: -``_integrate_package_primitives``, ``_rich_success``, ``_rich_error``, -``_copy_local_package``, ``_pre_deploy_security_scan``. All five private -helpers in this module (``_resolve_download_strategy``, -``_integrate_local_dep``, ``_integrate_cached_dep``, -``_integrate_fresh_dep``, ``_integrate_root_project``) honour this -contract via the ``_install_mod`` parameter. +Direct imports (no module-attribute indirection): collaborators are imported +from the install package itself, eliminating the legacy ``_install_mod`` +shim that previously routed through ``apm_cli.commands.install``. """ from __future__ import annotations @@ -34,6 +26,14 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple +from apm_cli.install.helpers.security_scan import _pre_deploy_security_scan +from apm_cli.install.phases.local_content import _copy_local_package +from apm_cli.install.services import ( + integrate_local_content, + integrate_package_primitives, +) +from apm_cli.utils.console import _rich_error, _rich_success + if TYPE_CHECKING: from apm_cli.install.context import InstallContext @@ -168,7 +168,6 @@ def _resolve_download_strategy( def _integrate_local_dep( ctx: "InstallContext", - _install_mod: Any, dep_ref: Any, install_path: Path, dep_key: str, @@ -201,7 +200,7 @@ def _integrate_local_dep( ) return None - result_path = _install_mod._copy_local_package(dep_ref, install_path, ctx.project_root, logger=logger) + result_path = _copy_local_package(dep_ref, install_path, ctx.project_root, logger=logger) if not result_path: diagnostics.error( f"Failed to copy local package: {dep_ref.local_path}", @@ -285,7 +284,7 @@ def _integrate_local_dep( # Run shared integration pipeline try: # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( + if not _pre_deploy_security_scan( install_path, diagnostics, package_name=dep_key, force=ctx.force, logger=logger, @@ -293,7 +292,7 @@ def _integrate_local_dep( ctx.package_deployed_files[dep_key] = [] return deltas - int_result = _install_mod._integrate_package_primitives( + int_result = integrate_package_primitives( package_info, ctx.project_root, targets=ctx.targets, prompt_integrator=ctx.integrators["prompt"], @@ -342,7 +341,6 @@ def _integrate_local_dep( def _integrate_cached_dep( ctx: "InstallContext", - _install_mod: Any, dep_ref: Any, install_path: Path, dep_key: str, @@ -469,7 +467,7 @@ def _integrate_cached_dep( ctx.package_types[dep_key] = cached_package_info.package_type.value # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( + if not _pre_deploy_security_scan( install_path, diagnostics, package_name=dep_key, force=ctx.force, logger=logger, @@ -477,7 +475,7 @@ def _integrate_cached_dep( ctx.package_deployed_files[dep_key] = [] return deltas - int_result = _install_mod._integrate_package_primitives( + int_result = integrate_package_primitives( cached_package_info, ctx.project_root, targets=ctx.targets, prompt_integrator=ctx.integrators["prompt"], @@ -525,7 +523,6 @@ def _integrate_cached_dep( def _integrate_fresh_dep( ctx: "InstallContext", - _install_mod: Any, dep_ref: Any, install_path: Path, dep_key: str, @@ -617,7 +614,7 @@ def _integrate_fresh_dep( _ref_suffix = f" #{_r}" elif _s: _ref_suffix = f" @{_s}" - _install_mod._rich_success(f"[+] {display_name}{_ref_suffix}") + _rich_success(f"[+] {display_name}{_ref_suffix}") # Track unpinned deps for aggregated diagnostic if not dep_ref.reference: @@ -653,7 +650,7 @@ def _integrate_fresh_dep( _fresh_hash = ctx.package_hashes[dep_ref.get_unique_key()] if _fresh_hash != dep_locked_chk.content_hash: safe_rmtree(install_path, ctx.apm_modules_dir) - _install_mod._rich_error( + _rich_error( f"Content hash mismatch for " f"{dep_ref.get_unique_key()}: " f"expected {dep_locked_chk.content_hash}, " @@ -686,7 +683,7 @@ def _integrate_fresh_dep( # Auto-integrate prompts and agents if enabled # Pre-deploy security gate - if not _install_mod._pre_deploy_security_scan( + if not _pre_deploy_security_scan( package_info.install_path, diagnostics, package_name=dep_ref.get_unique_key(), force=ctx.force, logger=logger, @@ -696,7 +693,7 @@ def _integrate_fresh_dep( if ctx.targets: try: - int_result = _install_mod._integrate_package_primitives( + int_result = integrate_package_primitives( package_info, ctx.project_root, targets=ctx.targets, prompt_integrator=ctx.integrators["prompt"], @@ -760,7 +757,6 @@ def _integrate_fresh_dep( def _integrate_root_project( ctx: "InstallContext", - _install_mod: Any, ) -> Optional[Dict[str, int]]: """Integrate root project's own .apm/ primitives (#714). @@ -770,7 +766,7 @@ def _integrate_root_project( found in /.apm/ are integrated after all declared dependency packages have been processed. - Delegates to ``_install_mod._integrate_local_content`` which creates a + Delegates to ``integrate_local_content`` which creates a synthetic ``_local`` APMPackage with ``PackageType.APM_PACKAGE`` so that a root-level ``SKILL.md`` is NOT deployed as a skill. Deployed files are tracked on ``ctx.local_deployed_files`` for the downstream @@ -806,7 +802,7 @@ def _integrate_root_project( logger.download_complete("", ref_suffix="local") logger.verbose_detail("Integrating local .apm/ content...") try: - _root_result = _install_mod._integrate_local_content( + _root_result = integrate_local_content( ctx.project_root, targets=ctx.targets, prompt_integrator=ctx.integrators["prompt"], @@ -879,14 +875,6 @@ def run(ctx: "InstallContext") -> None: ``total_instructions_integrated``, ``total_commands_integrated``, ``total_hooks_integrated``, ``total_links_resolved``. """ - # ------------------------------------------------------------------ - # Module-attribute access for late-patchability. - # Tests patch names at apm_cli.commands.install.X -- importing the - # MODULE (not the name) ensures the patched attribute is resolved at - # call time. - # ------------------------------------------------------------------ - from apm_cli.commands import install as _install_mod - from rich.progress import ( BarColumn, Progress, @@ -950,7 +938,7 @@ def run(ctx: "InstallContext") -> None: # --- Dispatch to per-source helper --- if dep_ref.is_local and dep_ref.local_path: deltas = _integrate_local_dep( - ctx, _install_mod, dep_ref, install_path, dep_key, + ctx, dep_ref, install_path, dep_key, ) else: resolved_ref, skip_download, dep_locked_chk, ref_changed = ( @@ -958,12 +946,12 @@ def run(ctx: "InstallContext") -> None: ) if skip_download: deltas = _integrate_cached_dep( - ctx, _install_mod, dep_ref, install_path, dep_key, + ctx, dep_ref, install_path, dep_key, resolved_ref, dep_locked_chk, ) else: deltas = _integrate_fresh_dep( - ctx, _install_mod, dep_ref, install_path, dep_key, + ctx, dep_ref, install_path, dep_key, resolved_ref, dep_locked_chk, ref_changed, progress, ) @@ -985,7 +973,7 @@ def run(ctx: "InstallContext") -> None: # ------------------------------------------------------------------ # Integrate root project's own .apm/ primitives (#714). # ------------------------------------------------------------------ - root_deltas = _integrate_root_project(ctx, _install_mod) + root_deltas = _integrate_root_project(ctx) if root_deltas: installed_count += root_deltas.get("installed", 0) total_prompts_integrated += root_deltas.get("prompts", 0) From 9c920c696b3e198cfea9a9352a573f69ee0af1b5 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 14:57:49 +0200 Subject: [PATCH 23/28] refactor(install): P2 -- introduce DependencySource Strategy + integration Template Method Replaces the three per-source helpers in `install/phases/integrate.py` (`_integrate_local_dep`, `_integrate_cached_dep`, `_integrate_fresh_dep`, ~600 LOC of overlapping code) with two new modules: - `install/sources.py` -- `DependencySource` ABC + three concrete Strategy implementations (`LocalDependencySource`, `CachedDependencySource`, `FreshDependencySource`) plus a `make_dependency_source(...)` factory. Each source encapsulates acquisition (copy / cache reuse / network download with supply-chain hash verification) and returns a `Materialization` describing the prepared package. - `install/template.py` -- `run_integration_template(source)` Template Method. After `acquire()`, every source funnels through the same flow: pre-deploy security scan, primitive integration, deployed-files tracking, per-package verbose diagnostics. Root-project integration (`/.apm/`) remains a sibling helper (`_integrate_root_project`) because its shape is structurally distinct (no `PackageInfo`, dedicated `ctx.local_deployed_files` tracking, different downstream cleanup semantics). `integrate.py:run()` now reads as the orchestration it always was: build the right source, run the template, accumulate counters. Module LOC drops from 1001 to 402, and the `KNOWN_LARGE_MODULES` exception is removed -- integrate.py is now well under the default 1000-LOC budget. All 3974 unit tests pass; no behavioural changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/phases/integrate.py | 645 +----------------- src/apm_cli/install/sources.py | 572 ++++++++++++++++ src/apm_cli/install/template.py | 146 ++++ .../install/test_architecture_invariants.py | 7 +- 4 files changed, 744 insertions(+), 626 deletions(-) create mode 100644 src/apm_cli/install/sources.py create mode 100644 src/apm_cli/install/template.py diff --git a/src/apm_cli/install/phases/integrate.py b/src/apm_cli/install/phases/integrate.py index 72783558..329099f9 100644 --- a/src/apm_cli/install/phases/integrate.py +++ b/src/apm_cli/install/phases/integrate.py @@ -1,38 +1,27 @@ """Sequential integration phase -- per-package integration loop. Reads all prior phase outputs from *ctx* (resolve, targets, download) and -processes each dependency sequentially: local-copy packages, cached packages, -and freshly-downloaded packages. For every package the loop: - -1. Builds a ``PackageInfo`` (or reuses the pre-downloaded result). -2. Runs the pre-deploy security scan. -3. Calls ``integrate_package_primitives`` from - ``apm_cli.install.services`` (the integration template). -4. Accumulates deployed-file lists, content hashes, and integration totals - on *ctx* for the downstream cleanup and lockfile phases. +processes each dependency sequentially. Per-source acquisition is handled +by ``DependencySource`` Strategy implementations +(``apm_cli.install.sources``); the shared post-acquire flow (security gate ++ primitive integration + diagnostics) lives in the Template Method +``apm_cli.install.template.run_integration_template``. After the dependency loop, root-project primitives (``/.apm/``) -are integrated when present (#714). - -Direct imports (no module-attribute indirection): collaborators are imported -from the install package itself, eliminating the legacy ``_install_mod`` -shim that previously routed through ``apm_cli.commands.install``. +are integrated when present (#714) -- this path is structurally distinct +(no ``PackageInfo``, dedicated ``ctx.local_deployed_files`` tracking) so it +remains a sibling helper here rather than a fourth ``DependencySource``. """ from __future__ import annotations import builtins -import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple -from apm_cli.install.helpers.security_scan import _pre_deploy_security_scan -from apm_cli.install.phases.local_content import _copy_local_package -from apm_cli.install.services import ( - integrate_local_content, - integrate_package_primitives, -) -from apm_cli.utils.console import _rich_error, _rich_success +from apm_cli.install.services import integrate_local_content +from apm_cli.install.sources import make_dependency_source +from apm_cli.install.template import run_integration_template if TYPE_CHECKING: from apm_cli.install.context import InstallContext @@ -166,594 +155,6 @@ def _resolve_download_strategy( return resolved_ref, skip_download, _dep_locked_chk, ref_changed -def _integrate_local_dep( - ctx: "InstallContext", - dep_ref: Any, - install_path: Path, - dep_key: str, -) -> Optional[Dict[str, int]]: - """Integrate a local (filesystem) package. - - Returns a counter-delta dict, or ``None`` if the dependency was - skipped (user scope, copy failure). - """ - from apm_cli.core.scope import InstallScope - from apm_cli.utils.content_hash import compute_package_hash as _compute_hash - - diagnostics = ctx.diagnostics - logger = ctx.logger - - # User scope: relative paths would resolve against $HOME - # instead of cwd, producing wrong results. Skip with a - # clear diagnostic rather than silently failing. - if ctx.scope is InstallScope.USER: - diagnostics.warn( - f"Skipped local package '{dep_ref.local_path}' " - "-- local paths are not supported at user scope (--global). " - "Use a remote reference (owner/repo) instead.", - package=dep_ref.local_path, - ) - if logger: - logger.verbose_detail( - f" Skipping {dep_ref.local_path} (local packages " - "resolve against cwd, not $HOME)" - ) - return None - - result_path = _copy_local_package(dep_ref, install_path, ctx.project_root, logger=logger) - if not result_path: - diagnostics.error( - f"Failed to copy local package: {dep_ref.local_path}", - package=dep_ref.local_path, - ) - return None - - deltas: Dict[str, int] = {"installed": 1} - if logger: - logger.download_complete(dep_ref.local_path, ref_suffix="local") - - # Build minimal PackageInfo for integration - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, - ) - from datetime import datetime - - local_apm_yml = install_path / "apm.yml" - if local_apm_yml.exists(): - local_pkg = APMPackage.from_apm_yml(local_apm_yml) - if not local_pkg.source: - local_pkg.source = dep_ref.local_path - else: - local_pkg = APMPackage( - name=Path(dep_ref.local_path).name, - version="0.0.0", - package_path=install_path, - source=dep_ref.local_path, - ) - - local_ref = ResolvedReference( - original_ref="local", - ref_type=GitReferenceType.BRANCH, - resolved_commit="local", - ref_name="local", - ) - local_info = PackageInfo( - package=local_pkg, - install_path=install_path, - resolved_reference=local_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, - ) - - # Detect package type - from apm_cli.models.validation import detect_package_type - pkg_type, plugin_json_path = detect_package_type(install_path) - local_info.package_type = pkg_type - if pkg_type == PackageType.MARKETPLACE_PLUGIN: - # Normalize: synthesize .apm/ from plugin.json so - # integration can discover and deploy primitives - from apm_cli.deps.plugin_parser import normalize_plugin_directory - normalize_plugin_directory(install_path, plugin_json_path) - - # Record for lockfile - from apm_cli.deps.installed_package import InstalledPackage - node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - ctx.installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=None, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=None, # local deps never go through registry - )) - dep_key = dep_ref.get_unique_key() - if install_path.is_dir() and not dep_ref.is_local: - ctx.package_hashes[dep_key] = _compute_hash(install_path) - dep_deployed_files: builtins.list = [] - - if hasattr(local_info, 'package_type') and local_info.package_type: - ctx.package_types[dep_key] = local_info.package_type.value - - # Use the same variable name as the rest of the loop - package_info = local_info - - # Run shared integration pipeline - try: - # Pre-deploy security gate - if not _pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=ctx.force, - logger=logger, - ): - ctx.package_deployed_files[dep_key] = [] - return deltas - - int_result = integrate_package_primitives( - package_info, ctx.project_root, - targets=ctx.targets, - prompt_integrator=ctx.integrators["prompt"], - agent_integrator=ctx.integrators["agent"], - skill_integrator=ctx.integrators["skill"], - instruction_integrator=ctx.integrators["instruction"], - command_integrator=ctx.integrators["command"], - hook_integrator=ctx.integrators["hook"], - force=ctx.force, - managed_files=ctx.managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=ctx.scope, - ) - deltas["prompts"] = int_result["prompts"] - deltas["agents"] = int_result["agents"] - deltas["skills"] = int_result["skills"] - deltas["sub_skills"] = int_result["sub_skills"] - deltas["instructions"] = int_result["instructions"] - deltas["commands"] = int_result["commands"] - deltas["hooks"] = int_result["hooks"] - deltas["links_resolved"] = int_result["links_resolved"] - dep_deployed_files.extend(int_result["deployed_files"]) - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from local package: {e}", - package=dep_ref.local_path, - ) - - ctx.package_deployed_files[dep_key] = dep_deployed_files - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - return deltas - - -def _integrate_cached_dep( - ctx: "InstallContext", - dep_ref: Any, - install_path: Path, - dep_key: str, - resolved_ref: Any, - dep_locked_chk: Any, -) -> Optional[Dict[str, int]]: - """Integrate a cached (already-downloaded) package. - - Returns a counter-delta dict. - """ - from apm_cli.constants import APM_YML_FILENAME - from apm_cli.utils.content_hash import compute_package_hash as _compute_hash - - logger = ctx.logger - diagnostics = ctx.diagnostics - - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - # Show resolved ref from lockfile for consistency with fresh installs - _ref = dep_ref.reference or "" - _sha = "" - if dep_locked_chk and dep_locked_chk.resolved_commit and dep_locked_chk.resolved_commit != "cached": - _sha = dep_locked_chk.resolved_commit[:8] - if logger: - logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) - - deltas: Dict[str, int] = {"installed": 1} - if not dep_ref.reference: - deltas["unpinned"] = 1 - - # Skip integration if not needed - if not ctx.targets: - return deltas - - # Integrate prompts for cached packages (zero-config behavior) - try: - # Create PackageInfo from cached package - from apm_cli.models.apm_package import ( - APMPackage, - PackageInfo, - PackageType, - ResolvedReference, - GitReferenceType, - ) - from datetime import datetime - - # Load package from apm.yml in install path - apm_yml_path = install_path / APM_YML_FILENAME - if apm_yml_path.exists(): - cached_package = APMPackage.from_apm_yml(apm_yml_path) - # Ensure source is set to the repo URL for sync matching - if not cached_package.source: - cached_package.source = dep_ref.repo_url - else: - # Virtual package or no apm.yml - create minimal package - cached_package = APMPackage( - name=dep_ref.repo_url.split("/")[-1], - version="unknown", - package_path=install_path, - source=dep_ref.repo_url, - ) - - # Use resolved reference from ref resolution if available - # (e.g. when update_refs matched the lockfile SHA), - # otherwise create a placeholder for cached packages. - resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( - original_ref=dep_ref.reference or "default", - ref_type=GitReferenceType.BRANCH, - resolved_commit="cached", # Mark as cached since we don't know exact commit - ref_name=dep_ref.reference or "default", - ) - - cached_package_info = PackageInfo( - package=cached_package, - install_path=install_path, - resolved_reference=resolved_or_cached_ref, - installed_at=datetime.now().isoformat(), - dependency_ref=dep_ref, # Store for canonical dependency string - ) - - # Detect package_type from disk contents so - # skill integration is not silently skipped - from apm_cli.models.validation import detect_package_type - pkg_type, _ = detect_package_type(install_path) - cached_package_info.package_type = pkg_type - - # Collect for lockfile (cached packages still need to be tracked) - from apm_cli.deps.installed_package import InstalledPackage - node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - # Get commit SHA: resolved ref > callback capture > existing lockfile > explicit reference - dep_key = dep_ref.get_unique_key() - cached_commit = None - if resolved_ref and resolved_ref.resolved_commit and resolved_ref.resolved_commit != "cached": - cached_commit = resolved_ref.resolved_commit - if not cached_commit: - cached_commit = ctx.callback_downloaded.get(dep_key) - if not cached_commit and ctx.existing_lockfile: - locked_dep = ctx.existing_lockfile.get_dependency(dep_key) - if locked_dep: - cached_commit = locked_dep.resolved_commit - if not cached_commit: - cached_commit = dep_ref.reference - # Determine if the cached package came from the registry: - # prefer the lockfile record, then the current registry config. - _cached_registry = None - if dep_locked_chk and dep_locked_chk.registry_prefix: - # Reconstruct RegistryConfig from lockfile to preserve original source - _cached_registry = ctx.registry_config - elif ctx.registry_config and not dep_ref.is_local: - _cached_registry = ctx.registry_config - ctx.installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=cached_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=_cached_registry, - )) - if install_path.is_dir(): - ctx.package_hashes[dep_key] = _compute_hash(install_path) - # Track package type for lockfile - if hasattr(cached_package_info, 'package_type') and cached_package_info.package_type: - ctx.package_types[dep_key] = cached_package_info.package_type.value - - # Pre-deploy security gate - if not _pre_deploy_security_scan( - install_path, diagnostics, - package_name=dep_key, force=ctx.force, - logger=logger, - ): - ctx.package_deployed_files[dep_key] = [] - return deltas - - int_result = integrate_package_primitives( - cached_package_info, ctx.project_root, - targets=ctx.targets, - prompt_integrator=ctx.integrators["prompt"], - agent_integrator=ctx.integrators["agent"], - skill_integrator=ctx.integrators["skill"], - instruction_integrator=ctx.integrators["instruction"], - command_integrator=ctx.integrators["command"], - hook_integrator=ctx.integrators["hook"], - force=ctx.force, - managed_files=ctx.managed_files, - diagnostics=diagnostics, - package_name=dep_key, - logger=logger, - scope=ctx.scope, - ) - deltas["prompts"] = int_result["prompts"] - deltas["agents"] = int_result["agents"] - deltas["skills"] = int_result["skills"] - deltas["sub_skills"] = int_result["sub_skills"] - deltas["instructions"] = int_result["instructions"] - deltas["commands"] = int_result["commands"] - deltas["hooks"] = int_result["hooks"] - deltas["links_resolved"] = int_result["links_resolved"] - dep_deployed = int_result["deployed_files"] - ctx.package_deployed_files[dep_key] = dep_deployed - except Exception as e: - diagnostics.error( - f"Failed to integrate primitives from cached package: {e}", - package=dep_key, - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - _skip_count = diagnostics.count_for_package(dep_key, "collision") - _err_count = diagnostics.count_for_package(dep_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - return deltas - - -def _integrate_fresh_dep( - ctx: "InstallContext", - dep_ref: Any, - install_path: Path, - dep_key: str, - resolved_ref: Any, - dep_locked_chk: Any, - ref_changed: bool, - progress: Any, -) -> Optional[Dict[str, int]]: - """Download and integrate a fresh (not cached) package. - - Returns a counter-delta dict, or ``None`` if the download failed. - """ - from apm_cli.drift import build_download_ref - from apm_cli.deps.installed_package import InstalledPackage - from apm_cli.utils.content_hash import compute_package_hash as _compute_hash - from apm_cli.utils.path_security import safe_rmtree - - diagnostics = ctx.diagnostics - logger = ctx.logger - - # Download the package with progress feedback - try: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - short_name = ( - display_name.split("/")[-1] - if "/" in display_name - else display_name - ) - - # Create a progress task for this download - task_id = progress.add_task( - description=f"Fetching {short_name}", - total=None, # Indeterminate initially; git will update with actual counts - ) - - # T5: Build download ref - use locked commit if available. - # build_download_ref() uses manifest ref when ref_changed is True. - download_ref = build_download_ref( - dep_ref, ctx.existing_lockfile, update_refs=ctx.update_refs, ref_changed=ref_changed - ) - - # Phase 4 (#171): Use pre-downloaded result if available - _dep_key = dep_ref.get_unique_key() - if _dep_key in ctx.pre_download_results: - package_info = ctx.pre_download_results[_dep_key] - else: - # Fallback: sequential download (should rarely happen) - package_info = ctx.downloader.download_package( - download_ref, - install_path, - progress_task_id=task_id, - progress_obj=progress, - ) - - # CRITICAL: Hide progress BEFORE printing success message to avoid overlap - progress.update(task_id, visible=False) - progress.refresh() # Force immediate refresh to hide the bar - - deltas: Dict[str, int] = {"installed": 1} - - # Show resolved ref alongside package name for visibility - resolved = getattr(package_info, 'resolved_reference', None) - if logger: - _ref = "" - _sha = "" - if resolved: - _ref = resolved.ref_name if resolved.ref_name else "" - _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - logger.download_complete(display_name, ref=_ref, sha=_sha) - # Log auth source for this download (verbose only) - if ctx.auth_resolver: - try: - _host = dep_ref.host or "github.com" - _org = dep_ref.repo_url.split('/')[0] if dep_ref.repo_url and '/' in dep_ref.repo_url else None - _ctx = ctx.auth_resolver.resolve(_host, org=_org) - logger.package_auth(_ctx.source, _ctx.token_type or "none") - except Exception: - pass - else: - _ref_suffix = "" - if resolved: - _r = resolved.ref_name if resolved.ref_name else "" - _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" - if _r and _s: - _ref_suffix = f" #{_r} @{_s}" - elif _r: - _ref_suffix = f" #{_r}" - elif _s: - _ref_suffix = f" @{_s}" - _rich_success(f"[+] {display_name}{_ref_suffix}") - - # Track unpinned deps for aggregated diagnostic - if not dep_ref.reference: - deltas["unpinned"] = 1 - - # Collect for lockfile: get resolved commit and depth - resolved_commit = None - if resolved: - resolved_commit = package_info.resolved_reference.resolved_commit - # Get depth from dependency tree - node = ctx.dependency_graph.dependency_tree.get_node(dep_ref.get_unique_key()) - depth = node.depth if node else 1 - resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None - _is_dev = node.is_dev if node else False - ctx.installed_packages.append(InstalledPackage( - dep_ref=dep_ref, resolved_commit=resolved_commit, - depth=depth, resolved_by=resolved_by, is_dev=_is_dev, - registry_config=ctx.registry_config if not dep_ref.is_local else None, - )) - if install_path.is_dir(): - ctx.package_hashes[dep_ref.get_unique_key()] = _compute_hash(install_path) - - # Supply chain protection: verify content hash on fresh - # downloads when the lockfile already records a hash. - # A mismatch means the downloaded content differs from - # what was previously locked -- possible tampering. - if ( - not ctx.update_refs - and dep_locked_chk - and dep_locked_chk.content_hash - and dep_ref.get_unique_key() in ctx.package_hashes - ): - _fresh_hash = ctx.package_hashes[dep_ref.get_unique_key()] - if _fresh_hash != dep_locked_chk.content_hash: - safe_rmtree(install_path, ctx.apm_modules_dir) - _rich_error( - f"Content hash mismatch for " - f"{dep_ref.get_unique_key()}: " - f"expected {dep_locked_chk.content_hash}, " - f"got {_fresh_hash}. " - "The downloaded content differs from the " - "lockfile record. This may indicate a " - "supply-chain attack. Use 'apm install " - "--update' to accept new content and " - "update the lockfile." - ) - sys.exit(1) - - # Track package type for lockfile - if hasattr(package_info, 'package_type') and package_info.package_type: - ctx.package_types[dep_ref.get_unique_key()] = package_info.package_type.value - - # Show package type in verbose mode - if hasattr(package_info, "package_type"): - from apm_cli.models.apm_package import PackageType - - package_type = package_info.package_type - _type_label = { - PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", - PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", - PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", - PackageType.APM_PACKAGE: "APM Package (apm.yml)", - }.get(package_type) - if _type_label and logger: - logger.package_type_info(_type_label) - - # Auto-integrate prompts and agents if enabled - # Pre-deploy security gate - if not _pre_deploy_security_scan( - package_info.install_path, diagnostics, - package_name=dep_ref.get_unique_key(), force=ctx.force, - logger=logger, - ): - ctx.package_deployed_files[dep_ref.get_unique_key()] = [] - return deltas - - if ctx.targets: - try: - int_result = integrate_package_primitives( - package_info, ctx.project_root, - targets=ctx.targets, - prompt_integrator=ctx.integrators["prompt"], - agent_integrator=ctx.integrators["agent"], - skill_integrator=ctx.integrators["skill"], - instruction_integrator=ctx.integrators["instruction"], - command_integrator=ctx.integrators["command"], - hook_integrator=ctx.integrators["hook"], - force=ctx.force, - managed_files=ctx.managed_files, - diagnostics=diagnostics, - package_name=dep_ref.get_unique_key(), - logger=logger, - scope=ctx.scope, - ) - deltas["prompts"] = int_result["prompts"] - deltas["agents"] = int_result["agents"] - deltas["skills"] = int_result["skills"] - deltas["sub_skills"] = int_result["sub_skills"] - deltas["instructions"] = int_result["instructions"] - deltas["commands"] = int_result["commands"] - deltas["hooks"] = int_result["hooks"] - deltas["links_resolved"] = int_result["links_resolved"] - dep_deployed_fresh = int_result["deployed_files"] - ctx.package_deployed_files[dep_ref.get_unique_key()] = dep_deployed_fresh - except Exception as e: - # Don't fail installation if integration fails - diagnostics.error( - f"Failed to integrate primitives: {e}", - package=dep_ref.get_unique_key(), - ) - - # In verbose mode, show inline skip/error count for this package - if logger and logger.verbose: - pkg_key = dep_ref.get_unique_key() - _skip_count = diagnostics.count_for_package(pkg_key, "collision") - _err_count = diagnostics.count_for_package(pkg_key, "error") - if _skip_count > 0: - noun = "file" if _skip_count == 1 else "files" - logger.package_inline_warning(f" [!] {_skip_count} {noun} skipped (local files exist)") - if _err_count > 0: - noun = "error" if _err_count == 1 else "errors" - logger.package_inline_warning(f" [!] {_err_count} integration {noun}") - - return deltas - - except Exception as e: - display_name = ( - str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url - ) - # Remove the progress task on error - if "task_id" in locals(): - progress.remove_task(task_id) - diagnostics.error( - f"Failed to install {display_name}: {e}", - package=dep_ref.get_unique_key(), - ) - # Continue with other packages instead of failing completely - return None - def _integrate_root_project( ctx: "InstallContext", @@ -935,25 +336,25 @@ def run(ctx: "InstallContext") -> None: ctx.logger.verbose_detail(f" Skipping {dep_key} (already failed during resolution)") continue - # --- Dispatch to per-source helper --- + # --- Build the right DependencySource and run the template --- if dep_ref.is_local and dep_ref.local_path: - deltas = _integrate_local_dep( + source = make_dependency_source( ctx, dep_ref, install_path, dep_key, ) else: resolved_ref, skip_download, dep_locked_chk, ref_changed = ( _resolve_download_strategy(ctx, dep_ref, install_path) ) - if skip_download: - deltas = _integrate_cached_dep( - ctx, dep_ref, install_path, dep_key, - resolved_ref, dep_locked_chk, - ) - else: - deltas = _integrate_fresh_dep( - ctx, dep_ref, install_path, dep_key, - resolved_ref, dep_locked_chk, ref_changed, progress, - ) + source = make_dependency_source( + ctx, dep_ref, install_path, dep_key, + resolved_ref=resolved_ref, + dep_locked_chk=dep_locked_chk, + ref_changed=ref_changed, + skip_download=skip_download, + progress=progress, + ) + + deltas = run_integration_template(source) if deltas is None: continue diff --git a/src/apm_cli/install/sources.py b/src/apm_cli/install/sources.py new file mode 100644 index 00000000..35f88281 --- /dev/null +++ b/src/apm_cli/install/sources.py @@ -0,0 +1,572 @@ +"""Dependency sources -- Strategy pattern for the install pipeline. + +Each ``DependencySource`` knows how to *acquire* one dependency: bring its +files onto disk, build a ``PackageInfo``, register it in the lockfile-bound +state, and return the metadata the integration template needs. + +After ``acquire()``, all sources flow through the same template +(``apm_cli.install.template.run_integration_template``) which handles the +security gate, primitive integration, and per-package diagnostics. + +This module deliberately contains *only* source-specific logic. Anything +shared across sources lives in the template. + +Sources +------- +- ``LocalDependencySource``: ``file://`` deps copied from the workspace. +- ``CachedDependencySource``: deps already extracted in ``apm_modules/``. +- ``FreshDependencySource``: deps that need a network download (with + supply-chain hash verification on top of the existing lockfile entry). + +The root-project integration (``/.apm/``) follows a +substantially different shape (no PackageInfo, dedicated tracking on +``ctx.local_deployed_files``) and is handled separately in +``phases/integrate.py``. +""" + +from __future__ import annotations + +import sys +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Optional + +from apm_cli.utils.console import _rich_error, _rich_success + +if TYPE_CHECKING: + from apm_cli.install.context import InstallContext + from apm_cli.models.apm_package import PackageInfo + + +@dataclass +class Materialization: + """Outcome of ``DependencySource.acquire()``. + + Carries everything the integration template needs to run the security + gate + primitive integration on a freshly-acquired package. + """ + + package_info: "PackageInfo" + install_path: Path + dep_key: str + deltas: Dict[str, int] = field(default_factory=lambda: {"installed": 1}) + + +class DependencySource(ABC): + """Strategy: acquire one dependency and prepare it for integration. + + Subclasses encapsulate source-specific concerns (filesystem copy, + cache reuse, fresh download with progress + hash verification). + The post-acquire template flow is the same for every source. + """ + + def __init__( + self, + ctx: "InstallContext", + dep_ref: Any, + install_path: Path, + dep_key: str, + ): + self.ctx = ctx + self.dep_ref = dep_ref + self.install_path = install_path + self.dep_key = dep_key + + @abstractmethod + def acquire(self) -> Optional[Materialization]: + """Materialise the dependency on disk and build PackageInfo. + + Returns ``None`` to skip integration entirely (e.g. local dep at + user scope, copy/download failure). Otherwise returns a + ``Materialization`` consumed by the integration template. + """ + + +class LocalDependencySource(DependencySource): + """Local (``file://``) dependency: copy from a filesystem path.""" + + def acquire(self) -> Optional[Materialization]: + from apm_cli.core.scope import InstallScope + from apm_cli.deps.installed_package import InstalledPackage + from apm_cli.install.phases.local_content import _copy_local_package + from apm_cli.models.apm_package import ( + APMPackage, + GitReferenceType, + PackageInfo, + PackageType, + ResolvedReference, + ) + from apm_cli.models.validation import detect_package_type + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + + ctx = self.ctx + dep_ref = self.dep_ref + install_path = self.install_path + dep_key = self.dep_key + diagnostics = ctx.diagnostics + logger = ctx.logger + + # User scope: relative paths would resolve against $HOME instead + # of cwd, producing wrong results. Skip with a clear diagnostic. + if ctx.scope is InstallScope.USER: + diagnostics.warn( + f"Skipped local package '{dep_ref.local_path}' " + "-- local paths are not supported at user scope (--global). " + "Use a remote reference (owner/repo) instead.", + package=dep_ref.local_path, + ) + if logger: + logger.verbose_detail( + f" Skipping {dep_ref.local_path} (local packages " + "resolve against cwd, not $HOME)" + ) + return None + + result_path = _copy_local_package( + dep_ref, install_path, ctx.project_root, logger=logger + ) + if not result_path: + diagnostics.error( + f"Failed to copy local package: {dep_ref.local_path}", + package=dep_ref.local_path, + ) + return None + + if logger: + logger.download_complete(dep_ref.local_path, ref_suffix="local") + + # Build minimal PackageInfo for integration + local_apm_yml = install_path / "apm.yml" + if local_apm_yml.exists(): + local_pkg = APMPackage.from_apm_yml(local_apm_yml) + if not local_pkg.source: + local_pkg.source = dep_ref.local_path + else: + local_pkg = APMPackage( + name=Path(dep_ref.local_path).name, + version="0.0.0", + package_path=install_path, + source=dep_ref.local_path, + ) + + local_ref = ResolvedReference( + original_ref="local", + ref_type=GitReferenceType.BRANCH, + resolved_commit="local", + ref_name="local", + ) + local_info = PackageInfo( + package=local_pkg, + install_path=install_path, + resolved_reference=local_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, + ) + + # Detect package type + pkg_type, plugin_json_path = detect_package_type(install_path) + local_info.package_type = pkg_type + if pkg_type == PackageType.MARKETPLACE_PLUGIN: + from apm_cli.deps.plugin_parser import normalize_plugin_directory + normalize_plugin_directory(install_path, plugin_json_path) + + # Record for lockfile + node = ctx.dependency_graph.dependency_tree.get_node(dep_key) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=None, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=None, + )) + if install_path.is_dir() and not dep_ref.is_local: + ctx.package_hashes[dep_key] = _compute_hash(install_path) + + if local_info.package_type: + ctx.package_types[dep_key] = local_info.package_type.value + + return Materialization( + package_info=local_info, + install_path=install_path, + dep_key=dep_key, + ) + + +class CachedDependencySource(DependencySource): + """Cached dependency: already extracted under ``apm_modules/``.""" + + def __init__( + self, + ctx: "InstallContext", + dep_ref: Any, + install_path: Path, + dep_key: str, + resolved_ref: Any, + dep_locked_chk: Any, + ): + super().__init__(ctx, dep_ref, install_path, dep_key) + self.resolved_ref = resolved_ref + self.dep_locked_chk = dep_locked_chk + + def acquire(self) -> Optional[Materialization]: + from apm_cli.constants import APM_YML_FILENAME + from apm_cli.deps.installed_package import InstalledPackage + from apm_cli.models.apm_package import ( + APMPackage, + GitReferenceType, + PackageInfo, + ResolvedReference, + ) + from apm_cli.models.validation import detect_package_type + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + + ctx = self.ctx + dep_ref = self.dep_ref + install_path = self.install_path + dep_key = self.dep_key + resolved_ref = self.resolved_ref + dep_locked_chk = self.dep_locked_chk + logger = ctx.logger + + display_name = str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + _ref = dep_ref.reference or "" + _sha = "" + if ( + dep_locked_chk + and dep_locked_chk.resolved_commit + and dep_locked_chk.resolved_commit != "cached" + ): + _sha = dep_locked_chk.resolved_commit[:8] + if logger: + logger.download_complete(display_name, ref=_ref, sha=_sha, cached=True) + + deltas: Dict[str, int] = {"installed": 1} + if not dep_ref.reference: + deltas["unpinned"] = 1 + + # Skip integration entirely if no targets + if not ctx.targets: + ctx.package_deployed_files[dep_key] = [] + # Caller will treat None as "no integration" -- but we still + # want to count this as installed. Convention: return a + # Materialization with package_info=None to signal "skip + # integration but keep deltas". We instead return a special + # marker by setting package_info to a sentinel; cleaner is to + # let the template detect ctx.targets falsy. + return Materialization( + package_info=None, # type: ignore[arg-type] + install_path=install_path, + dep_key=dep_key, + deltas=deltas, + ) + + # Load package from apm.yml + apm_yml_path = install_path / APM_YML_FILENAME + if apm_yml_path.exists(): + cached_package = APMPackage.from_apm_yml(apm_yml_path) + if not cached_package.source: + cached_package.source = dep_ref.repo_url + else: + cached_package = APMPackage( + name=dep_ref.repo_url.split("/")[-1], + version="unknown", + package_path=install_path, + source=dep_ref.repo_url, + ) + + resolved_or_cached_ref = resolved_ref if resolved_ref else ResolvedReference( + original_ref=dep_ref.reference or "default", + ref_type=GitReferenceType.BRANCH, + resolved_commit="cached", + ref_name=dep_ref.reference or "default", + ) + + cached_package_info = PackageInfo( + package=cached_package, + install_path=install_path, + resolved_reference=resolved_or_cached_ref, + installed_at=datetime.now().isoformat(), + dependency_ref=dep_ref, + ) + + pkg_type, _ = detect_package_type(install_path) + cached_package_info.package_type = pkg_type + + # Collect for lockfile + node = ctx.dependency_graph.dependency_tree.get_node(dep_key) + depth = node.depth if node else 1 + resolved_by = node.parent.dependency_ref.repo_url if node and node.parent else None + _is_dev = node.is_dev if node else False + + # Determine commit SHA: resolved > callback > existing lockfile > reference + cached_commit = None + if ( + resolved_ref + and resolved_ref.resolved_commit + and resolved_ref.resolved_commit != "cached" + ): + cached_commit = resolved_ref.resolved_commit + if not cached_commit: + cached_commit = ctx.callback_downloaded.get(dep_key) + if not cached_commit and ctx.existing_lockfile: + locked_dep = ctx.existing_lockfile.get_dependency(dep_key) + if locked_dep: + cached_commit = locked_dep.resolved_commit + if not cached_commit: + cached_commit = dep_ref.reference + + # Determine if cached package came from registry + _cached_registry = None + if dep_locked_chk and dep_locked_chk.registry_prefix: + _cached_registry = ctx.registry_config + elif ctx.registry_config and not dep_ref.is_local: + _cached_registry = ctx.registry_config + + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=cached_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=_cached_registry, + )) + if install_path.is_dir(): + ctx.package_hashes[dep_key] = _compute_hash(install_path) + if cached_package_info.package_type: + ctx.package_types[dep_key] = cached_package_info.package_type.value + + return Materialization( + package_info=cached_package_info, + install_path=install_path, + dep_key=dep_key, + deltas=deltas, + ) + + +class FreshDependencySource(DependencySource): + """Fresh dependency: needs a network download. + + Performs supply-chain hash verification (#763) and, on mismatch, + aborts the entire process via ``sys.exit(1)`` -- this matches the + legacy behaviour because content drift from the lockfile is treated + as a possible tampering event. + """ + + def __init__( + self, + ctx: "InstallContext", + dep_ref: Any, + install_path: Path, + dep_key: str, + resolved_ref: Any, + dep_locked_chk: Any, + ref_changed: bool, + progress: Any, + ): + super().__init__(ctx, dep_ref, install_path, dep_key) + self.resolved_ref = resolved_ref + self.dep_locked_chk = dep_locked_chk + self.ref_changed = ref_changed + self.progress = progress + + def acquire(self) -> Optional[Materialization]: + from apm_cli.deps.installed_package import InstalledPackage + from apm_cli.drift import build_download_ref + from apm_cli.models.apm_package import PackageType + from apm_cli.utils.content_hash import compute_package_hash as _compute_hash + from apm_cli.utils.path_security import safe_rmtree + + ctx = self.ctx + dep_ref = self.dep_ref + install_path = self.install_path + dep_key = self.dep_key + dep_locked_chk = self.dep_locked_chk + ref_changed = self.ref_changed + progress = self.progress + diagnostics = ctx.diagnostics + logger = ctx.logger + + try: + display_name = str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + short_name = ( + display_name.split("/")[-1] if "/" in display_name else display_name + ) + + task_id = progress.add_task( + description=f"Fetching {short_name}", + total=None, + ) + + download_ref = build_download_ref( + dep_ref, + ctx.existing_lockfile, + update_refs=ctx.update_refs, + ref_changed=ref_changed, + ) + + if dep_key in ctx.pre_download_results: + package_info = ctx.pre_download_results[dep_key] + else: + package_info = ctx.downloader.download_package( + download_ref, + install_path, + progress_task_id=task_id, + progress_obj=progress, + ) + + # CRITICAL: hide progress BEFORE printing success to avoid overlap + progress.update(task_id, visible=False) + progress.refresh() + + deltas: Dict[str, int] = {"installed": 1} + + resolved = getattr(package_info, "resolved_reference", None) + if logger: + _ref = "" + _sha = "" + if resolved: + _ref = resolved.ref_name if resolved.ref_name else "" + _sha = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + logger.download_complete(display_name, ref=_ref, sha=_sha) + if ctx.auth_resolver: + try: + _host = dep_ref.host or "github.com" + _org = ( + dep_ref.repo_url.split("/")[0] + if dep_ref.repo_url and "/" in dep_ref.repo_url + else None + ) + _ctx = ctx.auth_resolver.resolve(_host, org=_org) + logger.package_auth(_ctx.source, _ctx.token_type or "none") + except Exception: + pass + else: + _ref_suffix = "" + if resolved: + _r = resolved.ref_name if resolved.ref_name else "" + _s = resolved.resolved_commit[:8] if resolved.resolved_commit else "" + if _r and _s: + _ref_suffix = f" #{_r} @{_s}" + elif _r: + _ref_suffix = f" #{_r}" + elif _s: + _ref_suffix = f" @{_s}" + _rich_success(f"[+] {display_name}{_ref_suffix}") + + if not dep_ref.reference: + deltas["unpinned"] = 1 + + # Lockfile bookkeeping + resolved_commit = None + if resolved: + resolved_commit = package_info.resolved_reference.resolved_commit + node = ctx.dependency_graph.dependency_tree.get_node(dep_key) + depth = node.depth if node else 1 + resolved_by = ( + node.parent.dependency_ref.repo_url if node and node.parent else None + ) + _is_dev = node.is_dev if node else False + ctx.installed_packages.append(InstalledPackage( + dep_ref=dep_ref, resolved_commit=resolved_commit, + depth=depth, resolved_by=resolved_by, is_dev=_is_dev, + registry_config=ctx.registry_config if not dep_ref.is_local else None, + )) + if install_path.is_dir(): + ctx.package_hashes[dep_key] = _compute_hash(install_path) + + # Supply-chain protection: verify content hash on fresh + # downloads when the lockfile already records a hash. + if ( + not ctx.update_refs + and dep_locked_chk + and dep_locked_chk.content_hash + and dep_key in ctx.package_hashes + ): + _fresh_hash = ctx.package_hashes[dep_key] + if _fresh_hash != dep_locked_chk.content_hash: + safe_rmtree(install_path, ctx.apm_modules_dir) + _rich_error( + f"Content hash mismatch for " + f"{dep_key}: " + f"expected {dep_locked_chk.content_hash}, " + f"got {_fresh_hash}. " + "The downloaded content differs from the " + "lockfile record. This may indicate a " + "supply-chain attack. Use 'apm install " + "--update' to accept new content and " + "update the lockfile." + ) + sys.exit(1) + + if hasattr(package_info, "package_type") and package_info.package_type: + ctx.package_types[dep_key] = package_info.package_type.value + + if hasattr(package_info, "package_type"): + package_type = package_info.package_type + _type_label = { + PackageType.CLAUDE_SKILL: "Skill (SKILL.md detected)", + PackageType.MARKETPLACE_PLUGIN: "Marketplace Plugin (plugin.json detected)", + PackageType.HYBRID: "Hybrid (apm.yml + SKILL.md)", + PackageType.APM_PACKAGE: "APM Package (apm.yml)", + }.get(package_type) + if _type_label and logger: + logger.package_type_info(_type_label) + + # If no targets, skip integration but keep deltas + if not ctx.targets: + return Materialization( + package_info=None, # type: ignore[arg-type] + install_path=install_path, + dep_key=dep_key, + deltas=deltas, + ) + + return Materialization( + package_info=package_info, + install_path=package_info.install_path, + dep_key=dep_key, + deltas=deltas, + ) + + except Exception as e: + display_name = str(dep_ref) if dep_ref.is_virtual else dep_ref.repo_url + # task_id may not exist if progress.add_task failed; guard it. + try: + progress.remove_task(task_id) # type: ignore[name-defined] + except Exception: + pass + diagnostics.error( + f"Failed to install {display_name}: {e}", + package=dep_key, + ) + return None + + +def make_dependency_source( + ctx: "InstallContext", + dep_ref: Any, + install_path: Path, + dep_key: str, + *, + resolved_ref: Any = None, + dep_locked_chk: Any = None, + ref_changed: bool = False, + skip_download: bool = False, + progress: Any = None, +) -> DependencySource: + """Factory: pick the right ``DependencySource`` for *dep_ref*. + + Caller is responsible for resolving the download strategy (cached vs + fresh) before invoking the factory; the resolved-ref and + locked-checksum data flow into the appropriate source. + """ + if dep_ref.is_local and dep_ref.local_path: + return LocalDependencySource(ctx, dep_ref, install_path, dep_key) + if skip_download: + return CachedDependencySource( + ctx, dep_ref, install_path, dep_key, resolved_ref, dep_locked_chk, + ) + return FreshDependencySource( + ctx, dep_ref, install_path, dep_key, + resolved_ref, dep_locked_chk, ref_changed, progress, + ) diff --git a/src/apm_cli/install/template.py b/src/apm_cli/install/template.py new file mode 100644 index 00000000..ce9eddf6 --- /dev/null +++ b/src/apm_cli/install/template.py @@ -0,0 +1,146 @@ +"""Integration template -- shared post-acquire flow for all DependencySources. + +After ``DependencySource.acquire()`` materialises a package, every source +funnels through the same template: + +1. Pre-deploy security gate (``_pre_deploy_security_scan``). +2. Primitive integration (``integrate_package_primitives``). +3. Per-package verbose diagnostics (skip / error counts). + +This is the Template Method companion to the Strategy pattern in +``apm_cli.install.sources``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Optional + +from apm_cli.install.helpers.security_scan import _pre_deploy_security_scan +from apm_cli.install.services import integrate_package_primitives +from apm_cli.install.sources import DependencySource, Materialization + +if TYPE_CHECKING: + pass + + +def run_integration_template( + source: DependencySource, +) -> Optional[Dict[str, int]]: + """Run the shared post-acquire integration flow for one dependency. + + Returns a counter-delta dict for accumulation by the caller, or + ``None`` if the source declined to acquire (skipped, failed). + """ + materialization = source.acquire() + if materialization is None: + return None + + return _integrate_materialization(source, materialization) + + +def _integrate_materialization( + source: DependencySource, + m: Materialization, +) -> Dict[str, int]: + """Apply security gate + primitive integration on a materialised package. + + The caller has already populated ``ctx.installed_packages`` / + ``ctx.package_hashes`` / ``ctx.package_types`` inside ``acquire()``. + Here we focus on the deployment side: security scan, primitive + integration, deployed-files tracking, and per-package diagnostics. + """ + ctx = source.ctx + dep_ref = source.dep_ref + deltas = m.deltas + install_path = m.install_path + dep_key = m.dep_key + diagnostics = ctx.diagnostics + logger = ctx.logger + + # No-op when targets are empty or acquire decided to skip integration + # (signalled by package_info=None). Still record an empty deployed + # list so cleanup phase has a deterministic state. + if m.package_info is None or not ctx.targets: + ctx.package_deployed_files[dep_key] = [] + return deltas + + try: + # Pre-deploy security gate + if not _pre_deploy_security_scan( + install_path, diagnostics, + package_name=dep_key, force=ctx.force, + logger=logger, + ): + ctx.package_deployed_files[dep_key] = [] + return deltas + + int_result = integrate_package_primitives( + m.package_info, ctx.project_root, + targets=ctx.targets, + prompt_integrator=ctx.integrators["prompt"], + agent_integrator=ctx.integrators["agent"], + skill_integrator=ctx.integrators["skill"], + instruction_integrator=ctx.integrators["instruction"], + command_integrator=ctx.integrators["command"], + hook_integrator=ctx.integrators["hook"], + force=ctx.force, + managed_files=ctx.managed_files, + diagnostics=diagnostics, + package_name=dep_key, + logger=logger, + scope=ctx.scope, + ) + for k in ( + "prompts", "agents", "skills", "sub_skills", + "instructions", "commands", "hooks", "links_resolved", + ): + deltas[k] = int_result[k] + ctx.package_deployed_files[dep_key] = int_result["deployed_files"] + except Exception as e: + # Match legacy error message shape per source. Local packages + # use the local path; cached/fresh use the dep_key. + if dep_ref.is_local and dep_ref.local_path: + diagnostics.error( + f"Failed to integrate primitives from local package: {e}", + package=dep_ref.local_path, + ) + else: + # Both cached and fresh originally used different prefixes; + # we preserve the cached "from cached package" wording when + # the source is cached, otherwise the generic fresh message. + from apm_cli.install.sources import ( + CachedDependencySource, + FreshDependencySource, + ) + if isinstance(source, CachedDependencySource): + diagnostics.error( + f"Failed to integrate primitives from cached package: {e}", + package=dep_key, + ) + elif isinstance(source, FreshDependencySource): + diagnostics.error( + f"Failed to integrate primitives: {e}", + package=dep_key, + ) + else: + diagnostics.error( + f"Failed to integrate primitives: {e}", + package=dep_key, + ) + + # Verbose: inline skip / error count for this package + if logger and logger.verbose: + _skip_count = diagnostics.count_for_package(dep_key, "collision") + _err_count = diagnostics.count_for_package(dep_key, "error") + if _skip_count > 0: + noun = "file" if _skip_count == 1 else "files" + logger.package_inline_warning( + f" [!] {_skip_count} {noun} skipped (local files exist)" + ) + if _err_count > 0: + noun = "error" if _err_count == 1 else "errors" + logger.package_inline_warning( + f" [!] {_err_count} integration {noun}" + ) + + return deltas diff --git a/tests/unit/install/test_architecture_invariants.py b/tests/unit/install/test_architecture_invariants.py index 09714b08..9e41b0b0 100644 --- a/tests/unit/install/test_architecture_invariants.py +++ b/tests/unit/install/test_architecture_invariants.py @@ -41,10 +41,9 @@ def test_install_context_importable(): MAX_MODULE_LOC = 1000 KNOWN_LARGE_MODULES = { - # integrate.py hosts 4 per-package code paths + the root-project - # integration rewritten in F3 to use _integrate_local_content. - # Natural seam: decompose per-package helpers into a sub-module. - "phases/integrate.py": 1020, + # No exceptions: integrate.py was decomposed into Strategy + # (sources.py) + Template Method (template.py) and now sits well + # below the default budget. } From 1a34adad8049ed6cbc08e96c4e4991e40a1448d6 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 15:01:36 +0200 Subject: [PATCH 24/28] refactor(install): P3 -- introduce InstallService Application Service with typed InstallRequest Adds the Application Service pattern as the typed entry point for the install pipeline. Adapters (the Click handler today; programmatic / API callers tomorrow) build a frozen InstallRequest and call InstallService.run(request) -> InstallResult. * New install/request.py: frozen dataclass replacing the 11-kwarg ad-hoc parameter list with a typed, immutable record. * New install/service.py: InstallService class wrapping the pipeline. Stateless and reusable; documented as the future DI seam for collaborator injection (downloader, scanner, integrator factory). * commands/install.py: _install_apm_dependencies re-export now builds an InstallRequest and delegates through InstallService. All 55 existing test patches against the re-export keep working unchanged. * tests/unit/install/test_service.py: 6 direct-invocation tests exercising the typed Request -> Result contract without CliRunner. 3980 unit tests pass (3974 prior + 6 new). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + src/apm_cli/commands/install.py | 15 ++-- src/apm_cli/install/request.py | 39 +++++++++ src/apm_cli/install/service.py | 75 +++++++++++++++++ tests/unit/install/test_service.py | 129 +++++++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 5 deletions(-) create mode 100644 src/apm_cli/install/request.py create mode 100644 src/apm_cli/install/service.py create mode 100644 tests/unit/install/test_service.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e8b2f43d..aece3cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Refactor `apm install` internals to apply real design patterns: introduce a `DependencySource` Strategy hierarchy with shared `run_integration_template()` Template Method (kills ~300 LOC duplication across local/cached/fresh dep handlers), add `services.py` DI seam to eliminate `_install_mod` indirection, and wrap the pipeline in a typed `InstallService` Application Service consuming a frozen `InstallRequest`. `install/phases/integrate.py` shrinks from 1013 to ~400 LOC; the public `apm install` behaviour and CLI surface are unchanged. Backward-compatible: `_install_apm_dependencies` re-export and 55 healthy test patches keep working - `apm marketplace browse/search/add/update` now route through the registry proxy when `PROXY_REGISTRY_URL` is set; `PROXY_REGISTRY_ONLY=1` blocks direct GitHub API calls (#506) - Refactor `apm install` into a modular engine package (`apm_cli/install/`) with discrete phases (resolve, targets, download, integrate, cleanup, lockfile, finalize, post-deps local). Reduces `commands/install.py` from 2905 to ~933 LOC while preserving behaviour and the `#762` cleanup chokepoint (#764) diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index 7c07f423..b85bf869 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -727,18 +727,22 @@ def _install_apm_dependencies( target: str = None, marketplace_provenance: dict = None, ): - """Thin wrapper -- delegates to :func:`apm_cli.install.pipeline.run_install_pipeline`. + """Thin wrapper -- builds an :class:`InstallRequest` and delegates to + :class:`apm_cli.install.service.InstallService`. Kept here so that ``@patch("apm_cli.commands.install._install_apm_dependencies")`` - continues to intercept calls from the Click handler. + continues to intercept calls from the Click handler. The service + itself is the typed Application Service entry point for any future + programmatic callers. """ if not APM_DEPS_AVAILABLE: raise RuntimeError("APM dependency system not available") - from apm_cli.install.pipeline import run_install_pipeline + from apm_cli.install.request import InstallRequest + from apm_cli.install.service import InstallService - return run_install_pipeline( - apm_package, + request = InstallRequest( + apm_package=apm_package, update_refs=update_refs, verbose=verbose, only_packages=only_packages, @@ -750,6 +754,7 @@ def _install_apm_dependencies( target=target, marketplace_provenance=marketplace_provenance, ) + return InstallService().run(request) diff --git a/src/apm_cli/install/request.py b/src/apm_cli/install/request.py new file mode 100644 index 00000000..1e839734 --- /dev/null +++ b/src/apm_cli/install/request.py @@ -0,0 +1,39 @@ +"""Typed inputs for the install pipeline (Application Service input). + +Bundles the 11 kwargs previously passed to ``run_install_pipeline`` into a +single immutable record that the Click handler builds from CLI args and +the ``InstallService`` consumes. This is the typed-IO companion to +``InstallResult`` (the Service output, defined in ``apm_cli.models.results``). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +if TYPE_CHECKING: + from apm_cli.core.auth import AuthResolver + from apm_cli.core.command_logger import InstallLogger + from apm_cli.core.scope import InstallScope + from apm_cli.models.apm_package import APMPackage + + +@dataclass(frozen=True) +class InstallRequest: + """User intent for one install invocation. + + Frozen: never mutated by the pipeline. Built once by the Click + handler (or test harness) and handed to ``InstallService.run()``. + """ + + apm_package: "APMPackage" + update_refs: bool = False + verbose: bool = False + only_packages: Optional[List[str]] = None + force: bool = False + parallel_downloads: int = 4 + logger: Optional["InstallLogger"] = None + scope: Optional["InstallScope"] = None + auth_resolver: Optional["AuthResolver"] = None + target: Optional[str] = None + marketplace_provenance: Optional[Dict[str, Any]] = None diff --git a/src/apm_cli/install/service.py b/src/apm_cli/install/service.py new file mode 100644 index 00000000..0cb1a9de --- /dev/null +++ b/src/apm_cli/install/service.py @@ -0,0 +1,75 @@ +"""Application Service: orchestrates one install invocation. + +The ``InstallService`` is the *behaviour-bearing* entry point for installs. +Adapters (the Click handler today; programmatic / API callers tomorrow) +build an :class:`InstallRequest` and call :meth:`InstallService.run`, +which returns a :class:`InstallResult`. Adapters own presentation, +``sys.exit``, and CLI option parsing -- the service does not. + +Why a class rather than a free function? +---------------------------------------- +The class encapsulates the *seam* for future dependency injection. Today +the underlying ``run_install_pipeline`` builds collaborators internally; +when (and only when) a programmatic caller needs to swap the downloader +or integrator factories, the service can grow constructor parameters +without changing every call site. + +For now the service is intentionally lean: it validates that the dep +system is available, then delegates to the existing pipeline. This +gives every adapter a typed Request -> Result contract today without +the blast radius of a deeper DI rewrite. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from apm_cli.install.request import InstallRequest + +if TYPE_CHECKING: + from apm_cli.models.results import InstallResult + + +class InstallNotAvailableError(RuntimeError): + """Raised when the APM dependency subsystem failed to import.""" + + +class InstallService: + """Application service for the APM install pipeline. + + Stateless: a single instance can serve multiple ``run(request)`` + invocations. Constructor takes no arguments today but exists as the + extension point for collaborator injection (downloader, scanner, + integrator factory) when programmatic callers need to swap them. + """ + + def run(self, request: InstallRequest) -> "InstallResult": + """Execute the install pipeline and return the structured result. + + Raises: + InstallNotAvailableError: if the dependency subsystem failed + to import (e.g. missing optional extras). Adapters are + responsible for presenting this to the user. + """ + # Local import keeps service module import-cheap and matches the + # existing pipeline's lazy-import discipline. + try: + from apm_cli.install.pipeline import run_install_pipeline + except ImportError as e: # pragma: no cover -- defensive + raise InstallNotAvailableError( + f"APM dependency system not available: {e}" + ) from e + + return run_install_pipeline( + request.apm_package, + update_refs=request.update_refs, + verbose=request.verbose, + only_packages=request.only_packages, + force=request.force, + parallel_downloads=request.parallel_downloads, + logger=request.logger, + scope=request.scope, + auth_resolver=request.auth_resolver, + target=request.target, + marketplace_provenance=request.marketplace_provenance, + ) diff --git a/tests/unit/install/test_service.py b/tests/unit/install/test_service.py new file mode 100644 index 00000000..bd6af205 --- /dev/null +++ b/tests/unit/install/test_service.py @@ -0,0 +1,129 @@ +"""Direct tests for the InstallService Application Service. + +These tests bypass Click entirely -- they construct an InstallRequest +and call ``InstallService.run()`` directly. This is the contract that +future programmatic / API callers will depend on. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from apm_cli.install.request import InstallRequest +from apm_cli.install.service import InstallService + + +@pytest.fixture +def fake_apm_package(): + pkg = MagicMock() + pkg.dependencies = {"apm": []} + return pkg + + +def _make_request(pkg, **overrides): + base = dict( + apm_package=pkg, + update_refs=False, + verbose=False, + only_packages=None, + force=False, + parallel_downloads=4, + logger=None, + scope=None, + auth_resolver=None, + target=None, + marketplace_provenance=None, + ) + base.update(overrides) + return InstallRequest(**base) + + +class TestInstallRequest: + def test_request_is_frozen(self, fake_apm_package): + request = _make_request(fake_apm_package) + with pytest.raises((AttributeError, Exception)): + request.force = True + + def test_request_defaults(self, fake_apm_package): + request = InstallRequest(apm_package=fake_apm_package) + assert request.update_refs is False + assert request.parallel_downloads == 4 + assert request.only_packages is None + assert request.target is None + + +class TestInstallServiceDelegation: + def test_run_delegates_to_pipeline_with_request_fields(self, fake_apm_package): + request = _make_request( + fake_apm_package, + update_refs=True, + verbose=True, + force=True, + parallel_downloads=8, + target="copilot", + ) + with patch("apm_cli.install.pipeline.run_install_pipeline") as mock_run: + mock_run.return_value = "result-sentinel" + result = InstallService().run(request) + + assert result == "result-sentinel" + mock_run.assert_called_once() + args, kwargs = mock_run.call_args + assert args[0] is fake_apm_package + assert kwargs["update_refs"] is True + assert kwargs["verbose"] is True + assert kwargs["force"] is True + assert kwargs["parallel_downloads"] == 8 + assert kwargs["target"] == "copilot" + + def test_run_passes_optional_collaborators(self, fake_apm_package): + logger = MagicMock() + auth = MagicMock() + scope = MagicMock() + request = _make_request( + fake_apm_package, logger=logger, auth_resolver=auth, scope=scope + ) + with patch("apm_cli.install.pipeline.run_install_pipeline") as mock_run: + InstallService().run(request) + + kwargs = mock_run.call_args.kwargs + assert kwargs["logger"] is logger + assert kwargs["auth_resolver"] is auth + assert kwargs["scope"] is scope + + def test_service_is_reusable_across_invocations(self, fake_apm_package): + service = InstallService() + with patch("apm_cli.install.pipeline.run_install_pipeline") as mock_run: + mock_run.return_value = "ok" + service.run(_make_request(fake_apm_package)) + service.run(_make_request(fake_apm_package, force=True)) + assert mock_run.call_count == 2 + + +class TestClickWrapperUsesService: + def test_install_apm_dependencies_builds_request_and_uses_service( + self, fake_apm_package + ): + from apm_cli.commands import install as install_mod + + with patch("apm_cli.install.service.InstallService.run") as mock_run: + mock_run.return_value = "wrapped-result" + result = install_mod._install_apm_dependencies( + fake_apm_package, + update_refs=True, + force=True, + parallel_downloads=2, + target="claude", + ) + + assert result == "wrapped-result" + mock_run.assert_called_once() + request = mock_run.call_args.args[0] + assert isinstance(request, InstallRequest) + assert request.apm_package is fake_apm_package + assert request.update_refs is True + assert request.force is True + assert request.parallel_downloads == 2 + assert request.target == "claude" From 8c66be9b90cc9511a74aa9775292ab3a890e32d3 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 15:09:16 +0200 Subject: [PATCH 25/28] refactor(install): P4 -- address architect-review nits Cheap polish from the post-P3 architect review (APPROVE_WITH_NITS): * Strategy pattern leak fix: replace isinstance() switch in template.py exception handler with a per-source INTEGRATE_ERROR_PREFIX class attribute on DependencySource and overrides on Local/Cached. The template now reads source.INTEGRATE_ERROR_PREFIX -- no more type switches in the supposedly-polymorphic dispatch. * Remove redundant ctx.package_deployed_files write in CachedDependencySource.acquire() no-targets branch -- the template is the single source of truth for that bookkeeping. * Drop vestigial 'if TYPE_CHECKING: pass' block in template.py. * Drop unused 'field' import in install/request.py. * Tighten test_service.py: use FrozenInstanceError instead of bare Exception; cover only_packages and marketplace_provenance round-trip; document the shallow-immutability gotcha with an explicit test. 3981 unit tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/install/request.py | 2 +- src/apm_cli/install/sources.py | 23 +++++++++----- src/apm_cli/install/template.py | 48 +++++++++--------------------- tests/unit/install/test_service.py | 15 +++++++++- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/apm_cli/install/request.py b/src/apm_cli/install/request.py index 1e839734..190e7224 100644 --- a/src/apm_cli/install/request.py +++ b/src/apm_cli/install/request.py @@ -8,7 +8,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: diff --git a/src/apm_cli/install/sources.py b/src/apm_cli/install/sources.py index 35f88281..ce84dd3e 100644 --- a/src/apm_cli/install/sources.py +++ b/src/apm_cli/install/sources.py @@ -62,6 +62,11 @@ class DependencySource(ABC): The post-acquire template flow is the same for every source. """ + INTEGRATE_ERROR_PREFIX: str = "Failed to integrate primitives" + """Per-source error wording used by the integration template when + ``integrate_package_primitives`` raises. Subclasses override to + preserve the legacy diagnostic text shown to users.""" + def __init__( self, ctx: "InstallContext", @@ -87,6 +92,8 @@ def acquire(self) -> Optional[Materialization]: class LocalDependencySource(DependencySource): """Local (``file://``) dependency: copy from a filesystem path.""" + INTEGRATE_ERROR_PREFIX = "Failed to integrate primitives from local package" + def acquire(self) -> Optional[Materialization]: from apm_cli.core.scope import InstallScope from apm_cli.deps.installed_package import InstalledPackage @@ -198,6 +205,8 @@ def acquire(self) -> Optional[Materialization]: class CachedDependencySource(DependencySource): """Cached dependency: already extracted under ``apm_modules/``.""" + INTEGRATE_ERROR_PREFIX = "Failed to integrate primitives from cached package" + def __init__( self, ctx: "InstallContext", @@ -247,15 +256,11 @@ def acquire(self) -> Optional[Materialization]: if not dep_ref.reference: deltas["unpinned"] = 1 - # Skip integration entirely if no targets + # Skip integration entirely if no targets. The template will + # write the empty deployed_files entry on its own (single source + # of truth), so we just signal "skip integration" via + # package_info=None. if not ctx.targets: - ctx.package_deployed_files[dep_key] = [] - # Caller will treat None as "no integration" -- but we still - # want to count this as installed. Convention: return a - # Materialization with package_info=None to signal "skip - # integration but keep deltas". We instead return a special - # marker by setting package_info to a sentinel; cleaner is to - # let the template detect ctx.targets falsy. return Materialization( package_info=None, # type: ignore[arg-type] install_path=install_path, @@ -352,6 +357,8 @@ class FreshDependencySource(DependencySource): as a possible tampering event. """ + # Inherits the default "Failed to integrate primitives" prefix. + def __init__( self, ctx: "InstallContext", diff --git a/src/apm_cli/install/template.py b/src/apm_cli/install/template.py index ce9eddf6..9fd7b5b9 100644 --- a/src/apm_cli/install/template.py +++ b/src/apm_cli/install/template.py @@ -13,15 +13,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Optional +from typing import Dict, Optional from apm_cli.install.helpers.security_scan import _pre_deploy_security_scan from apm_cli.install.services import integrate_package_primitives from apm_cli.install.sources import DependencySource, Materialization -if TYPE_CHECKING: - pass - def run_integration_template( source: DependencySource, @@ -97,36 +94,19 @@ def _integrate_materialization( deltas[k] = int_result[k] ctx.package_deployed_files[dep_key] = int_result["deployed_files"] except Exception as e: - # Match legacy error message shape per source. Local packages - # use the local path; cached/fresh use the dep_key. - if dep_ref.is_local and dep_ref.local_path: - diagnostics.error( - f"Failed to integrate primitives from local package: {e}", - package=dep_ref.local_path, - ) - else: - # Both cached and fresh originally used different prefixes; - # we preserve the cached "from cached package" wording when - # the source is cached, otherwise the generic fresh message. - from apm_cli.install.sources import ( - CachedDependencySource, - FreshDependencySource, - ) - if isinstance(source, CachedDependencySource): - diagnostics.error( - f"Failed to integrate primitives from cached package: {e}", - package=dep_key, - ) - elif isinstance(source, FreshDependencySource): - diagnostics.error( - f"Failed to integrate primitives: {e}", - package=dep_key, - ) - else: - diagnostics.error( - f"Failed to integrate primitives: {e}", - package=dep_key, - ) + # Per-source error wording: each DependencySource subclass + # declares its own INTEGRATE_ERROR_PREFIX (Strategy pattern). + # Local packages key the diagnostic by local_path; cached/fresh + # key by dep_key -- a behavioural detail preserved from legacy. + package_key = ( + dep_ref.local_path + if (dep_ref.is_local and dep_ref.local_path) + else dep_key + ) + diagnostics.error( + f"{source.INTEGRATE_ERROR_PREFIX}: {e}", + package=package_key, + ) # Verbose: inline skip / error count for this package if logger and logger.verbose: diff --git a/tests/unit/install/test_service.py b/tests/unit/install/test_service.py index bd6af205..26de2b13 100644 --- a/tests/unit/install/test_service.py +++ b/tests/unit/install/test_service.py @@ -42,8 +42,9 @@ def _make_request(pkg, **overrides): class TestInstallRequest: def test_request_is_frozen(self, fake_apm_package): + from dataclasses import FrozenInstanceError request = _make_request(fake_apm_package) - with pytest.raises((AttributeError, Exception)): + with pytest.raises(FrozenInstanceError): request.force = True def test_request_defaults(self, fake_apm_package): @@ -53,6 +54,14 @@ def test_request_defaults(self, fake_apm_package): assert request.only_packages is None assert request.target is None + def test_only_packages_is_shallow_immutable(self, fake_apm_package): + # Documents the known limitation: frozen=True locks the + # InstallRequest fields themselves, but the list reference is + # still mutable. Future hardening could swap to a tuple. + request = _make_request(fake_apm_package, only_packages=["pkg-a"]) + request.only_packages.append("pkg-b") + assert request.only_packages == ["pkg-a", "pkg-b"] + class TestInstallServiceDelegation: def test_run_delegates_to_pipeline_with_request_fields(self, fake_apm_package): @@ -63,6 +72,8 @@ def test_run_delegates_to_pipeline_with_request_fields(self, fake_apm_package): force=True, parallel_downloads=8, target="copilot", + only_packages=["alpha", "beta"], + marketplace_provenance={"source": "test-marketplace"}, ) with patch("apm_cli.install.pipeline.run_install_pipeline") as mock_run: mock_run.return_value = "result-sentinel" @@ -77,6 +88,8 @@ def test_run_delegates_to_pipeline_with_request_fields(self, fake_apm_package): assert kwargs["force"] is True assert kwargs["parallel_downloads"] == 8 assert kwargs["target"] == "copilot" + assert kwargs["only_packages"] == ["alpha", "beta"] + assert kwargs["marketplace_provenance"] == {"source": "test-marketplace"} def test_run_passes_optional_collaborators(self, fake_apm_package): logger = MagicMock() From dbb126af1a4ade72d772a9d5c92ad44149af87d9 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Sun, 19 Apr 2026 16:04:43 +0200 Subject: [PATCH 26/28] fix(install): address PR review nits from #764 Four targeted fixes from copilot-pull-request-reviewer feedback on the modularization PR. Each is a real defect inherited from main but surfaced by the new module boundaries; all four sit inside files we own in this PR so they're appropriate to fix here. 1. resolve.py: callback_failures TypeError on local + user-scope callback_failures is initialized as a set (line 105) and used with .add() everywhere except the local-package + user-scope branch, which used dict-style assignment. Would raise TypeError at runtime when a local dep was encountered under --global. Switch to .add(). The discarded message string was never consumed downstream (ctx.callback_failures is only iterated for counting). 2. context.py: incorrect Dict[str, Dict[...]] type hints package_types and package_hashes are typed as nested dicts but used as Dict[str, str] at all 6 write sites in sources.py. Fix the annotations to match actual usage. No runtime impact. 3. install.py: dry-run dropped only_packages filtering The dry-run renderer accepts only_packages= and forwards it to detect_orphans() for accurate orphan-preview filtering, but the call site never passed it. Hoist the canonical only_pkgs computation before the dry-run branch and thread it through. The actual install path now reuses the same hoisted variable. 4. validation.py: PAT leak in verbose ls-remote stderr The verbose-mode stderr scrub only replaced two env-var values, but git almost always echoes the failing URL in error messages, and the URL we built embeds _url_token (a per-dep PAT for GHES/ADO). Apply _sanitize_git_error() (the same scrubber the downloader uses for clone errors) before the env-value redaction. All 3981 unit tests pass. No behaviour change for the happy path; fixes manifest only on rare branches that were latent bugs on main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/apm_cli/commands/install.py | 14 ++++++++++---- src/apm_cli/install/context.py | 4 ++-- src/apm_cli/install/phases/resolve.py | 8 ++++---- src/apm_cli/install/validation.py | 18 +++++++++++++----- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/apm_cli/commands/install.py b/src/apm_cli/commands/install.py index b85bf869..6d138c82 100644 --- a/src/apm_cli/commands/install.py +++ b/src/apm_cli/commands/install.py @@ -522,6 +522,13 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo "MCP servers skipped at user scope (workspace-scoped concept)" ) + # Compute the canonical only_packages list once -- used both by + # the dry-run orphan preview and the actual install path. When + # the user passed --packages, we restrict to validated_packages + # (canonical strings) rather than the raw input which may carry + # marketplace refs like NAME@MARKETPLACE. + only_pkgs = builtins.list(validated_packages) if packages else None + # Show what will be installed if dry run if dry_run: from apm_cli.install.presentation.dry_run import render_and_exit @@ -534,6 +541,7 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo dev_apm_deps=dev_apm_deps, should_install_mcp=should_install_mcp, update=update, + only_packages=only_pkgs, apm_dir=apm_dir, ) return @@ -573,10 +581,8 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo try: # If specific packages were requested, only install those # Otherwise install all from apm.yml. - # Use validated_packages (canonical strings) instead of - # raw packages (which may contain marketplace refs like - # NAME@MARKETPLACE that don't match resolved dep identities). - only_pkgs = builtins.list(validated_packages) if packages else None + # `only_pkgs` was computed above so the dry-run preview + # and the actual install share one canonical list. install_result = _install_apm_dependencies( apm_package, update, verbose, only_pkgs, force=force, parallel_downloads=parallel_downloads, diff --git a/src/apm_cli/install/context.py b/src/apm_cli/install/context.py index 7f5ac87a..c78cdc05 100644 --- a/src/apm_cli/install/context.py +++ b/src/apm_cli/install/context.py @@ -92,8 +92,8 @@ class InstallContext: # ------------------------------------------------------------------ intended_dep_keys: Set[str] = field(default_factory=set) package_deployed_files: Dict[str, List[str]] = field(default_factory=dict) - package_types: Dict[str, Dict[str, Any]] = field(default_factory=dict) - package_hashes: Dict[str, Dict[str, str]] = field(default_factory=dict) + package_types: Dict[str, str] = field(default_factory=dict) + package_hashes: Dict[str, str] = field(default_factory=dict) installed_count: int = 0 # integrate unpinned_count: int = 0 # integrate installed_packages: List[Any] = field(default_factory=list) # integrate diff --git a/src/apm_cli/install/phases/resolve.py b/src/apm_cli/install/phases/resolve.py index 7cf5eb8d..524422cd 100644 --- a/src/apm_cli/install/phases/resolve.py +++ b/src/apm_cli/install/phases/resolve.py @@ -131,10 +131,10 @@ def download_callback(dep_ref, modules_dir, parent_chain=""): # Handle local packages: copy instead of git clone if dep_ref.is_local and dep_ref.local_path: if scope is InstallScope.USER: - # Cannot resolve local paths at user scope - callback_failures[dep_ref.get_unique_key()] = ( - f"local package '{dep_ref.local_path}' skipped at user scope" - ) + # Cannot resolve local paths at user scope. + # Note: callback_failures is a set (see line ~105), + # so use .add() rather than dict-style assignment. + callback_failures.add(dep_ref.get_unique_key()) return None result_path = _copy_local_package( dep_ref, install_path, project_root, logger=logger diff --git a/src/apm_cli/install/validation.py b/src/apm_cli/install/validation.py index 80d0260f..c9a775c1 100644 --- a/src/apm_cli/install/validation.py +++ b/src/apm_cli/install/validation.py @@ -214,12 +214,20 @@ def _validate_package_exists(package, verbose=False, auth_resolver=None, logger= if result.returncode == 0: verbose_log(f"git ls-remote rc=0 for {package}") else: - # Sanitize stderr to avoid leaking tokens - stderr_snippet = (result.stderr or "").strip()[:200] + # Sanitize stderr to avoid leaking tokens. Two layers: + # 1) scrub PAT-bearing URLs (git often echoes the URL + # in error messages -- the URL we built above + # embeds _url_token). Use the same sanitizer the + # downloader uses for clone errors. + # 2) belt-and-suspenders: also redact any literal env + # values that may have leaked through unrelated + # diagnostics paths. + raw_stderr = (result.stderr or "").strip()[:200] + stderr_snippet = ado_downloader._sanitize_git_error(raw_stderr) for env_var in ("GIT_ASKPASS", "GIT_CONFIG_GLOBAL"): - stderr_snippet = stderr_snippet.replace( - validate_env.get(env_var, ""), "***" - ) + env_val = validate_env.get(env_var, "") + if env_val: + stderr_snippet = stderr_snippet.replace(env_val, "***") verbose_log(f"git ls-remote rc={result.returncode}: {stderr_snippet}") return result.returncode == 0 From 8357596e60d39395afffd7f4b7c15689d5308d0d Mon Sep 17 00:00:00 2001 From: Daniel Meppiel <51440732+danielmeppiel@users.noreply.github.com> Date: Sun, 19 Apr 2026 16:38:58 +0200 Subject: [PATCH 27/28] Update CHANGELOG.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aece3cd0..2f6c112b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactor `apm install` internals to apply real design patterns: introduce a `DependencySource` Strategy hierarchy with shared `run_integration_template()` Template Method (kills ~300 LOC duplication across local/cached/fresh dep handlers), add `services.py` DI seam to eliminate `_install_mod` indirection, and wrap the pipeline in a typed `InstallService` Application Service consuming a frozen `InstallRequest`. `install/phases/integrate.py` shrinks from 1013 to ~400 LOC; the public `apm install` behaviour and CLI surface are unchanged. Backward-compatible: `_install_apm_dependencies` re-export and 55 healthy test patches keep working - `apm marketplace browse/search/add/update` now route through the registry proxy when `PROXY_REGISTRY_URL` is set; `PROXY_REGISTRY_ONLY=1` blocks direct GitHub API calls (#506) -- Refactor `apm install` into a modular engine package (`apm_cli/install/`) with discrete phases (resolve, targets, download, integrate, cleanup, lockfile, finalize, post-deps local). Reduces `commands/install.py` from 2905 to ~933 LOC while preserving behaviour and the `#762` cleanup chokepoint (#764) +- Refactor `apm install` into a modular engine package (`apm_cli/install/`) with discrete phases (resolve, targets, download, integrate, cleanup, lockfile, finalize, post-deps local), preserving behaviour and the `#762` cleanup chokepoint (#764) ## [0.8.11] - 2026-04-06 From 4656c0baa6065b61944eb7d2af2e0193c7d22561 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 14:46:12 +0000 Subject: [PATCH 28/28] refactor(install): address PR review nits (unused locals, type hints, docstrings) Agent-Logs-Url: https://github.com/microsoft/apm/sessions/130893b0-cd40-40a5-9cfe-156b8035fac5 Co-authored-by: danielmeppiel <51440732+danielmeppiel@users.noreply.github.com> --- CHANGELOG.md | 2 +- src/apm_cli/install/__init__.py | 8 ++++++-- src/apm_cli/install/phases/local_content.py | 11 +++++------ src/apm_cli/install/pipeline.py | 5 ----- src/apm_cli/install/sources.py | 6 +++--- src/apm_cli/install/validation.py | 4 ++-- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6c112b..7a85bf01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Refactor `apm install` internals to apply real design patterns: introduce a `DependencySource` Strategy hierarchy with shared `run_integration_template()` Template Method (kills ~300 LOC duplication across local/cached/fresh dep handlers), add `services.py` DI seam to eliminate `_install_mod` indirection, and wrap the pipeline in a typed `InstallService` Application Service consuming a frozen `InstallRequest`. `install/phases/integrate.py` shrinks from 1013 to ~400 LOC; the public `apm install` behaviour and CLI surface are unchanged. Backward-compatible: `_install_apm_dependencies` re-export and 55 healthy test patches keep working +- Refactor `apm install` internals to apply real design patterns: introduce a `DependencySource` Strategy hierarchy with shared `run_integration_template()` Template Method (kills ~300 LOC duplication across local/cached/fresh dep handlers), add `services.py` DI seam to eliminate `_install_mod` indirection, and wrap the pipeline in a typed `InstallService` Application Service consuming a frozen `InstallRequest`. `install/phases/integrate.py` shrinks from 1013 to ~400 LOC; the public `apm install` behaviour and CLI surface are unchanged. Backward-compatible: `_install_apm_dependencies` re-export and 55 healthy test patches keep working (#764) - `apm marketplace browse/search/add/update` now route through the registry proxy when `PROXY_REGISTRY_URL` is set; `PROXY_REGISTRY_ONLY=1` blocks direct GitHub API calls (#506) - Refactor `apm install` into a modular engine package (`apm_cli/install/`) with discrete phases (resolve, targets, download, integrate, cleanup, lockfile, finalize, post-deps local), preserving behaviour and the `#762` cleanup chokepoint (#764) diff --git a/src/apm_cli/install/__init__.py b/src/apm_cli/install/__init__.py index 3f756f4a..351c4c71 100644 --- a/src/apm_cli/install/__init__.py +++ b/src/apm_cli/install/__init__.py @@ -3,11 +3,15 @@ This package implements the install pipeline that the `apm_cli.commands.install` Click command delegates to. -Architecture (in progress; see refactor/install-modularization branch): +Architecture: pipeline.py orchestrator that calls each phase in order context.py InstallContext dataclass (state passed between phases) - options.py InstallOptions dataclass (parsed CLI options) + request.py InstallRequest dataclass (typed CLI inputs) + service.py InstallService Application Service (entry point) + services.py DI seam re-exporting integration helpers + sources.py DependencySource Strategy hierarchy + template.py run_integration_template() Template Method validation.py manifest validation (dependency syntax, existence checks) phases/ one module per pipeline phase diff --git a/src/apm_cli/install/phases/local_content.py b/src/apm_cli/install/phases/local_content.py index 13accbca..1cf73cdf 100644 --- a/src/apm_cli/install/phases/local_content.py +++ b/src/apm_cli/install/phases/local_content.py @@ -12,12 +12,11 @@ a locally-referenced package into ``apm_modules/`` so the downstream integration pipeline can treat it uniformly. -The orchestrator ``_integrate_local_content`` remains in -``apm_cli.commands.install`` because it calls ``_integrate_package_primitives`` -via bare-name lookup, and tests patch -``apm_cli.commands.install._integrate_package_primitives`` to intercept that -call. Keeping the orchestrator co-located with the re-exported name preserves -``@patch`` compatibility without any test modifications. +The orchestrator ``_integrate_local_content`` lives in +``apm_cli.install.services`` (the DI seam) and is re-exported from +``apm_cli.commands.install`` for backward-compatible patching. Tests should +patch the symbol at the import path used by the code under test rather than +assuming the implementation lives in the commands module. Functions --------- diff --git a/src/apm_cli/install/pipeline.py b/src/apm_cli/install/pipeline.py index 5dadaf2e..f361ae45 100644 --- a/src/apm_cli/install/pipeline.py +++ b/src/apm_cli/install/pipeline.py @@ -168,7 +168,6 @@ def run_install_pipeline( # Future S-phases will fold them into the context one by one. # -------------------------------------------------------------- transitive_failures = ctx.transitive_failures - apm_modules_dir = ctx.apm_modules_dir diagnostics = DiagnosticCollector(verbose=verbose) @@ -180,12 +179,8 @@ def run_install_pipeline( from ..deps.lockfile import LockFile, get_lockfile_path from ..deps.installed_package import InstalledPackage from ..deps.registry_proxy import RegistryConfig - from ..utils.content_hash import compute_package_hash as _compute_hash installed_packages: List[InstalledPackage] = [] - package_deployed_files: builtins.dict = {} # dep_key -> list of relative deployed paths - package_types: builtins.dict = {} # dep_key -> package type string - _package_hashes: builtins.dict = {} # dep_key -> sha256 hash # Resolve registry proxy configuration once for this install session. registry_config = RegistryConfig.from_env() diff --git a/src/apm_cli/install/sources.py b/src/apm_cli/install/sources.py index ce84dd3e..d1f78415 100644 --- a/src/apm_cli/install/sources.py +++ b/src/apm_cli/install/sources.py @@ -48,7 +48,7 @@ class Materialization: gate + primitive integration on a freshly-acquired package. """ - package_info: "PackageInfo" + package_info: Optional["PackageInfo"] install_path: Path dep_key: str deltas: Dict[str, int] = field(default_factory=lambda: {"installed": 1}) @@ -262,7 +262,7 @@ def acquire(self) -> Optional[Materialization]: # package_info=None. if not ctx.targets: return Materialization( - package_info=None, # type: ignore[arg-type] + package_info=None, install_path=install_path, dep_key=dep_key, deltas=deltas, @@ -522,7 +522,7 @@ def acquire(self) -> Optional[Materialization]: # If no targets, skip integration but keep deltas if not ctx.targets: return Materialization( - package_info=None, # type: ignore[arg-type] + package_info=None, install_path=install_path, dep_key=dep_key, deltas=deltas, diff --git a/src/apm_cli/install/validation.py b/src/apm_cli/install/validation.py index c9a775c1..774b2c5c 100644 --- a/src/apm_cli/install/validation.py +++ b/src/apm_cli/install/validation.py @@ -50,7 +50,7 @@ def _local_path_failure_reason(dep_ref): return "no apm.yml, SKILL.md, or plugin.json found" -def _local_path_no_markers_hint(local_dir, verbose_log=None, logger=None): +def _local_path_no_markers_hint(local_dir, logger=None): """Scan two levels for sub-packages and print a hint if any are found.""" from apm_cli.utils.helpers import find_plugin_json @@ -122,7 +122,7 @@ def _validate_package_exists(package, verbose=False, auth_resolver=None, logger= if find_plugin_json(local) is not None: return True # Directory exists but lacks package markers -- surface a hint - _local_path_no_markers_hint(local, verbose_log, logger=logger) + _local_path_no_markers_hint(local, logger=logger) return False # For virtual packages, use the downloader's validation method