perf(size): Parallelize Apple binary analysis across processes

src/launchpad/artifacts/apple/zipped_xcarchive.py

@@ -73,7 +73,6 @@ def __init__(self, path: Path) -> None:
         self._provisioning_profile: dict[str, Any] | None = None
         self._dsym_info: dict[str, DsymInfo] | None = None
         self._binary_uuid_cache: dict[Path, str] = {}
-        self._lief_cache: dict[Path, lief.MachO.FatBinary] = {}
 
     def get_extract_dir(self) -> SafeDirectory:
         return self._extract_dir
@@ -343,8 +342,8 @@ def get_all_binary_paths(self) -> List[BinaryInfo]:
         watch_paths = self._discover_watch_binaries(app_bundle_path)
         all_binary_paths.extend(watch_paths)
 
-        # Phase 2: Parse and cache all binaries
-        self._parse_and_cache_all_binaries(all_binary_paths)
+        # Phase 2: Extract UUIDs from all binaries
+        self._extract_all_binary_uuids(all_binary_paths)
 
         # Phase 3: Build BinaryInfo objects using cached data
         binaries: List[BinaryInfo] = []
@@ -367,10 +366,6 @@ def get_all_binary_paths(self) -> List[BinaryInfo]:
 
         return binaries
 
-    def get_lief_cache(self) -> dict[Path, lief.MachO.FatBinary]:
-        """Get the LIEF cache of pre-parsed binaries"""
-        return self._lief_cache
-
     @sentry_sdk.trace
     def get_asset_catalog_details(self, relative_path: Path) -> List[AssetCatalogElement]:
         """Get the details of an asset catalog file (Assets.car) by returning the
@@ -516,11 +511,7 @@ def _parse_asset_element(self, item: dict[str, Any], parent_path: Path) -> Asset
             scale=scale,
         )
 
-    def _parse_and_cache_all_binaries(self, binary_paths: List[Path]) -> None:
-        """Parse all binaries once, extracting UUIDs and caching LIEF objects for those with dSYMs."""
-        if self._dsym_info is None:
-            self._find_dsym_files()
-
+    def _extract_all_binary_uuids(self, binary_paths: List[Path]) -> None:
         config = lief.MachO.ParserConfig()
         config.parse_dyld_exports = False
         config.parse_dyld_bindings = False
@@ -556,14 +547,8 @@ def _parse_and_cache_all_binaries(self, binary_paths: List[Path]) -> None:
 
                 self._binary_uuid_cache[binary_path] = extracted_uuid
 
-                if extracted_uuid in self._dsym_info:
-                    self._lief_cache[binary_path] = fat_binary
-                    logger.debug(f"Cached LIEF object for {binary_path.name} (has dSYM)")
-                else:
-                    logger.debug(f"Skipped LIEF cache for {binary_path.name} (no dSYM)")
-
             except Exception:
-                logger.exception(f"Failed to parse and cache binary {binary_path}")
+                logger.exception(f"Failed to extract UUID from binary {binary_path}")
                 continue
 
     def _extract_binary_uuid(self, binary_path: Path) -> str | None:

src/launchpad/artifacts/artifact.py

@@ -1,8 +1,6 @@
 from pathlib import Path
 from typing import Any, Callable
 
-import lief
-
 from launchpad.parsers.android.dex.dex_mapping import DexMapping
 
 from .android.manifest.manifest import AndroidManifest
@@ -52,9 +50,5 @@ def get_plist(self) -> dict[str, Any]:
         """Get the plist from the artifact."""
         raise NotImplementedError("Not implemented")
 
-    def get_lief_cache(self) -> dict[Path, lief.MachO.FatBinary]:
-        """Get the LIEF cache of pre-parsed binaries"""
-        return {}
-
     def generate_ipa(self, output_path: Path):
         raise NotImplementedError("Not implemented")

src/launchpad/sentry_sdk_init.py

@@ -20,9 +20,9 @@
 logger = get_logger(__name__)
 
 
-def initialize_sentry_sdk() -> None:
+def initialize_sentry_sdk(config: SentryConfig | None = None) -> None:
     """Initialize Sentry SDK with launchpad-specific configuration."""
-    config = get_sentry_config()
+    config = config or get_sentry_config()
 
     if config.environment.lower() in ("test", "development"):
         logger.debug(f"In {config.environment} environment, skipping Sentry SDK initialization")

src/launchpad/size/analyzers/apple.py

@@ -3,23 +3,29 @@
 from __future__ import annotations
 
 import gc
+import logging
 import os
 import tempfile
 import time
 
+from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
+from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Tuple
+from typing import Any, Callable, Dict, List, NamedTuple, Tuple
 
 import lief
 import sentry_sdk
 
 from cryptography import x509
+from sentry_sdk.utils import qualname_from_function
 
 from launchpad.artifacts.apple.zipped_xcarchive import BinaryInfo, ZippedXCArchive
 from launchpad.artifacts.artifact import AppleArtifact
+from launchpad.artifacts.providers.safe_directory import SafeDirectory
 from launchpad.parsers.apple.dwarf_relocations_parser import DwarfRelocationsParser
 from launchpad.parsers.apple.macho_parser import MachOParser, get_cpu_type_name
+from launchpad.sentry_sdk_init import SentryConfig, get_sentry_config, initialize_sentry_sdk
 from launchpad.size.constants import APPLE_FILESYSTEM_BLOCK_SIZE
 from launchpad.size.hermes.reporter import HermesReport
 from launchpad.size.hermes.utils import make_hermes_reports
@@ -43,10 +49,11 @@
 from launchpad.size.utils.apple_bundle_size import calculate_bundle_sizes
 from launchpad.size.utils.file_analysis import analyze_apple_files
 from launchpad.size.utils.insight_path_map import build_insight_path_map
+from launchpad.tracing import bind_request_id, current_request_id
 from launchpad.utils.apple.apple_strip import AppleStrip
 from launchpad.utils.apple.code_signature_validator import CodeSignatureValidator
 from launchpad.utils.file_utils import get_file_size, to_nearest_block_size
-from launchpad.utils.logging import get_logger
+from launchpad.utils.logging import get_logger, setup_logging
 from launchpad.utils.metadata_extractor import extract_metadata_from_zip
 
 from ..models.apple import (
@@ -63,6 +70,48 @@
 
 logger = get_logger(__name__)
 
+_DEFAULT_BINARY_ANALYSIS_WORKERS = 4
+
+
+class _WorkerContext(NamedTuple):
+    verbose: bool
+    request_id: str | None
+    sentry_config: SentryConfig | None
+    trace_headers: dict[str, str]
+
+    @classmethod
+    def capture(cls) -> _WorkerContext:
+        headers = {"sentry-trace": sentry_sdk.get_traceparent(), "baggage": sentry_sdk.get_baggage()}
+        return cls(
+            verbose=logging.getLogger().getEffectiveLevel() <= logging.DEBUG,
+            request_id=current_request_id(),
+            sentry_config=get_sentry_config() if sentry_sdk.get_client().is_active() else None,
+            trace_headers={k: v for k, v in headers.items() if v},
+        )
+
+
+def _binary_worker_init(context: _WorkerContext) -> None:
+    os.environ["LAUNCHPAD_NO_PARALLEL_DEMANGLE"] = "true"
+    setup_logging(verbose=context.verbose)
+    if context.request_id is not None:
+        bind_request_id(context.request_id)
+    if context.sentry_config is not None:
+        initialize_sentry_sdk(context.sentry_config)
+        sentry_sdk.continue_trace(context.trace_headers)
+
+
+def _analyze_and_flush(analyze: Callable[[BinaryInfo], _TimedBinary], binary_info: BinaryInfo) -> _TimedBinary:
+    try:
+        return analyze(binary_info)
+    finally:
+        sentry_sdk.flush(timeout=2)
+
+
+class _TimedBinary(NamedTuple):
+    binary: MachOBinaryAnalysis | None
+    started_at: float
+    finished_at: float
+
 
 class AppleAppAnalyzer:
     """Analyzer for Apple app bundles (.xcarchive directories)."""
@@ -155,37 +204,35 @@ def analyze(self, artifact: AppleArtifact) -> AppleAnalysisResults:
             binaries = artifact.get_all_binary_paths()
             logger.debug(f"Found {len(binaries)} binaries to analyze")
 
-            lief_cache = artifact.get_lief_cache()
-            for binary_info in binaries:
-                logger.info(
-                    "size.apple.binary_analysis_started",
-                    extra={
-                        "event": "size.binary_analysis_started",
-                        "binary_name": binary_info.name,
-                        "binary_path": str(binary_info.path.relative_to(app_bundle_path)),
-                        "has_dsym": binary_info.dsym_path is not None,
-                    },
+            workers = self._binary_analysis_worker_count(len(binaries))
+            analyze = partial(
+                self._analyze_binary_logged,
+                app_bundle_path=app_bundle_path,
+                extract_dir=artifact.get_extract_dir(),
+            )
+            if workers > 0:
+                logger.debug(f"Analyzing binaries with {workers} processes")
+                executor = ProcessPoolExecutor(
+                    max_workers=workers,
+                    initializer=_binary_worker_init,
+                    initargs=(_WorkerContext.capture(),),
                 )
-                if binary_info.dsym_path:
-                    logger.debug(
-                        f"Found dSYM file for {binary_info.name} at {binary_info.dsym_path.relative_to(artifact.get_extract_dir())}"
-                    )
-                binary = self._analyze_binary(binary_info, app_bundle_path, lief_cache)
-                if binary is not None:
-                    binary_analysis.append(binary)
-                    binary_analysis_map[str(binary_info.path.relative_to(app_bundle_path))] = binary
+                try:
+                    results = list(executor.map(partial(_analyze_and_flush, analyze), binaries))
+                    executor.shutdown(wait=True)
+                except BaseException:
+                    executor.kill_workers()
+                    raise
+            else:
+                logger.debug("Analyzing binaries in-process")
+                results = [analyze(binary_info) for binary_info in binaries]
 
-                logger.info(
-                    "size.apple.binary_analysis_completed",
-                    extra={
-                        "event": "size.binary_analysis_completed",
-                        "binary_name": binary_info.name,
-                        "symbol_count": (len(binary.symbol_info.symbol_sizes) if binary.symbol_info else 0),
-                        "swift_types_count": (len(binary.symbol_info.swift_type_groups) if binary.symbol_info else 0),
-                        "objc_types_count": (len(binary.symbol_info.objc_type_groups) if binary.symbol_info else 0),
-                    },
-                )
-                gc.collect()
+            for binary_info, timed in zip(binaries, results):
+                if workers > 0:
+                    self._record_binary_span(binary_info, timed)
+                if timed.binary is not None:
+                    binary_analysis.append(timed.binary)
+                    binary_analysis_map[str(binary_info.path.relative_to(app_bundle_path))] = timed.binary
 
             hermes_reports = make_hermes_reports(app_bundle_path)
 
@@ -454,12 +501,66 @@ def _generate_insight_with_tracing(
             )
             return result
 
+    def _binary_analysis_worker_count(self, num_binaries: int) -> int:
+        try:
+            configured = int(os.getenv("LAUNCHPAD_BINARY_ANALYSIS_WORKERS", _DEFAULT_BINARY_ANALYSIS_WORKERS))
+        except ValueError:
+            configured = _DEFAULT_BINARY_ANALYSIS_WORKERS
+        return min(configured, num_binaries)
+
+    def _analyze_binary_logged(
+        self, binary_info: BinaryInfo, app_bundle_path: Path, extract_dir: SafeDirectory
+    ) -> _TimedBinary:
+        self._log_binary_started(binary_info, app_bundle_path, extract_dir)
+        started_at = time.time()
+        start = time.monotonic()
+        binary = self._analyze_binary(binary_info, app_bundle_path)
+        elapsed_s = time.monotonic() - start
+        gc.collect()
+        if binary is not None:
+            self._log_binary_completed(binary_info, binary, elapsed_s)
+        return _TimedBinary(binary, started_at, started_at + elapsed_s)
+
+    def _record_binary_span(self, binary_info: BinaryInfo, timed: _TimedBinary) -> None:
+        span = sentry_sdk.start_span(
+            op="function",
+            name=qualname_from_function(self._analyze_binary),
+            start_timestamp=timed.started_at,
+        )
+        span.set_data("binary_name", binary_info.name)
+        span.finish(end_timestamp=timed.finished_at)
+
+    def _log_binary_started(self, binary_info: BinaryInfo, app_bundle_path: Path, extract_dir: SafeDirectory) -> None:
+        logger.info(
+            "size.apple.binary_analysis_started",
+            extra={
+                "event": "size.binary_analysis_started",
+                "binary_name": binary_info.name,
+                "binary_path": str(binary_info.path.relative_to(app_bundle_path)),
+                "has_dsym": binary_info.dsym_path is not None,
+            },
+        )
+        if binary_info.dsym_path:
+            logger.debug(f"Found dSYM file for {binary_info.name} at {binary_info.dsym_path.relative_to(extract_dir)}")
+
+    def _log_binary_completed(self, binary_info: BinaryInfo, binary: MachOBinaryAnalysis, elapsed_s: float) -> None:
+        logger.info(
+            "size.apple.binary_analysis_completed",
+            extra={
+                "event": "size.binary_analysis_completed",
+                "binary_name": binary_info.name,
+                "elapsed_s": round(elapsed_s, 3),
+                "symbol_count": (len(binary.symbol_info.symbol_sizes) if binary.symbol_info else 0),
+                "swift_types_count": (len(binary.symbol_info.swift_type_groups) if binary.symbol_info else 0),
+                "objc_types_count": (len(binary.symbol_info.objc_type_groups) if binary.symbol_info else 0),
+            },
+        )
+
     @sentry_sdk.trace
     def _analyze_binary(
         self,
         binary_info: BinaryInfo,
         app_bundle_path: Path,
-        lief_cache: dict[Path, lief.MachO.FatBinary] | None = None,
         skip_swift_metadata: bool = False,
     ) -> MachOBinaryAnalysis | None:
         binary_path = binary_info.path
@@ -472,18 +573,13 @@ def _analyze_binary(
 
         logger.debug(f"Analyzing binary: {binary_path}")
 
-        # Only binaries with dSYMs are pre-cached. Pop from cache to free memory immediately.
-        # Binaries without dSYMs will be parsed on-demand here.
-        fat_binary = lief_cache.pop(binary_path, None) if lief_cache else None
-        if fat_binary is None:
-            logger.debug(f"Binary not in LIEF cache, parsing now: {binary_path.name}")
-            with open(binary_path, "rb") as f:
-                config = lief.MachO.ParserConfig()
-                config.parse_dyld_exports = False
-                config.parse_dyld_bindings = False
-                config.parse_dyld_rebases = False
-
-                fat_binary = lief.MachO.parse(f, config)  # type: ignore
+        with open(binary_path, "rb") as f:
+            config = lief.MachO.ParserConfig()
+            config.parse_dyld_exports = False
+            config.parse_dyld_bindings = False
+            config.parse_dyld_rebases = False
+
+            fat_binary = lief.MachO.parse(f, config)  # type: ignore
 
         if fat_binary is None or fat_binary.size == 0:
             raise RuntimeError(f"Failed to parse binary with LIEF: {binary_path}")

src/launchpad/tracing.py

@@ -17,6 +17,14 @@ def request_context():
         _request_id.reset(token)
 
 
+def current_request_id() -> str | None:
+    return _request_id.get(None)
+
+
+def bind_request_id(request_id: str) -> None:
+    _request_id.set(request_id)
+
+
 class RequestLogFilter:
     """Logging filter that adds request_id to log records.."""