Richer core dumps: structured errors, sync_steps, LLM trace, ANSI strip

pdd/commands/modify.py

@@ -189,29 +189,51 @@ def change(
 
 @click.command()
 @click.argument("files", nargs=-1)
+@click.option(
+    "--all",
+    "all_",
+    is_flag=True,
+    default=False,
+    help="Repository-wide update (same as passing no file arguments).",
+)
 @click.option("--extensions", help="Comma-separated extensions for repo mode.")
 @click.option("--directory", help="Directory to scan for repo mode.")
 @click.option("--git", is_flag=True, help="Use git history for original code.")
 @click.option("--output", help="Output path for the updated prompt.")
 @click.option("--simple", is_flag=True, default=False, help="Use legacy simple update.")
 @click.option("--base-branch", type=str, default="main", help="Base branch for change detection in repo mode (default: main).")
+@click.option(
+    "--budget",
+    type=float,
+    default=None,
+    help="Repository-wide only: stop processing once total update cost reaches this cap.",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    help="Repository-wide only: show which prompts would be updated without calling the LLM or writing outputs.",
+)
 @click.pass_context
 @log_operation(operation="update", clears_run_report=True)
 @track_cost
 def update(
     ctx: click.Context,
     files: Tuple[str, ...],
+    all_: bool,
     extensions: Optional[str],
     directory: Optional[str],
     git: bool,
     output: Optional[str],
     simple: bool,
     base_branch: str,
+    budget: Optional[float],
+    dry_run: bool,
 ) -> Optional[Tuple[Any, float, str]]:
     """
     Update the original prompt file based on code changes.
 
-    Repo-wide mode (no args): Scan entire repo.
+    Repo-wide mode (no args, or --all): Scan entire repo.
     Single-file mode (1 arg): Update prompt for specific code file.
     """
     ctx.ensure_object(dict)
@@ -227,10 +249,16 @@ def update(
         )
     if len(files) > 3:
         raise click.UsageError("Too many arguments. Max 3: <prompt> <modified_code> <original_code>")
+    if all_ and len(files) > 0:
+        raise click.UsageError(
+            "Cannot combine --all with file paths; use repository-wide mode with no arguments or only --all."
+        )
+    if budget is not None and budget <= 0:
+        raise click.UsageError("--budget must be a positive number")
 
     try:
         # Handle argument counts per modify_python.prompt spec (aligned with README)
-        if len(files) == 0:
+        if len(files) == 0 or all_:
             # Repo-wide mode
             is_repo_mode = True
             input_prompt_file = None
@@ -280,6 +308,14 @@ def update(
                 raise click.UsageError(
                     "--base-branch can only be used in repository-wide mode"
                 )
+            if dry_run:
+                raise click.UsageError(
+                    "--dry-run is only valid in repository-wide mode (no file arguments, or use --all)."
+                )
+            if budget is not None:
+                raise click.UsageError(
+                    "--budget is only valid in repository-wide mode (no file arguments, or use --all)."
+                )
 
         # Call update_main with correct parameters
         ret = update_main(
@@ -294,6 +330,8 @@ def update(
             directory=directory,
             simple=simple,
             base_branch=base_branch,
+            budget=budget,
+            dry_run=dry_run,
         )
 
         if ret is None:

pdd/core/cli.py

@@ -25,9 +25,12 @@
 
 def _strip_ansi_codes(text: str) -> str:
     """Remove ANSI escape codes from text for clean log output."""
-    # Pattern matches ANSI escape sequences
-    ansi_escape = re.compile(r'\x1b\[[0-9;]*m')
-    return ansi_escape.sub('', text)
+    # Covers common CSI sequences (\x1b[...m, \x1b[...K, cursor moves),
+    # plus OSC sequences (\x1b]...BEL or \x1b]...\x1b\\) used by some terminals.
+    csi = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+    osc = re.compile(r"\x1b\].*?(?:\x07|\x1b\\)")
+    text = osc.sub("", text)
+    return csi.sub("", text)
 
 
 class OutputCapture:

pdd/core/dump.py

@@ -21,6 +21,51 @@
 from .errors import console, get_core_dump_errors
 
 
+def _extract_sync_steps_from_file_contents(file_contents: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Build per-operation sync step records from any attached *_sync.log files.
+
+    The sync log is JSONL. We only convert "operation" entries (not "event" entries)
+    into a simplified list suitable for core dump inspection.
+    """
+    steps: List[Dict[str, Any]] = []
+    for path_key, content in (file_contents or {}).items():
+        if not isinstance(path_key, str) or not path_key.endswith("_sync.log"):
+            continue
+        if not isinstance(content, str) or content.startswith("<"):
+            continue
+        for line in content.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except Exception:
+                continue
+            # Skip events; keep operation rows
+            if entry.get("type") == "event":
+                continue
+            op = entry.get("operation")
+            if not op:
+                continue
+            details = entry.get("details") or {}
+            steps.append(
+                {
+                    "operation": op,
+                    "success": bool(entry.get("success")),
+                    "cost": entry.get("actual_cost"),
+                    "model": entry.get("model") or "unknown",
+                    "duration": entry.get("duration"),
+                    "reason": entry.get("reason"),
+                    "error": entry.get("error"),
+                    "failure_summary": entry.get("error") or (details.get("failure_reason") if isinstance(details, dict) else None),
+                    "test_output_excerpt": details.get("test_output_excerpt") if isinstance(details, dict) else None,
+                    "source_log": path_key,
+                }
+            )
+    return steps
+
+
 def garbage_collect_core_dumps(keep: int = 10) -> int:
     """Delete old core dumps, keeping only the most recent `keep` files.
 
@@ -77,13 +122,13 @@ def _write_core_dump(
         dump_path = core_dump_dir / f"pdd-core-{timestamp}.json"
 
         steps: List[Dict[str, Any]] = []
-        for i, result_tuple in enumerate(normalized_results):
-            command_name = (
-                invoked_subcommands[i] if i < len(invoked_subcommands) else f"Unknown Command {i+1}"
-            )
+        step_count = max(len(invoked_subcommands), len(normalized_results))
+        for i in range(step_count):
+            command_name = invoked_subcommands[i] if i < len(invoked_subcommands) else f"Unknown Command {i+1}"
+            result_tuple = normalized_results[i] if i < len(normalized_results) else None
 
-            cost = None
-            model_name = None
+            cost: Optional[float] = None
+            model_name: Optional[str] = None
             if isinstance(result_tuple, tuple) and len(result_tuple) == 3:
                 _result_data, cost, model_name = result_tuple
 
@@ -92,7 +137,7 @@ def _write_core_dump(
                     "step": i + 1,
                     "command": command_name,
                     "cost": cost,
-                    "model": model_name,
+                    "model": (model_name or "unknown"),
                 }
             )
 
@@ -128,6 +173,12 @@ def _write_core_dump(
                         meta_file.stem.endswith(f"_{c}") for c in ["generate", "test", "run", "fix", "update"]
                     ):
                         core_dump_files.add(str(meta_file.resolve()))
+                # Include operation logs and run reports (critical for sync debugging)
+                # These are line-delimited JSON logs and are safe to attach (size-gated below).
+                for log_file in meta_dir.glob("*_sync.log"):
+                    core_dump_files.add(str(log_file.resolve()))
+                for run_file in meta_dir.glob("*_run.json"):
+                    core_dump_files.add(str(run_file.resolve()))
 
         # Auto-include PDD config files if they exist
         config_files = [
@@ -174,7 +225,7 @@ def _write_core_dump(
                     console.print(f"[warning]Debug snapshot: Error reading {file_path}: {e}[/warning]")
 
         payload: Dict[str, Any] = {
-            "schema_version": 1,
+            "schema_version": 2,
             "pdd_version": __version__,
             "timestamp_utc": timestamp,
             "argv": sys.argv[1:],  # without the 'pdd' binary name
@@ -205,6 +256,9 @@ def _write_core_dump(
             "file_contents": file_contents,
             "terminal_output": terminal_output,
         }
+        sync_steps = _extract_sync_steps_from_file_contents(file_contents)
+        if sync_steps:
+            payload["sync_steps"] = sync_steps
         if exit_reason is not None:
             payload["exit_reason"] = exit_reason
 

pdd/core/errors.py

@@ -3,7 +3,7 @@
 """
 import os
 import traceback
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 import click
 from rich.console import Console
 from rich.markup import MarkupError, escape
@@ -41,6 +41,32 @@ def clear_core_dump_errors() -> None:
     _core_dump_errors.clear()
 
 
+def record_core_dump_error(
+    *,
+    command: str,
+    type: str,
+    message: str,
+    details: Optional[Dict[str, Any]] = None,
+    traceback_text: Optional[str] = None,
+) -> None:
+    """Record a structured error entry for core dumps.
+
+    Use this for non-exception "logical failures" (budget exhaustion, retry limits,
+    cycle detection, etc.) so core dumps contain actionable context even when the
+    CLI run terminates without raising an exception.
+    """
+    error_record: Dict[str, Any] = {
+        "command": command,
+        "type": type,
+        "message": message,
+    }
+    if details:
+        error_record["details"] = details
+    if traceback_text:
+        error_record["traceback"] = traceback_text
+    _core_dump_errors.append(error_record)
+
+
 def _format_interrupt_reason(ctx: Dict[str, Any]) -> str:
     """Build a human-readable reason from agentic interrupt context."""
     step = ctx.get("current_step")

pdd/core/llm_trace.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import json
+import re
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, asdict
+from typing import Any, Dict, Iterator, Optional
+
+
+@dataclass
+class LLMTracePair:
+    prompt: str
+    response: str
+    model: str = "unknown"
+
+
+_current_operation: ContextVar[Optional[str]] = ContextVar("pdd_current_operation", default=None)
+_last_pair_by_operation: ContextVar[Dict[str, LLMTracePair]] = ContextVar(
+    "pdd_llm_last_pair_by_operation", default={}
+)
+
+
+_SENSITIVE_PATTERNS = [
+    # Common bearer tokens / API keys
+    re.compile(r"(?i)\b(bearer)\s+[a-z0-9\-_\.=:+/]{10,}"),
+    re.compile(r"(?i)\b(api[_-]?key|token|secret|password)\b\s*[:=]\s*[^\s\"']{6,}"),
+]
+
+
+def set_current_operation(operation: Optional[str]) -> None:
+    _current_operation.set(operation)
+
+
+@contextmanager
+def operation_scope(operation: str) -> Iterator[None]:
+    token = _current_operation.set(operation)
+    try:
+        yield
+    finally:
+        _current_operation.reset(token)
+
+
+def _truncate(text: str, limit_chars: int) -> str:
+    if text is None:
+        return ""
+    if len(text) <= limit_chars:
+        return text
+    return text[:limit_chars] + f"\n... (truncated, {len(text)} total chars)"
+
+
+def _redact(text: str) -> str:
+    if not text:
+        return text
+    redacted = text
+    for pat in _SENSITIVE_PATTERNS:
+        redacted = pat.sub("<redacted>", redacted)
+    return redacted
+
+
+def record_llm_pair(
+    *,
+    prompt: Any,
+    response: Any,
+    model: str = "unknown",
+    prompt_limit_chars: int = 20_000,
+    response_limit_chars: int = 20_000,
+) -> None:
+    """
+    Record the most recent (prompt, raw_response) pair for the current operation.
+    Intended to be called by llm_invoke.
+    """
+    op = _current_operation.get()
+    if not op:
+        return
+
+    try:
+        prompt_text = prompt if isinstance(prompt, str) else json.dumps(prompt, ensure_ascii=False, default=str)
+    except Exception:
+        prompt_text = str(prompt)
+
+    try:
+        response_text = response if isinstance(response, str) else json.dumps(response, ensure_ascii=False, default=str)
+    except Exception:
+        response_text = str(response)
+
+    pair = LLMTracePair(
+        prompt=_truncate(_redact(prompt_text), prompt_limit_chars),
+        response=_truncate(_redact(response_text), response_limit_chars),
+        model=str(model or "unknown"),
+    )
+
+    current = dict(_last_pair_by_operation.get() or {})
+    current[op] = pair
+    _last_pair_by_operation.set(current)
+
+
+def pop_last_pair(operation: str) -> Optional[Dict[str, Any]]:
+    """Pop and return the last recorded pair for an operation (as a dict)."""
+    current = dict(_last_pair_by_operation.get() or {})
+    pair = current.pop(operation, None)
+    _last_pair_by_operation.set(current)
+    return asdict(pair) if pair else None
+