From a7609ae896d60f3164a3d3368cf201f3e555fd19 Mon Sep 17 00:00:00 2001 From: limityan Date: Wed, 22 Jul 2026 17:00:59 +0800 Subject: [PATCH] fix: refresh bundled skill safety and workflows --- .../core/builtin-skills-upstreams.json | 43 ++ .../builtin_skills/agent-browser/SKILL.md | 472 ++--------------- .../references/authentication.md | 202 ------- .../agent-browser/references/commands.md | 263 ---------- .../agent-browser/references/profiling.md | 120 ----- .../agent-browser/references/proxy-support.md | 194 ------- .../references/session-management.md | 193 ------- .../agent-browser/references/snapshot-refs.md | 194 ------- .../references/video-recording.md | 173 ------ .../templates/authenticated-session.sh | 100 ---- .../templates/capture-workflow.sh | 69 --- .../templates/form-automation.sh | 62 --- .../core/builtin_skills/docx/SKILL.md | 491 ++---------------- .../builtin_skills/docx/scripts/comment.py | 316 ++++++----- .../builtin_skills/docx/scripts/merge_runs.py | 310 +++++++++++ .../docx/scripts/office/helpers/__init__.py | 150 ++++++ .../docx/scripts/office/helpers/merge_runs.py | 199 ------- .../docx/scripts/office/helpers/pptx_chart.py | 170 ++++++ .../docx/scripts/office/helpers/pptx_slide.py | 60 +++ .../docx/scripts/office/helpers/pptx_theme.py | 114 ++++ .../office/helpers/simplify_redlines.py | 197 ------- .../docx/scripts/office/pack.py | 159 ------ .../docx/scripts/office/soffice.py | 25 +- .../docx/scripts/office/unpack.py | 132 ----- .../docx/scripts/office/validate.py | 118 ++++- .../docx/scripts/office/validators/base.py | 68 ++- .../docx/scripts/office/validators/docx.py | 94 ++-- .../docx/scripts/office/validators/pptx.py | 180 ++++++- .../scripts/office/validators/redlining.py | 140 +++-- .../builtin_skills/gstack-autoplan/SKILL.md | 18 +- .../gstack-design-consultation/SKILL.md | 16 +- .../gstack-design-review/SKILL.md | 62 +-- .../gstack-office-hours/SKILL.md | 18 +- .../gstack-plan-ceo-review/SKILL.md | 3 - .../gstack-plan-design-review/SKILL.md | 17 +- .../gstack-plan-eng-review/SKILL.md | 3 - .../builtin_skills/gstack-qa-only/SKILL.md | 84 +-- .../core/builtin_skills/gstack-qa/SKILL.md | 97 ++-- .../core/builtin_skills/gstack-ship/SKILL.md | 7 +- .../core/builtin_skills/pptx/SKILL.md | 188 +++---- .../core/builtin_skills/pptx/editing.md | 205 -------- .../core/builtin_skills/pptx/pptxgenjs.md | 420 --------------- .../builtin_skills/pptx/scripts/add_slide.py | 378 ++++++++++---- .../core/builtin_skills/pptx/scripts/clean.py | 137 +++-- .../pptx/scripts/office/helpers/__init__.py | 150 ++++++ .../pptx/scripts/office/helpers/merge_runs.py | 199 ------- .../pptx/scripts/office/helpers/pptx_chart.py | 170 ++++++ .../pptx/scripts/office/helpers/pptx_slide.py | 60 +++ .../pptx/scripts/office/helpers/pptx_theme.py | 114 ++++ .../office/helpers/simplify_redlines.py | 197 ------- .../pptx/scripts/office/pack.py | 159 ------ .../pptx/scripts/office/soffice.py | 25 +- .../pptx/scripts/office/unpack.py | 132 ----- .../pptx/scripts/office/validate.py | 118 ++++- .../pptx/scripts/office/validators/base.py | 68 ++- .../pptx/scripts/office/validators/docx.py | 94 ++-- .../pptx/scripts/office/validators/pptx.py | 180 ++++++- .../scripts/office/validators/redlining.py | 140 +++-- .../builtin_skills/pptx/scripts/thumbnail.py | 76 ++- .../core/builtin_skills/xlsx/SKILL.md | 343 +++--------- .../xlsx/scripts/office/helpers/__init__.py | 150 ++++++ .../xlsx/scripts/office/helpers/merge_runs.py | 199 ------- .../xlsx/scripts/office/helpers/pptx_chart.py | 170 ++++++ .../xlsx/scripts/office/helpers/pptx_slide.py | 60 +++ .../xlsx/scripts/office/helpers/pptx_theme.py | 114 ++++ .../office/helpers/simplify_redlines.py | 197 ------- .../xlsx/scripts/office/pack.py | 159 ------ .../xlsx/scripts/office/soffice.py | 25 +- .../xlsx/scripts/office/unpack.py | 132 ----- .../xlsx/scripts/office/validate.py | 118 ++++- .../xlsx/scripts/office/validators/base.py | 68 ++- .../xlsx/scripts/office/validators/docx.py | 94 ++-- .../xlsx/scripts/office/validators/pptx.py | 180 ++++++- .../scripts/office/validators/redlining.py | 140 +++-- .../builtin_skills/xlsx/scripts/recalc.py | 212 ++++++-- .../tools/implementations/skills/builtin.rs | 250 +++++++++ .../core/tests/office_archive_safety.py | 117 +++++ 77 files changed, 4821 insertions(+), 6450 deletions(-) create mode 100644 src/crates/assembly/core/builtin-skills-upstreams.json delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/authentication.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/commands.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/profiling.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/proxy-support.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/session-management.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/snapshot-refs.md delete mode 100644 src/crates/assembly/core/builtin_skills/agent-browser/references/video-recording.md delete mode 100755 src/crates/assembly/core/builtin_skills/agent-browser/templates/authenticated-session.sh delete mode 100755 src/crates/assembly/core/builtin_skills/agent-browser/templates/capture-workflow.sh delete mode 100755 src/crates/assembly/core/builtin_skills/agent-browser/templates/form-automation.sh create mode 100755 src/crates/assembly/core/builtin_skills/docx/scripts/merge_runs.py delete mode 100644 src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/merge_runs.py create mode 100644 src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_chart.py create mode 100644 src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_slide.py create mode 100644 src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_theme.py delete mode 100644 src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/simplify_redlines.py delete mode 100755 src/crates/assembly/core/builtin_skills/docx/scripts/office/pack.py delete mode 100755 src/crates/assembly/core/builtin_skills/docx/scripts/office/unpack.py delete mode 100644 src/crates/assembly/core/builtin_skills/pptx/editing.md delete mode 100644 src/crates/assembly/core/builtin_skills/pptx/pptxgenjs.md delete mode 100644 src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/merge_runs.py create mode 100644 src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_chart.py create mode 100644 src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_slide.py create mode 100644 src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_theme.py delete mode 100644 src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/simplify_redlines.py delete mode 100755 src/crates/assembly/core/builtin_skills/pptx/scripts/office/pack.py delete mode 100755 src/crates/assembly/core/builtin_skills/pptx/scripts/office/unpack.py delete mode 100644 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/merge_runs.py create mode 100644 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_chart.py create mode 100644 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_slide.py create mode 100644 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_theme.py delete mode 100644 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/simplify_redlines.py delete mode 100755 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/pack.py delete mode 100755 src/crates/assembly/core/builtin_skills/xlsx/scripts/office/unpack.py create mode 100644 src/crates/assembly/core/tests/office_archive_safety.py diff --git a/src/crates/assembly/core/builtin-skills-upstreams.json b/src/crates/assembly/core/builtin-skills-upstreams.json new file mode 100644 index 0000000000..54dd13fc1b --- /dev/null +++ b/src/crates/assembly/core/builtin-skills-upstreams.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "synced_on": "2026-07-22", + "sources": [ + { + "skills": ["docx", "pptx", "xlsx"], + "repository": "https://github.com/anthropics/skills", + "revision": "fa0fa64bdc967915dc8399e803be67759e1e62b8", + "local_patches": [ + "normalize upstream trailing whitespace", + "route documented archive editing through safe_extract and rezip", + "bound archive extraction and reject normalized-path collisions", + "use BitFun as the default Word comment author", + "retain the BitFun 2026 year-format example" + ] + }, + { + "skills": ["agent-browser"], + "repository": "https://github.com/vercel-labs/agent-browser", + "revision": "81c336c1c20b80ac648e0416a7b6e0c0ae7878bb", + "package_version": "0.32.3", + "local_patches": [ + "route web and supported Electron work to agent-browser and native desktop work to BitFun ComputerUse", + "pin the documented install version and require user approval", + "preserve explicit missing-prerequisite and no-silent-fallback behavior" + ] + }, + { + "skills": ["gstack-*"], + "repository": "https://github.com/garrytan/gstack", + "revision": "7e96fe299b085010fb2e34d9c4fbfc7e44b617e1", + "revision_basis": "latest upstream commit before the BitFun import timestamp", + "bitfun_import_commit": "6341358703da217c3b2dd8e887ca877c35acaafc", + "local_patches": [ + "preserve existing BitFun Team Mode orchestration, Task, AGENTS.md, and .bitfun/team conventions", + "repair bundled skill references and remove references to absent workflows", + "replace pseudo browser commands with version-checked agent-browser commands", + "preserve BitFun ComputerUse for native desktop UI", + "avoid credential leakage and cross-target auth-profile reuse" + ] + } + ] +} diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md b/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md index 023e5319bf..a0ece3da22 100644 --- a/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md @@ -1,465 +1,53 @@ --- name: agent-browser -description: Browser automation CLI for AI agents. Use when the user needs to interact with websites, including navigating pages, filling forms, clicking buttons, taking screenshots, extracting data, testing web apps, or automating any browser task. Triggers include requests to "open a website", "fill out a form", "click a button", "take a screenshot", "scrape data from a page", "test this web app", "login to a site", "automate browser actions", or any task requiring programmatic web interaction. -allowed-tools: Bash(npx agent-browser:*), Bash(agent-browser:*) +description: Browser automation CLI for AI agents. Use when the user needs to interact with websites, including navigating pages, filling forms, clicking buttons, taking screenshots, extracting data, testing web apps, or automating any browser task. Triggers include requests to "open a website", "fill out a form", "click a button", "take a screenshot", "scrape data from a page", "test this web app", "login to a site", "automate browser actions", or any task requiring programmatic web interaction. Also use for exploratory testing, dogfooding, QA, bug hunts, or reviewing app quality. Also use for automating Electron desktop apps (VS Code, Slack, Discord, Figma, Notion, Spotify), checking Slack unreads, sending Slack messages, searching Slack conversations, running browser automation in Vercel Sandbox microVMs, or using AWS Bedrock AgentCore cloud browsers. Prefer agent-browser for web and supported Electron automation; use BitFun ComputerUse for native desktop UI that agent-browser cannot reach. +allowed-tools: Bash(agent-browser:*), Bash(npx agent-browser:*) +hidden: true --- -# Browser Automation with agent-browser +# agent-browser -## Prerequisites (required) +Fast browser automation CLI for AI agents. Chrome/Chromium via CDP with accessibility-tree snapshots and compact `@eN` element refs. -This skill relies on the external `agent-browser` CLI plus a local Chromium browser binary. +Install only after user approval: `npm i -g agent-browser@0.32.3 && agent-browser install` -Before using this skill, confirm prerequisites are satisfied: +If the CLI is unavailable and the user declines installation, explain the missing prerequisite and offer a non-browser fallback; do not silently switch tools. -1. `agent-browser` is available in PATH (or via `npx`) -2. Chromium is installed for Playwright (one-time download) +## Start here -If the CLI is missing, ask the user whether to install it (this may download binaries): +This file is a discovery stub, not the usage guide. Before running any `agent-browser` command, load the actual workflow content from the CLI: ```bash -# Option A: global install (recommended for repeated use) -npm install -g agent-browser - -# Option B: no global install (runs via npx) -npx agent-browser --version -``` - -Then install the browser binary (one-time download): - -```bash -agent-browser install -# or: -npx agent-browser install -``` - -Linux only (if Chromium fails to launch due to missing shared libraries): - -```bash -agent-browser install --with-deps -# or: -npx playwright install-deps chromium -``` - -If prerequisites are not available and the user does not want to install anything, do not silently switch tools. Tell the user what is missing and offer a non-browser fallback. - -## Core Workflow - -Every browser automation follows this pattern: - -1. **Navigate**: `agent-browser open ` -2. **Snapshot**: `agent-browser snapshot -i` (get element refs like `@e1`, `@e2`) -3. **Interact**: Use refs to click, fill, select -4. **Re-snapshot**: After navigation or DOM changes, get fresh refs - -```bash -agent-browser open https://example.com/form -agent-browser snapshot -i -# Output: @e1 [input type="email"], @e2 [input type="password"], @e3 [button] "Submit" - -agent-browser fill @e1 "user@example.com" -agent-browser fill @e2 "password123" -agent-browser click @e3 -agent-browser wait --load networkidle -agent-browser snapshot -i # Check result -``` - -## Command Chaining - -Commands can be chained with `&&` in a single shell invocation. The browser persists between commands via a background daemon, so chaining is safe and more efficient than separate calls. - -```bash -# Chain open + wait + snapshot in one call -agent-browser open https://example.com && agent-browser wait --load networkidle && agent-browser snapshot -i - -# Chain multiple interactions -agent-browser fill @e1 "user@example.com" && agent-browser fill @e2 "password123" && agent-browser click @e3 - -# Navigate and capture -agent-browser open https://example.com && agent-browser wait --load networkidle && agent-browser screenshot page.png -``` - -**When to chain:** Use `&&` when you don't need to read the output of an intermediate command before proceeding (e.g., open + wait + screenshot). Run commands separately when you need to parse the output first (e.g., snapshot to discover refs, then interact using those refs). - -## Essential Commands - -```bash -# Navigation -agent-browser open # Navigate (aliases: goto, navigate) -agent-browser close # Close browser - -# Snapshot -agent-browser snapshot -i # Interactive elements with refs (recommended) -agent-browser snapshot -i -C # Include cursor-interactive elements (divs with onclick, cursor:pointer) -agent-browser snapshot -s "#selector" # Scope to CSS selector - -# Interaction (use @refs from snapshot) -agent-browser click @e1 # Click element -agent-browser click @e1 --new-tab # Click and open in new tab -agent-browser fill @e2 "text" # Clear and type text -agent-browser type @e2 "text" # Type without clearing -agent-browser select @e1 "option" # Select dropdown option -agent-browser check @e1 # Check checkbox -agent-browser press Enter # Press key -agent-browser keyboard type "text" # Type at current focus (no selector) -agent-browser keyboard inserttext "text" # Insert without key events -agent-browser scroll down 500 # Scroll page - -# Get information -agent-browser get text @e1 # Get element text -agent-browser get url # Get current URL -agent-browser get title # Get page title - -# Wait -agent-browser wait @e1 # Wait for element -agent-browser wait --load networkidle # Wait for network idle -agent-browser wait --url "**/page" # Wait for URL pattern -agent-browser wait 2000 # Wait milliseconds - -# Capture -agent-browser screenshot # Screenshot to temp dir -agent-browser screenshot --full # Full page screenshot -agent-browser screenshot --annotate # Annotated screenshot with numbered element labels -agent-browser pdf output.pdf # Save as PDF - -# Diff (compare page states) -agent-browser diff snapshot # Compare current vs last snapshot -agent-browser diff snapshot --baseline before.txt # Compare current vs saved file -agent-browser diff screenshot --baseline before.png # Visual pixel diff -agent-browser diff url # Compare two pages -agent-browser diff url --wait-until networkidle # Custom wait strategy -agent-browser diff url --selector "#main" # Scope to element -``` - -## Common Patterns - -### Form Submission - -```bash -agent-browser open https://example.com/signup -agent-browser snapshot -i -agent-browser fill @e1 "Jane Doe" -agent-browser fill @e2 "jane@example.com" -agent-browser select @e3 "California" -agent-browser check @e4 -agent-browser click @e5 -agent-browser wait --load networkidle -``` - -### Authentication with State Persistence - -```bash -# Login once and save state -agent-browser open https://app.example.com/login -agent-browser snapshot -i -agent-browser fill @e1 "$USERNAME" -agent-browser fill @e2 "$PASSWORD" -agent-browser click @e3 -agent-browser wait --url "**/dashboard" -agent-browser state save auth.json - -# Reuse in future sessions -agent-browser state load auth.json -agent-browser open https://app.example.com/dashboard -``` - -### Session Persistence - -```bash -# Auto-save/restore cookies and localStorage across browser restarts -agent-browser --session-name myapp open https://app.example.com/login -# ... login flow ... -agent-browser close # State auto-saved to ~/.agent-browser/sessions/ - -# Next time, state is auto-loaded -agent-browser --session-name myapp open https://app.example.com/dashboard - -# Encrypt state at rest -export AGENT_BROWSER_ENCRYPTION_KEY=$(openssl rand -hex 32) -agent-browser --session-name secure open https://app.example.com - -# Manage saved states -agent-browser state list -agent-browser state show myapp-default.json -agent-browser state clear myapp -agent-browser state clean --older-than 7 -``` - -### Data Extraction - -```bash -agent-browser open https://example.com/products -agent-browser snapshot -i -agent-browser get text @e5 # Get specific element text -agent-browser get text body > page.txt # Get all page text - -# JSON output for parsing -agent-browser snapshot -i --json -agent-browser get text @e1 --json +agent-browser skills get core # start here — workflows, common patterns, troubleshooting +agent-browser skills get core --full # include full command reference and templates ``` -### Parallel Sessions +The CLI serves skill content that always matches the installed version, so instructions never go stale. The content in this stub cannot change between releases, which is why it just points at `skills get core`. -```bash -agent-browser --session site1 open https://site-a.com -agent-browser --session site2 open https://site-b.com - -agent-browser --session site1 snapshot -i -agent-browser --session site2 snapshot -i +## Specialized skills -agent-browser session list -``` - -### Connect to Existing Chrome +Load a specialized skill when the task falls outside browser web pages: ```bash -# Auto-discover running Chrome with remote debugging enabled -agent-browser --auto-connect open https://example.com -agent-browser --auto-connect snapshot - -# Or with explicit CDP port -agent-browser --cdp 9222 snapshot +agent-browser skills get electron # Electron desktop apps (VS Code, Slack, Discord, Figma, ...) +agent-browser skills get slack # Slack workspace automation +agent-browser skills get dogfood # Exploratory testing / QA / bug hunts +agent-browser skills get derive-client # Record a HAR, derive a standalone API client for a site +agent-browser skills get vercel-sandbox # agent-browser inside Vercel Sandbox microVMs +agent-browser skills get agentcore # AWS Bedrock AgentCore cloud browsers ``` -### Color Scheme (Dark Mode) - -```bash -# Persistent dark mode via flag (applies to all pages and new tabs) -agent-browser --color-scheme dark open https://example.com - -# Or via environment variable -AGENT_BROWSER_COLOR_SCHEME=dark agent-browser open https://example.com - -# Or set during session (persists for subsequent commands) -agent-browser set media dark -``` +Run `agent-browser skills list` to see everything available on the installed version. -### Visual Browser (Debugging) +## Why agent-browser -```bash -agent-browser --headed open https://example.com -agent-browser highlight @e1 # Highlight element -agent-browser record start demo.webm # Record session -agent-browser profiler start # Start Chrome DevTools profiling -agent-browser profiler stop trace.json # Stop and save profile (path optional) -``` - -### Local Files (PDFs, HTML) - -```bash -# Open local files with file:// URLs -agent-browser --allow-file-access open file:///path/to/document.pdf -agent-browser --allow-file-access open file:///path/to/page.html -agent-browser screenshot output.png -``` - -### iOS Simulator (Mobile Safari) - -```bash -# List available iOS simulators -agent-browser device list +- Fast native Rust CLI, not a Node.js wrapper +- Works with any AI agent (Cursor, Claude Code, Codex, Continue, Windsurf, etc.) +- Chrome/Chromium via CDP with no Playwright or Puppeteer dependency +- Accessibility-tree snapshots with element refs for reliable interaction +- Sessions, authentication vault, state persistence, video recording +- Specialized skills for Electron apps, Slack, exploratory testing, cloud providers -# Launch Safari on a specific device -agent-browser -p ios --device "iPhone 16 Pro" open https://example.com +## Observability Dashboard -# Same workflow as desktop - snapshot, interact, re-snapshot -agent-browser -p ios snapshot -i -agent-browser -p ios tap @e1 # Tap (alias for click) -agent-browser -p ios fill @e2 "text" -agent-browser -p ios swipe up # Mobile-specific gesture - -# Take screenshot -agent-browser -p ios screenshot mobile.png - -# Close session (shuts down simulator) -agent-browser -p ios close -``` - -**Requirements:** macOS with Xcode, Appium (`npm install -g appium && appium driver install xcuitest`) - -**Real devices:** Works with physical iOS devices if pre-configured. Use `--device ""` where UDID is from `xcrun xctrace list devices`. - -## Diffing (Verifying Changes) - -Use `diff snapshot` after performing an action to verify it had the intended effect. This compares the current accessibility tree against the last snapshot taken in the session. - -```bash -# Typical workflow: snapshot -> action -> diff -agent-browser snapshot -i # Take baseline snapshot -agent-browser click @e2 # Perform action -agent-browser diff snapshot # See what changed (auto-compares to last snapshot) -``` - -For visual regression testing or monitoring: - -```bash -# Save a baseline screenshot, then compare later -agent-browser screenshot baseline.png -# ... time passes or changes are made ... -agent-browser diff screenshot --baseline baseline.png - -# Compare staging vs production -agent-browser diff url https://staging.example.com https://prod.example.com --screenshot -``` - -`diff snapshot` output uses `+` for additions and `-` for removals, similar to git diff. `diff screenshot` produces a diff image with changed pixels highlighted in red, plus a mismatch percentage. - -## Timeouts and Slow Pages - -The default Playwright timeout is 25 seconds for local browsers. This can be overridden with the `AGENT_BROWSER_DEFAULT_TIMEOUT` environment variable (value in milliseconds). For slow websites or large pages, use explicit waits instead of relying on the default timeout: - -```bash -# Wait for network activity to settle (best for slow pages) -agent-browser wait --load networkidle - -# Wait for a specific element to appear -agent-browser wait "#content" -agent-browser wait @e1 - -# Wait for a specific URL pattern (useful after redirects) -agent-browser wait --url "**/dashboard" - -# Wait for a JavaScript condition -agent-browser wait --fn "document.readyState === 'complete'" - -# Wait a fixed duration (milliseconds) as a last resort -agent-browser wait 5000 -``` - -When dealing with consistently slow websites, use `wait --load networkidle` after `open` to ensure the page is fully loaded before taking a snapshot. If a specific element is slow to render, wait for it directly with `wait ` or `wait @ref`. - -## Session Management and Cleanup - -When running multiple agents or automations concurrently, always use named sessions to avoid conflicts: - -```bash -# Each agent gets its own isolated session -agent-browser --session agent1 open site-a.com -agent-browser --session agent2 open site-b.com - -# Check active sessions -agent-browser session list -``` - -Always close your browser session when done to avoid leaked processes: - -```bash -agent-browser close # Close default session -agent-browser --session agent1 close # Close specific session -``` - -If a previous session was not closed properly, the daemon may still be running. Use `agent-browser close` to clean it up before starting new work. - -## Ref Lifecycle (Important) - -Refs (`@e1`, `@e2`, etc.) are invalidated when the page changes. Always re-snapshot after: - -- Clicking links or buttons that navigate -- Form submissions -- Dynamic content loading (dropdowns, modals) - -```bash -agent-browser click @e5 # Navigates to new page -agent-browser snapshot -i # MUST re-snapshot -agent-browser click @e1 # Use new refs -``` - -## Annotated Screenshots (Vision Mode) - -Use `--annotate` to take a screenshot with numbered labels overlaid on interactive elements. Each label `[N]` maps to ref `@eN`. This also caches refs, so you can interact with elements immediately without a separate snapshot. - -```bash -agent-browser screenshot --annotate -# Output includes the image path and a legend: -# [1] @e1 button "Submit" -# [2] @e2 link "Home" -# [3] @e3 textbox "Email" -agent-browser click @e2 # Click using ref from annotated screenshot -``` - -Use annotated screenshots when: -- The page has unlabeled icon buttons or visual-only elements -- You need to verify visual layout or styling -- Canvas or chart elements are present (invisible to text snapshots) -- You need spatial reasoning about element positions - -## Semantic Locators (Alternative to Refs) - -When refs are unavailable or unreliable, use semantic locators: - -```bash -agent-browser find text "Sign In" click -agent-browser find label "Email" fill "user@test.com" -agent-browser find role button click --name "Submit" -agent-browser find placeholder "Search" type "query" -agent-browser find testid "submit-btn" click -``` - -## JavaScript Evaluation (eval) - -Use `eval` to run JavaScript in the browser context. **Shell quoting can corrupt complex expressions** -- use `--stdin` or `-b` to avoid issues. - -```bash -# Simple expressions work with regular quoting -agent-browser eval 'document.title' -agent-browser eval 'document.querySelectorAll("img").length' - -# Complex JS: use --stdin with heredoc (RECOMMENDED) -agent-browser eval --stdin <<'EVALEOF' -JSON.stringify( - Array.from(document.querySelectorAll("img")) - .filter(i => !i.alt) - .map(i => ({ src: i.src.split("/").pop(), width: i.width })) -) -EVALEOF - -# Alternative: base64 encoding (avoids all shell escaping issues) -agent-browser eval -b "$(echo -n 'Array.from(document.querySelectorAll("a")).map(a => a.href)' | base64)" -``` - -**Why this matters:** When the shell processes your command, inner double quotes, `!` characters (history expansion), backticks, and `$()` can all corrupt the JavaScript before it reaches agent-browser. The `--stdin` and `-b` flags bypass shell interpretation entirely. - -**Rules of thumb:** -- Single-line, no nested quotes -> regular `eval 'expression'` with single quotes is fine -- Nested quotes, arrow functions, template literals, or multiline -> use `eval --stdin <<'EVALEOF'` -- Programmatic/generated scripts -> use `eval -b` with base64 - -## Configuration File - -Create `agent-browser.json` in the project root for persistent settings: - -```json -{ - "headed": true, - "proxy": "http://localhost:8080", - "profile": "./browser-data" -} -``` - -Priority (lowest to highest): `~/.agent-browser/config.json` < `./agent-browser.json` < env vars < CLI flags. Use `--config ` or `AGENT_BROWSER_CONFIG` env var for a custom config file (exits with error if missing/invalid). All CLI options map to camelCase keys (e.g., `--executable-path` -> `"executablePath"`). Boolean flags accept `true`/`false` values (e.g., `--headed false` overrides config). Extensions from user and project configs are merged, not replaced. - -## Deep-Dive Documentation - -| Reference | When to Use | -|-----------|-------------| -| [references/commands.md](references/commands.md) | Full command reference with all options | -| [references/snapshot-refs.md](references/snapshot-refs.md) | Ref lifecycle, invalidation rules, troubleshooting | -| [references/session-management.md](references/session-management.md) | Parallel sessions, state persistence, concurrent scraping | -| [references/authentication.md](references/authentication.md) | Login flows, OAuth, 2FA handling, state reuse | -| [references/video-recording.md](references/video-recording.md) | Recording workflows for debugging and documentation | -| [references/profiling.md](references/profiling.md) | Chrome DevTools profiling for performance analysis | -| [references/proxy-support.md](references/proxy-support.md) | Proxy configuration, geo-testing, rotating proxies | - -## Ready-to-Use Templates - -| Template | Description | -|----------|-------------| -| [templates/form-automation.sh](templates/form-automation.sh) | Form filling with validation | -| [templates/authenticated-session.sh](templates/authenticated-session.sh) | Login once, reuse state | -| [templates/capture-workflow.sh](templates/capture-workflow.sh) | Content extraction with screenshots | - -```bash -./templates/form-automation.sh https://example.com/form -./templates/authenticated-session.sh https://app.example.com/login -./templates/capture-workflow.sh https://example.com ./output -``` +The dashboard runs independently of browser sessions on port 4848 and can also be opened through a proxied or forwarded URL such as `https://dashboard.agent-browser.localhost`. Agents should stay on the dashboard origin: session tabs, status, and stream traffic are proxied internally, so session ports do not need to be exposed. diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/authentication.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/authentication.md deleted file mode 100644 index 12ef5e41be..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/authentication.md +++ /dev/null @@ -1,202 +0,0 @@ -# Authentication Patterns - -Login flows, session persistence, OAuth, 2FA, and authenticated browsing. - -**Related**: [session-management.md](session-management.md) for state persistence details, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [Basic Login Flow](#basic-login-flow) -- [Saving Authentication State](#saving-authentication-state) -- [Restoring Authentication](#restoring-authentication) -- [OAuth / SSO Flows](#oauth--sso-flows) -- [Two-Factor Authentication](#two-factor-authentication) -- [HTTP Basic Auth](#http-basic-auth) -- [Cookie-Based Auth](#cookie-based-auth) -- [Token Refresh Handling](#token-refresh-handling) -- [Security Best Practices](#security-best-practices) - -## Basic Login Flow - -```bash -# Navigate to login page -agent-browser open https://app.example.com/login -agent-browser wait --load networkidle - -# Get form elements -agent-browser snapshot -i -# Output: @e1 [input type="email"], @e2 [input type="password"], @e3 [button] "Sign In" - -# Fill credentials -agent-browser fill @e1 "user@example.com" -agent-browser fill @e2 "password123" - -# Submit -agent-browser click @e3 -agent-browser wait --load networkidle - -# Verify login succeeded -agent-browser get url # Should be dashboard, not login -``` - -## Saving Authentication State - -After logging in, save state for reuse: - -```bash -# Login first (see above) -agent-browser open https://app.example.com/login -agent-browser snapshot -i -agent-browser fill @e1 "user@example.com" -agent-browser fill @e2 "password123" -agent-browser click @e3 -agent-browser wait --url "**/dashboard" - -# Save authenticated state -agent-browser state save ./auth-state.json -``` - -## Restoring Authentication - -Skip login by loading saved state: - -```bash -# Load saved auth state -agent-browser state load ./auth-state.json - -# Navigate directly to protected page -agent-browser open https://app.example.com/dashboard - -# Verify authenticated -agent-browser snapshot -i -``` - -## OAuth / SSO Flows - -For OAuth redirects: - -```bash -# Start OAuth flow -agent-browser open https://app.example.com/auth/google - -# Handle redirects automatically -agent-browser wait --url "**/accounts.google.com**" -agent-browser snapshot -i - -# Fill Google credentials -agent-browser fill @e1 "user@gmail.com" -agent-browser click @e2 # Next button -agent-browser wait 2000 -agent-browser snapshot -i -agent-browser fill @e3 "password" -agent-browser click @e4 # Sign in - -# Wait for redirect back -agent-browser wait --url "**/app.example.com**" -agent-browser state save ./oauth-state.json -``` - -## Two-Factor Authentication - -Handle 2FA with manual intervention: - -```bash -# Login with credentials -agent-browser open https://app.example.com/login --headed # Show browser -agent-browser snapshot -i -agent-browser fill @e1 "user@example.com" -agent-browser fill @e2 "password123" -agent-browser click @e3 - -# Wait for user to complete 2FA manually -echo "Complete 2FA in the browser window..." -agent-browser wait --url "**/dashboard" --timeout 120000 - -# Save state after 2FA -agent-browser state save ./2fa-state.json -``` - -## HTTP Basic Auth - -For sites using HTTP Basic Authentication: - -```bash -# Set credentials before navigation -agent-browser set credentials username password - -# Navigate to protected resource -agent-browser open https://protected.example.com/api -``` - -## Cookie-Based Auth - -Manually set authentication cookies: - -```bash -# Set auth cookie -agent-browser cookies set session_token "abc123xyz" - -# Navigate to protected page -agent-browser open https://app.example.com/dashboard -``` - -## Token Refresh Handling - -For sessions with expiring tokens: - -```bash -#!/bin/bash -# Wrapper that handles token refresh - -STATE_FILE="./auth-state.json" - -# Try loading existing state -if [[ -f "$STATE_FILE" ]]; then - agent-browser state load "$STATE_FILE" - agent-browser open https://app.example.com/dashboard - - # Check if session is still valid - URL=$(agent-browser get url) - if [[ "$URL" == *"/login"* ]]; then - echo "Session expired, re-authenticating..." - # Perform fresh login - agent-browser snapshot -i - agent-browser fill @e1 "$USERNAME" - agent-browser fill @e2 "$PASSWORD" - agent-browser click @e3 - agent-browser wait --url "**/dashboard" - agent-browser state save "$STATE_FILE" - fi -else - # First-time login - agent-browser open https://app.example.com/login - # ... login flow ... -fi -``` - -## Security Best Practices - -1. **Never commit state files** - They contain session tokens - ```bash - echo "*.auth-state.json" >> .gitignore - ``` - -2. **Use environment variables for credentials** - ```bash - agent-browser fill @e1 "$APP_USERNAME" - agent-browser fill @e2 "$APP_PASSWORD" - ``` - -3. **Clean up after automation** - ```bash - agent-browser cookies clear - rm -f ./auth-state.json - ``` - -4. **Use short-lived sessions for CI/CD** - ```bash - # Don't persist state in CI - agent-browser open https://app.example.com/login - # ... login and perform actions ... - agent-browser close # Session ends, nothing persisted - ``` diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/commands.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/commands.md deleted file mode 100644 index e77196cdd3..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/commands.md +++ /dev/null @@ -1,263 +0,0 @@ -# Command Reference - -Complete reference for all agent-browser commands. For quick start and common patterns, see SKILL.md. - -## Navigation - -```bash -agent-browser open # Navigate to URL (aliases: goto, navigate) - # Supports: https://, http://, file://, about:, data:// - # Auto-prepends https:// if no protocol given -agent-browser back # Go back -agent-browser forward # Go forward -agent-browser reload # Reload page -agent-browser close # Close browser (aliases: quit, exit) -agent-browser connect 9222 # Connect to browser via CDP port -``` - -## Snapshot (page analysis) - -```bash -agent-browser snapshot # Full accessibility tree -agent-browser snapshot -i # Interactive elements only (recommended) -agent-browser snapshot -c # Compact output -agent-browser snapshot -d 3 # Limit depth to 3 -agent-browser snapshot -s "#main" # Scope to CSS selector -``` - -## Interactions (use @refs from snapshot) - -```bash -agent-browser click @e1 # Click -agent-browser click @e1 --new-tab # Click and open in new tab -agent-browser dblclick @e1 # Double-click -agent-browser focus @e1 # Focus element -agent-browser fill @e2 "text" # Clear and type -agent-browser type @e2 "text" # Type without clearing -agent-browser press Enter # Press key (alias: key) -agent-browser press Control+a # Key combination -agent-browser keydown Shift # Hold key down -agent-browser keyup Shift # Release key -agent-browser hover @e1 # Hover -agent-browser check @e1 # Check checkbox -agent-browser uncheck @e1 # Uncheck checkbox -agent-browser select @e1 "value" # Select dropdown option -agent-browser select @e1 "a" "b" # Select multiple options -agent-browser scroll down 500 # Scroll page (default: down 300px) -agent-browser scrollintoview @e1 # Scroll element into view (alias: scrollinto) -agent-browser drag @e1 @e2 # Drag and drop -agent-browser upload @e1 file.pdf # Upload files -``` - -## Get Information - -```bash -agent-browser get text @e1 # Get element text -agent-browser get html @e1 # Get innerHTML -agent-browser get value @e1 # Get input value -agent-browser get attr @e1 href # Get attribute -agent-browser get title # Get page title -agent-browser get url # Get current URL -agent-browser get count ".item" # Count matching elements -agent-browser get box @e1 # Get bounding box -agent-browser get styles @e1 # Get computed styles (font, color, bg, etc.) -``` - -## Check State - -```bash -agent-browser is visible @e1 # Check if visible -agent-browser is enabled @e1 # Check if enabled -agent-browser is checked @e1 # Check if checked -``` - -## Screenshots and PDF - -```bash -agent-browser screenshot # Save to temporary directory -agent-browser screenshot path.png # Save to specific path -agent-browser screenshot --full # Full page -agent-browser pdf output.pdf # Save as PDF -``` - -## Video Recording - -```bash -agent-browser record start ./demo.webm # Start recording -agent-browser click @e1 # Perform actions -agent-browser record stop # Stop and save video -agent-browser record restart ./take2.webm # Stop current + start new -``` - -## Wait - -```bash -agent-browser wait @e1 # Wait for element -agent-browser wait 2000 # Wait milliseconds -agent-browser wait --text "Success" # Wait for text (or -t) -agent-browser wait --url "**/dashboard" # Wait for URL pattern (or -u) -agent-browser wait --load networkidle # Wait for network idle (or -l) -agent-browser wait --fn "window.ready" # Wait for JS condition (or -f) -``` - -## Mouse Control - -```bash -agent-browser mouse move 100 200 # Move mouse -agent-browser mouse down left # Press button -agent-browser mouse up left # Release button -agent-browser mouse wheel 100 # Scroll wheel -``` - -## Semantic Locators (alternative to refs) - -```bash -agent-browser find role button click --name "Submit" -agent-browser find text "Sign In" click -agent-browser find text "Sign In" click --exact # Exact match only -agent-browser find label "Email" fill "user@test.com" -agent-browser find placeholder "Search" type "query" -agent-browser find alt "Logo" click -agent-browser find title "Close" click -agent-browser find testid "submit-btn" click -agent-browser find first ".item" click -agent-browser find last ".item" click -agent-browser find nth 2 "a" hover -``` - -## Browser Settings - -```bash -agent-browser set viewport 1920 1080 # Set viewport size -agent-browser set device "iPhone 14" # Emulate device -agent-browser set geo 37.7749 -122.4194 # Set geolocation (alias: geolocation) -agent-browser set offline on # Toggle offline mode -agent-browser set headers '{"X-Key":"v"}' # Extra HTTP headers -agent-browser set credentials user pass # HTTP basic auth (alias: auth) -agent-browser set media dark # Emulate color scheme -agent-browser set media light reduced-motion # Light mode + reduced motion -``` - -## Cookies and Storage - -```bash -agent-browser cookies # Get all cookies -agent-browser cookies set name value # Set cookie -agent-browser cookies clear # Clear cookies -agent-browser storage local # Get all localStorage -agent-browser storage local key # Get specific key -agent-browser storage local set k v # Set value -agent-browser storage local clear # Clear all -``` - -## Network - -```bash -agent-browser network route # Intercept requests -agent-browser network route --abort # Block requests -agent-browser network route --body '{}' # Mock response -agent-browser network unroute [url] # Remove routes -agent-browser network requests # View tracked requests -agent-browser network requests --filter api # Filter requests -``` - -## Tabs and Windows - -```bash -agent-browser tab # List tabs -agent-browser tab new [url] # New tab -agent-browser tab 2 # Switch to tab by index -agent-browser tab close # Close current tab -agent-browser tab close 2 # Close tab by index -agent-browser window new # New window -``` - -## Frames - -```bash -agent-browser frame "#iframe" # Switch to iframe -agent-browser frame main # Back to main frame -``` - -## Dialogs - -```bash -agent-browser dialog accept [text] # Accept dialog -agent-browser dialog dismiss # Dismiss dialog -``` - -## JavaScript - -```bash -agent-browser eval "document.title" # Simple expressions only -agent-browser eval -b "" # Any JavaScript (base64 encoded) -agent-browser eval --stdin # Read script from stdin -``` - -Use `-b`/`--base64` or `--stdin` for reliable execution. Shell escaping with nested quotes and special characters is error-prone. - -```bash -# Base64 encode your script, then: -agent-browser eval -b "ZG9jdW1lbnQucXVlcnlTZWxlY3RvcignW3NyYyo9Il9uZXh0Il0nKQ==" - -# Or use stdin with heredoc for multiline scripts: -cat <<'EOF' | agent-browser eval --stdin -const links = document.querySelectorAll('a'); -Array.from(links).map(a => a.href); -EOF -``` - -## State Management - -```bash -agent-browser state save auth.json # Save cookies, storage, auth state -agent-browser state load auth.json # Restore saved state -``` - -## Global Options - -```bash -agent-browser --session ... # Isolated browser session -agent-browser --json ... # JSON output for parsing -agent-browser --headed ... # Show browser window (not headless) -agent-browser --full ... # Full page screenshot (-f) -agent-browser --cdp ... # Connect via Chrome DevTools Protocol -agent-browser -p ... # Cloud browser provider (--provider) -agent-browser --proxy ... # Use proxy server -agent-browser --proxy-bypass # Hosts to bypass proxy -agent-browser --headers ... # HTTP headers scoped to URL's origin -agent-browser --executable-path

# Custom browser executable -agent-browser --extension ... # Load browser extension (repeatable) -agent-browser --ignore-https-errors # Ignore SSL certificate errors -agent-browser --help # Show help (-h) -agent-browser --version # Show version (-V) -agent-browser --help # Show detailed help for a command -``` - -## Debugging - -```bash -agent-browser --headed open example.com # Show browser window -agent-browser --cdp 9222 snapshot # Connect via CDP port -agent-browser connect 9222 # Alternative: connect command -agent-browser console # View console messages -agent-browser console --clear # Clear console -agent-browser errors # View page errors -agent-browser errors --clear # Clear errors -agent-browser highlight @e1 # Highlight element -agent-browser trace start # Start recording trace -agent-browser trace stop trace.zip # Stop and save trace -agent-browser profiler start # Start Chrome DevTools profiling -agent-browser profiler stop trace.json # Stop and save profile -``` - -## Environment Variables - -```bash -AGENT_BROWSER_SESSION="mysession" # Default session name -AGENT_BROWSER_EXECUTABLE_PATH="/path/chrome" # Custom browser path -AGENT_BROWSER_EXTENSIONS="/ext1,/ext2" # Comma-separated extension paths -AGENT_BROWSER_PROVIDER="browserbase" # Cloud browser provider -AGENT_BROWSER_STREAM_PORT="9223" # WebSocket streaming port -AGENT_BROWSER_HOME="/path/to/agent-browser" # Custom install location -``` diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/profiling.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/profiling.md deleted file mode 100644 index bd47eaa0ce..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/profiling.md +++ /dev/null @@ -1,120 +0,0 @@ -# Profiling - -Capture Chrome DevTools performance profiles during browser automation for performance analysis. - -**Related**: [commands.md](commands.md) for full command reference, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [Basic Profiling](#basic-profiling) -- [Profiler Commands](#profiler-commands) -- [Categories](#categories) -- [Use Cases](#use-cases) -- [Output Format](#output-format) -- [Viewing Profiles](#viewing-profiles) -- [Limitations](#limitations) - -## Basic Profiling - -```bash -# Start profiling -agent-browser profiler start - -# Perform actions -agent-browser navigate https://example.com -agent-browser click "#button" -agent-browser wait 1000 - -# Stop and save -agent-browser profiler stop ./trace.json -``` - -## Profiler Commands - -```bash -# Start profiling with default categories -agent-browser profiler start - -# Start with custom trace categories -agent-browser profiler start --categories "devtools.timeline,v8.execute,blink.user_timing" - -# Stop profiling and save to file -agent-browser profiler stop ./trace.json -``` - -## Categories - -The `--categories` flag accepts a comma-separated list of Chrome trace categories. Default categories include: - -- `devtools.timeline` -- standard DevTools performance traces -- `v8.execute` -- time spent running JavaScript -- `blink` -- renderer events -- `blink.user_timing` -- `performance.mark()` / `performance.measure()` calls -- `latencyInfo` -- input-to-latency tracking -- `renderer.scheduler` -- task scheduling and execution -- `toplevel` -- broad-spectrum basic events - -Several `disabled-by-default-*` categories are also included for detailed timeline, call stack, and V8 CPU profiling data. - -## Use Cases - -### Diagnosing Slow Page Loads - -```bash -agent-browser profiler start -agent-browser navigate https://app.example.com -agent-browser wait --load networkidle -agent-browser profiler stop ./page-load-profile.json -``` - -### Profiling User Interactions - -```bash -agent-browser navigate https://app.example.com -agent-browser profiler start -agent-browser click "#submit" -agent-browser wait 2000 -agent-browser profiler stop ./interaction-profile.json -``` - -### CI Performance Regression Checks - -```bash -#!/bin/bash -agent-browser profiler start -agent-browser navigate https://app.example.com -agent-browser wait --load networkidle -agent-browser profiler stop "./profiles/build-${BUILD_ID}.json" -``` - -## Output Format - -The output is a JSON file in Chrome Trace Event format: - -```json -{ - "traceEvents": [ - { "cat": "devtools.timeline", "name": "RunTask", "ph": "X", "ts": 12345, "dur": 100, ... }, - ... - ], - "metadata": { - "clock-domain": "LINUX_CLOCK_MONOTONIC" - } -} -``` - -The `metadata.clock-domain` field is set based on the host platform (Linux or macOS). On Windows it is omitted. - -## Viewing Profiles - -Load the output JSON file in any of these tools: - -- **Chrome DevTools**: Performance panel > Load profile (Ctrl+Shift+I > Performance) -- **Perfetto UI**: https://ui.perfetto.dev/ -- drag and drop the JSON file -- **Trace Viewer**: `chrome://tracing` in any Chromium browser - -## Limitations - -- Only works with Chromium-based browsers (Chrome, Edge). Not supported on Firefox or WebKit. -- Trace data accumulates in memory while profiling is active (capped at 5 million events). Stop profiling promptly after the area of interest. -- Data collection on stop has a 30-second timeout. If the browser is unresponsive, the stop command may fail. diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/proxy-support.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/proxy-support.md deleted file mode 100644 index e86a8fe33e..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/proxy-support.md +++ /dev/null @@ -1,194 +0,0 @@ -# Proxy Support - -Proxy configuration for geo-testing, rate limiting avoidance, and corporate environments. - -**Related**: [commands.md](commands.md) for global options, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [Basic Proxy Configuration](#basic-proxy-configuration) -- [Authenticated Proxy](#authenticated-proxy) -- [SOCKS Proxy](#socks-proxy) -- [Proxy Bypass](#proxy-bypass) -- [Common Use Cases](#common-use-cases) -- [Verifying Proxy Connection](#verifying-proxy-connection) -- [Troubleshooting](#troubleshooting) -- [Best Practices](#best-practices) - -## Basic Proxy Configuration - -Use the `--proxy` flag or set proxy via environment variable: - -```bash -# Via CLI flag -agent-browser --proxy "http://proxy.example.com:8080" open https://example.com - -# Via environment variable -export HTTP_PROXY="http://proxy.example.com:8080" -agent-browser open https://example.com - -# HTTPS proxy -export HTTPS_PROXY="https://proxy.example.com:8080" -agent-browser open https://example.com - -# Both -export HTTP_PROXY="http://proxy.example.com:8080" -export HTTPS_PROXY="http://proxy.example.com:8080" -agent-browser open https://example.com -``` - -## Authenticated Proxy - -For proxies requiring authentication: - -```bash -# Include credentials in URL -export HTTP_PROXY="http://username:password@proxy.example.com:8080" -agent-browser open https://example.com -``` - -## SOCKS Proxy - -```bash -# SOCKS5 proxy -export ALL_PROXY="socks5://proxy.example.com:1080" -agent-browser open https://example.com - -# SOCKS5 with auth -export ALL_PROXY="socks5://user:pass@proxy.example.com:1080" -agent-browser open https://example.com -``` - -## Proxy Bypass - -Skip proxy for specific domains using `--proxy-bypass` or `NO_PROXY`: - -```bash -# Via CLI flag -agent-browser --proxy "http://proxy.example.com:8080" --proxy-bypass "localhost,*.internal.com" open https://example.com - -# Via environment variable -export NO_PROXY="localhost,127.0.0.1,.internal.company.com" -agent-browser open https://internal.company.com # Direct connection -agent-browser open https://external.com # Via proxy -``` - -## Common Use Cases - -### Geo-Location Testing - -```bash -#!/bin/bash -# Test site from different regions using geo-located proxies - -PROXIES=( - "http://us-proxy.example.com:8080" - "http://eu-proxy.example.com:8080" - "http://asia-proxy.example.com:8080" -) - -for proxy in "${PROXIES[@]}"; do - export HTTP_PROXY="$proxy" - export HTTPS_PROXY="$proxy" - - region=$(echo "$proxy" | grep -oP '^\w+-\w+') - echo "Testing from: $region" - - agent-browser --session "$region" open https://example.com - agent-browser --session "$region" screenshot "./screenshots/$region.png" - agent-browser --session "$region" close -done -``` - -### Rotating Proxies for Scraping - -```bash -#!/bin/bash -# Rotate through proxy list to avoid rate limiting - -PROXY_LIST=( - "http://proxy1.example.com:8080" - "http://proxy2.example.com:8080" - "http://proxy3.example.com:8080" -) - -URLS=( - "https://site.com/page1" - "https://site.com/page2" - "https://site.com/page3" -) - -for i in "${!URLS[@]}"; do - proxy_index=$((i % ${#PROXY_LIST[@]})) - export HTTP_PROXY="${PROXY_LIST[$proxy_index]}" - export HTTPS_PROXY="${PROXY_LIST[$proxy_index]}" - - agent-browser open "${URLS[$i]}" - agent-browser get text body > "output-$i.txt" - agent-browser close - - sleep 1 # Polite delay -done -``` - -### Corporate Network Access - -```bash -#!/bin/bash -# Access internal sites via corporate proxy - -export HTTP_PROXY="http://corpproxy.company.com:8080" -export HTTPS_PROXY="http://corpproxy.company.com:8080" -export NO_PROXY="localhost,127.0.0.1,.company.com" - -# External sites go through proxy -agent-browser open https://external-vendor.com - -# Internal sites bypass proxy -agent-browser open https://intranet.company.com -``` - -## Verifying Proxy Connection - -```bash -# Check your apparent IP -agent-browser open https://httpbin.org/ip -agent-browser get text body -# Should show proxy's IP, not your real IP -``` - -## Troubleshooting - -### Proxy Connection Failed - -```bash -# Test proxy connectivity first -curl -x http://proxy.example.com:8080 https://httpbin.org/ip - -# Check if proxy requires auth -export HTTP_PROXY="http://user:pass@proxy.example.com:8080" -``` - -### SSL/TLS Errors Through Proxy - -Some proxies perform SSL inspection. If you encounter certificate errors: - -```bash -# For testing only - not recommended for production -agent-browser open https://example.com --ignore-https-errors -``` - -### Slow Performance - -```bash -# Use proxy only when necessary -export NO_PROXY="*.cdn.com,*.static.com" # Direct CDN access -``` - -## Best Practices - -1. **Use environment variables** - Don't hardcode proxy credentials -2. **Set NO_PROXY appropriately** - Avoid routing local traffic through proxy -3. **Test proxy before automation** - Verify connectivity with simple requests -4. **Handle proxy failures gracefully** - Implement retry logic for unstable proxies -5. **Rotate proxies for large scraping jobs** - Distribute load and avoid bans diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/session-management.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/session-management.md deleted file mode 100644 index bb5312dbdb..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/session-management.md +++ /dev/null @@ -1,193 +0,0 @@ -# Session Management - -Multiple isolated browser sessions with state persistence and concurrent browsing. - -**Related**: [authentication.md](authentication.md) for login patterns, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [Named Sessions](#named-sessions) -- [Session Isolation Properties](#session-isolation-properties) -- [Session State Persistence](#session-state-persistence) -- [Common Patterns](#common-patterns) -- [Default Session](#default-session) -- [Session Cleanup](#session-cleanup) -- [Best Practices](#best-practices) - -## Named Sessions - -Use `--session` flag to isolate browser contexts: - -```bash -# Session 1: Authentication flow -agent-browser --session auth open https://app.example.com/login - -# Session 2: Public browsing (separate cookies, storage) -agent-browser --session public open https://example.com - -# Commands are isolated by session -agent-browser --session auth fill @e1 "user@example.com" -agent-browser --session public get text body -``` - -## Session Isolation Properties - -Each session has independent: -- Cookies -- LocalStorage / SessionStorage -- IndexedDB -- Cache -- Browsing history -- Open tabs - -## Session State Persistence - -### Save Session State - -```bash -# Save cookies, storage, and auth state -agent-browser state save /path/to/auth-state.json -``` - -### Load Session State - -```bash -# Restore saved state -agent-browser state load /path/to/auth-state.json - -# Continue with authenticated session -agent-browser open https://app.example.com/dashboard -``` - -### State File Contents - -```json -{ - "cookies": [...], - "localStorage": {...}, - "sessionStorage": {...}, - "origins": [...] -} -``` - -## Common Patterns - -### Authenticated Session Reuse - -```bash -#!/bin/bash -# Save login state once, reuse many times - -STATE_FILE="/tmp/auth-state.json" - -# Check if we have saved state -if [[ -f "$STATE_FILE" ]]; then - agent-browser state load "$STATE_FILE" - agent-browser open https://app.example.com/dashboard -else - # Perform login - agent-browser open https://app.example.com/login - agent-browser snapshot -i - agent-browser fill @e1 "$USERNAME" - agent-browser fill @e2 "$PASSWORD" - agent-browser click @e3 - agent-browser wait --load networkidle - - # Save for future use - agent-browser state save "$STATE_FILE" -fi -``` - -### Concurrent Scraping - -```bash -#!/bin/bash -# Scrape multiple sites concurrently - -# Start all sessions -agent-browser --session site1 open https://site1.com & -agent-browser --session site2 open https://site2.com & -agent-browser --session site3 open https://site3.com & -wait - -# Extract from each -agent-browser --session site1 get text body > site1.txt -agent-browser --session site2 get text body > site2.txt -agent-browser --session site3 get text body > site3.txt - -# Cleanup -agent-browser --session site1 close -agent-browser --session site2 close -agent-browser --session site3 close -``` - -### A/B Testing Sessions - -```bash -# Test different user experiences -agent-browser --session variant-a open "https://app.com?variant=a" -agent-browser --session variant-b open "https://app.com?variant=b" - -# Compare -agent-browser --session variant-a screenshot /tmp/variant-a.png -agent-browser --session variant-b screenshot /tmp/variant-b.png -``` - -## Default Session - -When `--session` is omitted, commands use the default session: - -```bash -# These use the same default session -agent-browser open https://example.com -agent-browser snapshot -i -agent-browser close # Closes default session -``` - -## Session Cleanup - -```bash -# Close specific session -agent-browser --session auth close - -# List active sessions -agent-browser session list -``` - -## Best Practices - -### 1. Name Sessions Semantically - -```bash -# GOOD: Clear purpose -agent-browser --session github-auth open https://github.com -agent-browser --session docs-scrape open https://docs.example.com - -# AVOID: Generic names -agent-browser --session s1 open https://github.com -``` - -### 2. Always Clean Up - -```bash -# Close sessions when done -agent-browser --session auth close -agent-browser --session scrape close -``` - -### 3. Handle State Files Securely - -```bash -# Don't commit state files (contain auth tokens!) -echo "*.auth-state.json" >> .gitignore - -# Delete after use -rm /tmp/auth-state.json -``` - -### 4. Timeout Long Sessions - -```bash -# Set timeout for automated scripts -timeout 60 agent-browser --session long-task get text body -``` diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/snapshot-refs.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/snapshot-refs.md deleted file mode 100644 index c5868d51cf..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/snapshot-refs.md +++ /dev/null @@ -1,194 +0,0 @@ -# Snapshot and Refs - -Compact element references that reduce context usage dramatically for AI agents. - -**Related**: [commands.md](commands.md) for full command reference, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [How Refs Work](#how-refs-work) -- [Snapshot Command](#the-snapshot-command) -- [Using Refs](#using-refs) -- [Ref Lifecycle](#ref-lifecycle) -- [Best Practices](#best-practices) -- [Ref Notation Details](#ref-notation-details) -- [Troubleshooting](#troubleshooting) - -## How Refs Work - -Traditional approach: -``` -Full DOM/HTML → AI parses → CSS selector → Action (~3000-5000 tokens) -``` - -agent-browser approach: -``` -Compact snapshot → @refs assigned → Direct interaction (~200-400 tokens) -``` - -## The Snapshot Command - -```bash -# Basic snapshot (shows page structure) -agent-browser snapshot - -# Interactive snapshot (-i flag) - RECOMMENDED -agent-browser snapshot -i -``` - -### Snapshot Output Format - -``` -Page: Example Site - Home -URL: https://example.com - -@e1 [header] - @e2 [nav] - @e3 [a] "Home" - @e4 [a] "Products" - @e5 [a] "About" - @e6 [button] "Sign In" - -@e7 [main] - @e8 [h1] "Welcome" - @e9 [form] - @e10 [input type="email"] placeholder="Email" - @e11 [input type="password"] placeholder="Password" - @e12 [button type="submit"] "Log In" - -@e13 [footer] - @e14 [a] "Privacy Policy" -``` - -## Using Refs - -Once you have refs, interact directly: - -```bash -# Click the "Sign In" button -agent-browser click @e6 - -# Fill email input -agent-browser fill @e10 "user@example.com" - -# Fill password -agent-browser fill @e11 "password123" - -# Submit the form -agent-browser click @e12 -``` - -## Ref Lifecycle - -**IMPORTANT**: Refs are invalidated when the page changes! - -```bash -# Get initial snapshot -agent-browser snapshot -i -# @e1 [button] "Next" - -# Click triggers page change -agent-browser click @e1 - -# MUST re-snapshot to get new refs! -agent-browser snapshot -i -# @e1 [h1] "Page 2" ← Different element now! -``` - -## Best Practices - -### 1. Always Snapshot Before Interacting - -```bash -# CORRECT -agent-browser open https://example.com -agent-browser snapshot -i # Get refs first -agent-browser click @e1 # Use ref - -# WRONG -agent-browser open https://example.com -agent-browser click @e1 # Ref doesn't exist yet! -``` - -### 2. Re-Snapshot After Navigation - -```bash -agent-browser click @e5 # Navigates to new page -agent-browser snapshot -i # Get new refs -agent-browser click @e1 # Use new refs -``` - -### 3. Re-Snapshot After Dynamic Changes - -```bash -agent-browser click @e1 # Opens dropdown -agent-browser snapshot -i # See dropdown items -agent-browser click @e7 # Select item -``` - -### 4. Snapshot Specific Regions - -For complex pages, snapshot specific areas: - -```bash -# Snapshot just the form -agent-browser snapshot @e9 -``` - -## Ref Notation Details - -``` -@e1 [tag type="value"] "text content" placeholder="hint" -│ │ │ │ │ -│ │ │ │ └─ Additional attributes -│ │ │ └─ Visible text -│ │ └─ Key attributes shown -│ └─ HTML tag name -└─ Unique ref ID -``` - -### Common Patterns - -``` -@e1 [button] "Submit" # Button with text -@e2 [input type="email"] # Email input -@e3 [input type="password"] # Password input -@e4 [a href="/page"] "Link Text" # Anchor link -@e5 [select] # Dropdown -@e6 [textarea] placeholder="Message" # Text area -@e7 [div class="modal"] # Container (when relevant) -@e8 [img alt="Logo"] # Image -@e9 [checkbox] checked # Checked checkbox -@e10 [radio] selected # Selected radio -``` - -## Troubleshooting - -### "Ref not found" Error - -```bash -# Ref may have changed - re-snapshot -agent-browser snapshot -i -``` - -### Element Not Visible in Snapshot - -```bash -# Scroll down to reveal element -agent-browser scroll down 1000 -agent-browser snapshot -i - -# Or wait for dynamic content -agent-browser wait 1000 -agent-browser snapshot -i -``` - -### Too Many Elements - -```bash -# Snapshot specific container -agent-browser snapshot @e5 - -# Or use get text for content-only extraction -agent-browser get text @e5 -``` diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/references/video-recording.md b/src/crates/assembly/core/builtin_skills/agent-browser/references/video-recording.md deleted file mode 100644 index e6a9fb4e2f..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/references/video-recording.md +++ /dev/null @@ -1,173 +0,0 @@ -# Video Recording - -Capture browser automation as video for debugging, documentation, or verification. - -**Related**: [commands.md](commands.md) for full command reference, [SKILL.md](../SKILL.md) for quick start. - -## Contents - -- [Basic Recording](#basic-recording) -- [Recording Commands](#recording-commands) -- [Use Cases](#use-cases) -- [Best Practices](#best-practices) -- [Output Format](#output-format) -- [Limitations](#limitations) - -## Basic Recording - -```bash -# Start recording -agent-browser record start ./demo.webm - -# Perform actions -agent-browser open https://example.com -agent-browser snapshot -i -agent-browser click @e1 -agent-browser fill @e2 "test input" - -# Stop and save -agent-browser record stop -``` - -## Recording Commands - -```bash -# Start recording to file -agent-browser record start ./output.webm - -# Stop current recording -agent-browser record stop - -# Restart with new file (stops current + starts new) -agent-browser record restart ./take2.webm -``` - -## Use Cases - -### Debugging Failed Automation - -```bash -#!/bin/bash -# Record automation for debugging - -agent-browser record start ./debug-$(date +%Y%m%d-%H%M%S).webm - -# Run your automation -agent-browser open https://app.example.com -agent-browser snapshot -i -agent-browser click @e1 || { - echo "Click failed - check recording" - agent-browser record stop - exit 1 -} - -agent-browser record stop -``` - -### Documentation Generation - -```bash -#!/bin/bash -# Record workflow for documentation - -agent-browser record start ./docs/how-to-login.webm - -agent-browser open https://app.example.com/login -agent-browser wait 1000 # Pause for visibility - -agent-browser snapshot -i -agent-browser fill @e1 "demo@example.com" -agent-browser wait 500 - -agent-browser fill @e2 "password" -agent-browser wait 500 - -agent-browser click @e3 -agent-browser wait --load networkidle -agent-browser wait 1000 # Show result - -agent-browser record stop -``` - -### CI/CD Test Evidence - -```bash -#!/bin/bash -# Record E2E test runs for CI artifacts - -TEST_NAME="${1:-e2e-test}" -RECORDING_DIR="./test-recordings" -mkdir -p "$RECORDING_DIR" - -agent-browser record start "$RECORDING_DIR/$TEST_NAME-$(date +%s).webm" - -# Run test -if run_e2e_test; then - echo "Test passed" -else - echo "Test failed - recording saved" -fi - -agent-browser record stop -``` - -## Best Practices - -### 1. Add Pauses for Clarity - -```bash -# Slow down for human viewing -agent-browser click @e1 -agent-browser wait 500 # Let viewer see result -``` - -### 2. Use Descriptive Filenames - -```bash -# Include context in filename -agent-browser record start ./recordings/login-flow-2024-01-15.webm -agent-browser record start ./recordings/checkout-test-run-42.webm -``` - -### 3. Handle Recording in Error Cases - -```bash -#!/bin/bash -set -e - -cleanup() { - agent-browser record stop 2>/dev/null || true - agent-browser close 2>/dev/null || true -} -trap cleanup EXIT - -agent-browser record start ./automation.webm -# ... automation steps ... -``` - -### 4. Combine with Screenshots - -```bash -# Record video AND capture key frames -agent-browser record start ./flow.webm - -agent-browser open https://example.com -agent-browser screenshot ./screenshots/step1-homepage.png - -agent-browser click @e1 -agent-browser screenshot ./screenshots/step2-after-click.png - -agent-browser record stop -``` - -## Output Format - -- Default format: WebM (VP8/VP9 codec) -- Compatible with all modern browsers and video players -- Compressed but high quality - -## Limitations - -- Recording adds slight overhead to automation -- Large recordings can consume significant disk space -- Some headless environments may have codec limitations diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/templates/authenticated-session.sh b/src/crates/assembly/core/builtin_skills/agent-browser/templates/authenticated-session.sh deleted file mode 100755 index f9984c61e9..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/templates/authenticated-session.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -# Template: Authenticated Session Workflow -# Purpose: Login once, save state, reuse for subsequent runs -# Usage: ./authenticated-session.sh [state-file] -# -# Environment variables: -# APP_USERNAME - Login username/email -# APP_PASSWORD - Login password -# -# Two modes: -# 1. Discovery mode (default): Shows form structure so you can identify refs -# 2. Login mode: Performs actual login after you update the refs -# -# Setup steps: -# 1. Run once to see form structure (discovery mode) -# 2. Update refs in LOGIN FLOW section below -# 3. Set APP_USERNAME and APP_PASSWORD -# 4. Delete the DISCOVERY section - -set -euo pipefail - -LOGIN_URL="${1:?Usage: $0 [state-file]}" -STATE_FILE="${2:-./auth-state.json}" - -echo "Authentication workflow: $LOGIN_URL" - -# ================================================================ -# SAVED STATE: Skip login if valid saved state exists -# ================================================================ -if [[ -f "$STATE_FILE" ]]; then - echo "Loading saved state from $STATE_FILE..." - if agent-browser --state "$STATE_FILE" open "$LOGIN_URL" 2>/dev/null; then - agent-browser wait --load networkidle - - CURRENT_URL=$(agent-browser get url) - if [[ "$CURRENT_URL" != *"login"* ]] && [[ "$CURRENT_URL" != *"signin"* ]]; then - echo "Session restored successfully" - agent-browser snapshot -i - exit 0 - fi - echo "Session expired, performing fresh login..." - agent-browser close 2>/dev/null || true - else - echo "Failed to load state, re-authenticating..." - fi - rm -f "$STATE_FILE" -fi - -# ================================================================ -# DISCOVERY MODE: Shows form structure (delete after setup) -# ================================================================ -echo "Opening login page..." -agent-browser open "$LOGIN_URL" -agent-browser wait --load networkidle - -echo "" -echo "Login form structure:" -echo "---" -agent-browser snapshot -i -echo "---" -echo "" -echo "Next steps:" -echo " 1. Note the refs: username=@e?, password=@e?, submit=@e?" -echo " 2. Update the LOGIN FLOW section below with your refs" -echo " 3. Set: export APP_USERNAME='...' APP_PASSWORD='...'" -echo " 4. Delete this DISCOVERY MODE section" -echo "" -agent-browser close -exit 0 - -# ================================================================ -# LOGIN FLOW: Uncomment and customize after discovery -# ================================================================ -# : "${APP_USERNAME:?Set APP_USERNAME environment variable}" -# : "${APP_PASSWORD:?Set APP_PASSWORD environment variable}" -# -# agent-browser open "$LOGIN_URL" -# agent-browser wait --load networkidle -# agent-browser snapshot -i -# -# # Fill credentials (update refs to match your form) -# agent-browser fill @e1 "$APP_USERNAME" -# agent-browser fill @e2 "$APP_PASSWORD" -# agent-browser click @e3 -# agent-browser wait --load networkidle -# -# # Verify login succeeded -# FINAL_URL=$(agent-browser get url) -# if [[ "$FINAL_URL" == *"login"* ]] || [[ "$FINAL_URL" == *"signin"* ]]; then -# echo "Login failed - still on login page" -# agent-browser screenshot /tmp/login-failed.png -# agent-browser close -# exit 1 -# fi -# -# # Save state for future runs -# echo "Saving state to $STATE_FILE" -# agent-browser state save "$STATE_FILE" -# echo "Login successful" -# agent-browser snapshot -i diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/templates/capture-workflow.sh b/src/crates/assembly/core/builtin_skills/agent-browser/templates/capture-workflow.sh deleted file mode 100755 index 3bc93ad0c1..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/templates/capture-workflow.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Template: Content Capture Workflow -# Purpose: Extract content from web pages (text, screenshots, PDF) -# Usage: ./capture-workflow.sh [output-dir] -# -# Outputs: -# - page-full.png: Full page screenshot -# - page-structure.txt: Page element structure with refs -# - page-text.txt: All text content -# - page.pdf: PDF version -# -# Optional: Load auth state for protected pages - -set -euo pipefail - -TARGET_URL="${1:?Usage: $0 [output-dir]}" -OUTPUT_DIR="${2:-.}" - -echo "Capturing: $TARGET_URL" -mkdir -p "$OUTPUT_DIR" - -# Optional: Load authentication state -# if [[ -f "./auth-state.json" ]]; then -# echo "Loading authentication state..." -# agent-browser state load "./auth-state.json" -# fi - -# Navigate to target -agent-browser open "$TARGET_URL" -agent-browser wait --load networkidle - -# Get metadata -TITLE=$(agent-browser get title) -URL=$(agent-browser get url) -echo "Title: $TITLE" -echo "URL: $URL" - -# Capture full page screenshot -agent-browser screenshot --full "$OUTPUT_DIR/page-full.png" -echo "Saved: $OUTPUT_DIR/page-full.png" - -# Get page structure with refs -agent-browser snapshot -i > "$OUTPUT_DIR/page-structure.txt" -echo "Saved: $OUTPUT_DIR/page-structure.txt" - -# Extract all text content -agent-browser get text body > "$OUTPUT_DIR/page-text.txt" -echo "Saved: $OUTPUT_DIR/page-text.txt" - -# Save as PDF -agent-browser pdf "$OUTPUT_DIR/page.pdf" -echo "Saved: $OUTPUT_DIR/page.pdf" - -# Optional: Extract specific elements using refs from structure -# agent-browser get text @e5 > "$OUTPUT_DIR/main-content.txt" - -# Optional: Handle infinite scroll pages -# for i in {1..5}; do -# agent-browser scroll down 1000 -# agent-browser wait 1000 -# done -# agent-browser screenshot --full "$OUTPUT_DIR/page-scrolled.png" - -# Cleanup -agent-browser close - -echo "" -echo "Capture complete:" -ls -la "$OUTPUT_DIR" diff --git a/src/crates/assembly/core/builtin_skills/agent-browser/templates/form-automation.sh b/src/crates/assembly/core/builtin_skills/agent-browser/templates/form-automation.sh deleted file mode 100755 index 6784fcd3a5..0000000000 --- a/src/crates/assembly/core/builtin_skills/agent-browser/templates/form-automation.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Template: Form Automation Workflow -# Purpose: Fill and submit web forms with validation -# Usage: ./form-automation.sh -# -# This template demonstrates the snapshot-interact-verify pattern: -# 1. Navigate to form -# 2. Snapshot to get element refs -# 3. Fill fields using refs -# 4. Submit and verify result -# -# Customize: Update the refs (@e1, @e2, etc.) based on your form's snapshot output - -set -euo pipefail - -FORM_URL="${1:?Usage: $0 }" - -echo "Form automation: $FORM_URL" - -# Step 1: Navigate to form -agent-browser open "$FORM_URL" -agent-browser wait --load networkidle - -# Step 2: Snapshot to discover form elements -echo "" -echo "Form structure:" -agent-browser snapshot -i - -# Step 3: Fill form fields (customize these refs based on snapshot output) -# -# Common field types: -# agent-browser fill @e1 "John Doe" # Text input -# agent-browser fill @e2 "user@example.com" # Email input -# agent-browser fill @e3 "SecureP@ss123" # Password input -# agent-browser select @e4 "Option Value" # Dropdown -# agent-browser check @e5 # Checkbox -# agent-browser click @e6 # Radio button -# agent-browser fill @e7 "Multi-line text" # Textarea -# agent-browser upload @e8 /path/to/file.pdf # File upload -# -# Uncomment and modify: -# agent-browser fill @e1 "Test User" -# agent-browser fill @e2 "test@example.com" -# agent-browser click @e3 # Submit button - -# Step 4: Wait for submission -# agent-browser wait --load networkidle -# agent-browser wait --url "**/success" # Or wait for redirect - -# Step 5: Verify result -echo "" -echo "Result:" -agent-browser get url -agent-browser snapshot -i - -# Optional: Capture evidence -agent-browser screenshot /tmp/form-result.png -echo "Screenshot saved: /tmp/form-result.png" - -# Cleanup -agent-browser close -echo "Done" diff --git a/src/crates/assembly/core/builtin_skills/docx/SKILL.md b/src/crates/assembly/core/builtin_skills/docx/SKILL.md index 196bc08509..ab3cbd09a5 100644 --- a/src/crates/assembly/core/builtin_skills/docx/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/docx/SKILL.md @@ -1,481 +1,92 @@ --- name: docx -description: "Use this skill whenever the user wants to create, read, edit, or manipulate Word documents (.docx files). Triggers include: any mention of \"Word doc\", \"word document\", \".docx\", or requests to produce professional documents with formatting like tables of contents, headings, page numbers, or letterheads. Also use when extracting or reorganizing content from .docx files, inserting or replacing images in documents, performing find-and-replace in Word files, working with tracked changes or comments, or converting content into a polished Word document. If the user asks for a \"report\", \"memo\", \"letter\", \"template\", or similar deliverable as a Word or .docx file, use this skill. Do NOT use for PDFs, spreadsheets, Google Docs, or general coding tasks unrelated to document generation." +description: "Use this skill whenever the user wants to create, read, edit, or manipulate Word documents (.docx files) or Word templates (.dotx files). Triggers include: any mention of 'Word doc', 'word document', '.docx', '.dotx', or requests to produce professional documents with formatting like tables of contents, headings, page numbers, or letterheads. Also use when extracting or reorganizing content from .docx or .dotx files, inserting or replacing images in documents, performing find-and-replace in Word files, working with tracked changes or comments, or converting content into a polished Word document. If the user asks for a 'report', 'memo', 'letter', 'template', or similar deliverable as a Word or .docx file, use this skill. Do NOT use for PDFs, spreadsheets, Google Docs, or general coding tasks unrelated to document generation." license: Proprietary. LICENSE.txt has complete terms --- # DOCX creation, editing, and analysis -## Overview - -A .docx file is a ZIP archive containing XML files. - -## Quick Reference +A `.docx` is a ZIP archive of XML files. Choose your approach by task: | Task | Approach | -|------|----------| -| Read/analyze content | `pandoc` or unpack for raw XML | -| Create new document | Use `docx-js` - see Creating New Documents below | -| Edit existing document | Unpack → edit XML → repack - see Editing Existing Documents below | +|---|---| +| **Create** a new document | Write a `docx` (npm) script — see gotchas below | +| **Edit** an existing document | `safe_extract` → edit `word/document.xml` → `rezip` (docx-js cannot open existing files) | +| **Read** content | `pandoc -t markdown file.docx` | -### Converting .doc to .docx +> Script paths below are relative to this skill's directory. -Legacy `.doc` files must be converted before editing: +## Creating with docx-js — gotchas -```bash -python scripts/office/soffice.py --headless --convert-to docx document.doc -``` +`docx` is preinstalled — do not run `npm install` first; write the script and `require('docx')` directly. Only if that require fails: `npm install docx`. The model knows the API; these are the footguns: -### Reading Content +- **Page size defaults to A4.** For US Letter set `page: { size: { width: 12240, height: 15840 } }` (DXA; 1440 = 1″). +- **Landscape:** pass portrait dimensions and `orientation: PageOrientation.LANDSCAPE` — docx-js swaps width/height internally. +- **Tables need dual widths:** set `columnWidths` on the table AND `width` on every cell, both in `WidthType.DXA` (PERCENTAGE breaks in Google Docs). Column widths must sum to the table width. +- **Table shading:** use `ShadingType.CLEAR`, never `SOLID` (renders black). +- **Lists:** never insert `•` literally; use a `numbering` config with `LevelFormat.BULLET`. +- **`ImageRun` requires `type:`** (`"png"`, `"jpg"`, …). +- **`PageBreak` must be inside a `Paragraph`.** +- **Never use `\n`** — use separate `Paragraph` elements. +- **TOC:** headings must use built-in `HeadingLevel.*`; custom heading styles need `outlineLevel` set or they won't appear. +- **Don't use a table as a horizontal rule** — use a paragraph bottom border instead. +- **Dot-leader / right-aligned-on-same-line:** use `PositionalTab` (`alignment: PositionalTabAlignment.RIGHT`, `leader: PositionalTabLeader.DOT`) inside a `TextRun`, not literal `.` or space padding. -```bash -# Text extraction with tracked changes -pandoc --track-changes=all document.docx -o output.md - -# Raw XML access -python scripts/office/unpack.py document.docx unpacked/ -``` +## Verify the output -### Converting to Images +After writing a `.docx`, render it and look at it: ```bash -python scripts/office/soffice.py --headless --convert-to pdf document.docx -pdftoppm -jpeg -r 150 document.pdf page +python scripts/office/soffice.py --headless --convert-to pdf output.docx +pdftoppm -jpeg -r 100 output.pdf page +ls page-*.jpg # then Read the images ``` -### Accepting Tracked Changes - -To produce a clean document with all tracked changes accepted (requires LibreOffice): - -```bash -python scripts/accept_changes.py input.docx output.docx -``` - ---- +`pdftoppm` zero-pads page numbers to the width of the page count (`page-01.jpg`…`page-12.jpg`). -## Creating New Documents +## Editing existing documents -Generate .docx files with JavaScript, then validate. Install: `npm install -g docx` +Legacy `.doc` files must be converted first: `python scripts/office/soffice.py --headless --convert-to docx file.doc`. -### Setup -```javascript -const { Document, Packer, Paragraph, TextRun, Table, TableRow, TableCell, ImageRun, - Header, Footer, AlignmentType, PageOrientation, LevelFormat, ExternalHyperlink, - TableOfContents, HeadingLevel, BorderStyle, WidthType, ShadingType, - VerticalAlign, PageNumber, PageBreak } = require('docx'); - -const doc = new Document({ sections: [{ children: [/* content */] }] }); -Packer.toBuffer(doc).then(buffer => fs.writeFileSync("doc.docx", buffer)); -``` - -### Validation -After creating the file, validate it. If validation fails, unpack, fix the XML, and repack. ```bash -python scripts/office/validate.py doc.docx -``` - -### Page Size - -```javascript -// CRITICAL: docx-js defaults to A4, not US Letter -// Always set page size explicitly for consistent results -sections: [{ - properties: { - page: { - size: { - width: 12240, // 8.5 inches in DXA - height: 15840 // 11 inches in DXA - }, - margin: { top: 1440, right: 1440, bottom: 1440, left: 1440 } // 1 inch margins - } - }, - children: [/* content */] -}] -``` - -**Common page sizes (DXA units, 1440 DXA = 1 inch):** - -| Paper | Width | Height | Content Width (1" margins) | -|-------|-------|--------|---------------------------| -| US Letter | 12,240 | 15,840 | 9,360 | -| A4 (default) | 11,906 | 16,838 | 9,026 | - -**Landscape orientation:** docx-js swaps width/height internally, so pass portrait dimensions and let it handle the swap: -```javascript -size: { - width: 12240, // Pass SHORT edge as width - height: 15840, // Pass LONG edge as height - orientation: PageOrientation.LANDSCAPE // docx-js swaps them in the XML -}, -// Content width = 15840 - left margin - right margin (uses the long edge) -``` - -### Styles (Override Built-in Headings) - -Use Arial as the default font (universally supported). Keep titles black for readability. - -```javascript -const doc = new Document({ - styles: { - default: { document: { run: { font: "Arial", size: 24 } } }, // 12pt default - paragraphStyles: [ - // IMPORTANT: Use exact IDs to override built-in styles - { id: "Heading1", name: "Heading 1", basedOn: "Normal", next: "Normal", quickFormat: true, - run: { size: 32, bold: true, font: "Arial" }, - paragraph: { spacing: { before: 240, after: 240 }, outlineLevel: 0 } }, // outlineLevel required for TOC - { id: "Heading2", name: "Heading 2", basedOn: "Normal", next: "Normal", quickFormat: true, - run: { size: 28, bold: true, font: "Arial" }, - paragraph: { spacing: { before: 180, after: 180 }, outlineLevel: 1 } }, - ] - }, - sections: [{ - children: [ - new Paragraph({ heading: HeadingLevel.HEADING_1, children: [new TextRun("Title")] }), - ] - }] -}); -``` - -### Lists (NEVER use unicode bullets) - -```javascript -// ❌ WRONG - never manually insert bullet characters -new Paragraph({ children: [new TextRun("• Item")] }) // BAD -new Paragraph({ children: [new TextRun("\u2022 Item")] }) // BAD - -// ✅ CORRECT - use numbering config with LevelFormat.BULLET -const doc = new Document({ - numbering: { - config: [ - { reference: "bullets", - levels: [{ level: 0, format: LevelFormat.BULLET, text: "•", alignment: AlignmentType.LEFT, - style: { paragraph: { indent: { left: 720, hanging: 360 } } } }] }, - { reference: "numbers", - levels: [{ level: 0, format: LevelFormat.DECIMAL, text: "%1.", alignment: AlignmentType.LEFT, - style: { paragraph: { indent: { left: 720, hanging: 360 } } } }] }, - ] - }, - sections: [{ - children: [ - new Paragraph({ numbering: { reference: "bullets", level: 0 }, - children: [new TextRun("Bullet item")] }), - new Paragraph({ numbering: { reference: "numbers", level: 0 }, - children: [new TextRun("Numbered item")] }), - ] - }] -}); - -// ⚠️ Each reference creates INDEPENDENT numbering -// Same reference = continues (1,2,3 then 4,5,6) -// Different reference = restarts (1,2,3 then 1,2,3) -``` - -### Tables - -**CRITICAL: Tables need dual widths** - set both `columnWidths` on the table AND `width` on each cell. Without both, tables render incorrectly on some platforms. - -```javascript -// CRITICAL: Always set table width for consistent rendering -// CRITICAL: Use ShadingType.CLEAR (not SOLID) to prevent black backgrounds -const border = { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }; -const borders = { top: border, bottom: border, left: border, right: border }; - -new Table({ - width: { size: 9360, type: WidthType.DXA }, // Always use DXA (percentages break in Google Docs) - columnWidths: [4680, 4680], // Must sum to table width (DXA: 1440 = 1 inch) - rows: [ - new TableRow({ - children: [ - new TableCell({ - borders, - width: { size: 4680, type: WidthType.DXA }, // Also set on each cell - shading: { fill: "D5E8F0", type: ShadingType.CLEAR }, // CLEAR not SOLID - margins: { top: 80, bottom: 80, left: 120, right: 120 }, // Cell padding (internal, not added to width) - children: [new Paragraph({ children: [new TextRun("Cell")] })] - }) - ] - }) - ] -}) -``` - -**Table width calculation:** - -Always use `WidthType.DXA` — `WidthType.PERCENTAGE` breaks in Google Docs. - -```javascript -// Table width = sum of columnWidths = content width -// US Letter with 1" margins: 12240 - 2880 = 9360 DXA -width: { size: 9360, type: WidthType.DXA }, -columnWidths: [7000, 2360] // Must sum to table width -``` - -**Width rules:** -- **Always use `WidthType.DXA`** — never `WidthType.PERCENTAGE` (incompatible with Google Docs) -- Table width must equal the sum of `columnWidths` -- Cell `width` must match corresponding `columnWidth` -- Cell `margins` are internal padding - they reduce content area, not add to cell width -- For full-width tables: use content width (page width minus left and right margins) - -### Images - -```javascript -// CRITICAL: type parameter is REQUIRED -new Paragraph({ - children: [new ImageRun({ - type: "png", // Required: png, jpg, jpeg, gif, bmp, svg - data: fs.readFileSync("image.png"), - transformation: { width: 200, height: 150 }, - altText: { title: "Title", description: "Desc", name: "Name" } // All three required - })] -}) -``` - -### Page Breaks - -```javascript -// CRITICAL: PageBreak must be inside a Paragraph -new Paragraph({ children: [new PageBreak()] }) - -// Or use pageBreakBefore -new Paragraph({ pageBreakBefore: true, children: [new TextRun("New page")] }) +python -c "import sys,zipfile; from pathlib import Path; from scripts.office.helpers import safe_extract; zf=zipfile.ZipFile(sys.argv[1]); safe_extract(zf, Path(sys.argv[2])); zf.close()" doc.docx unpacked +python scripts/merge_runs.py unpacked/ # coalesce fragmented runs so text is findable +# edit unpacked/word/document.xml in place — do NOT reformat or pretty-print +python -c "from pathlib import Path; from scripts.office.helpers import rezip; rezip(Path('unpacked'), Path('out.docx'))" +python scripts/office/validate.py out.docx --original doc.docx # XSD checks; --auto-repair fixes common issues +# redlining? add --author "" to check every edit is tracked ``` -### Table of Contents +Word splits text across many `` runs (revision ids, spell-check markers), so a phrase you can see in the document often doesn't exist as a contiguous string in the XML. `merge_runs.py` merges adjacent identically-formatted runs in `word/document.xml` without changing content or rendering; it also accepts a `.docx` directly (`python scripts/merge_runs.py doc.docx -o merged.docx`). -```javascript -// CRITICAL: Headings must use HeadingLevel ONLY - no custom styles -new TableOfContents("Table of Contents", { hyperlink: true, headingStyleRange: "1-3" }) -``` +Use "BitFun" as the author for tracked changes and comments unless the user explicitly requests a different name. -### Headers/Footers +**Tracked changes:** when redlining, validate with `--author ""` (needs `--original`) — it reports any text you changed without a ``/`` around it, which is easy to do by accident and invisible in the accepted view. Wrap runs in ``/`` with `w:id`, `w:author`, `w:date` attributes. Inside ``, the text element is ``, not ``. A deleted paragraph mark (``) means "merge this paragraph into the next" — so deleting a paragraph outright is that plus a `` around every run. The `` must come before the rPr's other children; their order is schema-enforced. -```javascript -sections: [{ - properties: { - page: { margin: { top: 1440, right: 1440, bottom: 1440, left: 1440 } } // 1440 = 1 inch - }, - headers: { - default: new Header({ children: [new Paragraph({ children: [new TextRun("Header")] })] }) - }, - footers: { - default: new Footer({ children: [new Paragraph({ - children: [new TextRun("Page "), new TextRun({ children: [PageNumber.CURRENT] })] - })] }) - }, - children: [/* content */] -}] -``` +To produce a clean copy with all tracked changes accepted: `python scripts/accept_changes.py in.docx out.docx`. -### Critical Rules for docx-js +Accepting a deleted paragraph mark should join that paragraph to the one below it, so a paragraph whose runs are *all* deleted vanishes. Word does this; `accept_changes.py` and `pandoc --track-changes=accept` don't always. Both fail the same way — they strip the deleted text but leave the emptied paragraph behind, which reads as a stray empty bullet when it was auto-numbered: -- **Set page size explicitly** - docx-js defaults to A4; use US Letter (12240 x 15840 DXA) for US documents -- **Landscape: pass portrait dimensions** - docx-js swaps width/height internally; pass short edge as `width`, long edge as `height`, and set `orientation: PageOrientation.LANDSCAPE` -- **Never use `\n`** - use separate Paragraph elements -- **Never use unicode bullets** - use `LevelFormat.BULLET` with numbering config -- **PageBreak must be in Paragraph** - standalone creates invalid XML -- **ImageRun requires `type`** - always specify png/jpg/etc -- **Always set table `width` with DXA** - never use `WidthType.PERCENTAGE` (breaks in Google Docs) -- **Tables need dual widths** - `columnWidths` array AND cell `width`, both must match -- **Table width = sum of columnWidths** - for DXA, ensure they add up exactly -- **Always add cell margins** - use `margins: { top: 80, bottom: 80, left: 120, right: 120 }` for readable padding -- **Use `ShadingType.CLEAR`** - never SOLID for table shading -- **TOC requires HeadingLevel only** - no custom styles on heading paragraphs -- **Override built-in styles** - use exact IDs: "Heading1", "Heading2", etc. -- **Include `outlineLevel`** - required for TOC (0 for H1, 1 for H2, etc.) +- `pandoc --track-changes=accept` never joins the paragraphs. +- `accept_changes.py` (LibreOffice) joins them correctly, except when the deleted paragraph is followed by an empty spacer paragraph. ---- +An empty bullet in either view is an artifact of that view, not a defect in the document. Check paragraph deletions in the XML. -## Editing Existing Documents +## Comments -**Follow all 3 steps in order.** +Comments require six cross-linked files. Use the helper — directory mode when you'll also be editing `document.xml` (saves an unzip/rezip cycle), `.docx`-direct mode otherwise: -### Step 1: Unpack ```bash -python scripts/office/unpack.py document.docx unpacked/ -``` -Extracts XML, pretty-prints, merges adjacent runs, and converts smart quotes to XML entities (`“` etc.) so they survive editing. Use `--merge-runs false` to skip run merging. - -### Step 2: Edit XML - -Edit files in `unpacked/word/`. See XML Reference below for patterns. - -**Use "BitFun" as the author** for tracked changes and comments, unless the user explicitly requests use of a different name. +# Against an already-unpacked directory (preferred when also placing markers) +python scripts/comment.py unpacked/ "Fees & expenses cap is too low" +python scripts/comment.py unpacked/ "Agreed" --parent 0 -**Use the Edit tool directly for string replacement. Do not write Python scripts.** Scripts introduce unnecessary complexity. The Edit tool shows exactly what is being replaced. - -**CRITICAL: Use smart quotes for new content.** When adding text with apostrophes or quotes, use XML entities to produce smart quotes: -```xml - -Here’s a quote: “Hello” -``` -| Entity | Character | -|--------|-----------| -| `‘` | ‘ (left single) | -| `’` | ’ (right single / apostrophe) | -| `“` | “ (left double) | -| `”` | ” (right double) | - -**Adding comments:** Use `comment.py` to handle boilerplate across multiple XML files (text must be pre-escaped XML): -```bash -python scripts/comment.py unpacked/ 0 "Comment text with & and ’" -python scripts/comment.py unpacked/ 1 "Reply text" --parent 0 # reply to comment 0 -python scripts/comment.py unpacked/ 0 "Text" --author "Custom Author" # custom author name +# Against a .docx directly +python scripts/comment.py contract.docx "This cap is too low" -o annotated.docx ``` -Then add markers to document.xml (see Comments in XML Reference). -### Step 3: Pack -```bash -python scripts/office/pack.py unpacked/ output.docx --original document.docx -``` -Validates with auto-repair, condenses XML, and creates DOCX. Use `--validate false` to skip. - -**Auto-repair will fix:** -- `durableId` >= 0x7FFFFFFF (regenerates valid ID) -- Missing `xml:space="preserve"` on `` with whitespace - -**Auto-repair won't fix:** -- Malformed XML, invalid element nesting, missing relationships, schema violations - -### Common Pitfalls - -- **Replace entire `` elements**: When adding tracked changes, replace the whole `...` block with `......` as siblings. Don't inject tracked change tags inside a run. -- **Preserve `` formatting**: Copy the original run's `` block into your tracked change runs to maintain bold, font size, etc. - ---- - -## XML Reference - -### Schema Compliance - -- **Element order in ``**: ``, ``, ``, ``, ``, `` last -- **Whitespace**: Add `xml:space="preserve"` to `` with leading/trailing spaces -- **RSIDs**: Must be 8-digit hex (e.g., `00AB1234`) - -### Tracked Changes - -**Insertion:** -```xml - - inserted text - -``` - -**Deletion:** -```xml - - deleted text - -``` - -**Inside ``**: Use `` instead of ``, and `` instead of ``. - -**Minimal edits** - only mark what changes: -```xml - -The term is - - 30 - - - 60 - - days. -``` - -**Deleting entire paragraphs/list items** - when removing ALL content from a paragraph, also mark the paragraph mark as deleted so it merges with the next paragraph. Add `` inside ``: -```xml - - - ... - - - - - - Entire paragraph content being deleted... - - -``` -Without the `` in ``, accepting changes leaves an empty paragraph/list item. - -**Rejecting another author's insertion** - nest deletion inside their insertion: -```xml - - - their inserted text - - -``` - -**Restoring another author's deletion** - add insertion after (don't modify their deletion): -```xml - - deleted text - - - deleted text - -``` - -### Comments - -After running `comment.py` (see Step 2), add markers to document.xml. For replies, use `--parent` flag and nest markers inside the parent's. - -**CRITICAL: `` and `` are siblings of ``, never inside ``.** - -```xml - - - - deleted - - more text - - - - - - - text - - - - -``` - -### Images - -1. Add image file to `word/media/` -2. Add relationship to `word/_rels/document.xml.rels`: -```xml - -``` -3. Add content type to `[Content_Types].xml`: -```xml - -``` -4. Reference in document.xml: -```xml - - - - - - - - - - - - -``` - ---- +The script writes `comments.xml`, `commentsExtended.xml`, `commentsIds.xml`, `commentsExtensible.xml`, the relationships, and the content-type overrides. Comment IDs are auto-assigned. It then prints the ``/``/`` snippet to add to `word/document.xml` so the comment anchors to specific text — until you place those markers, the comment exists but is not visible. ## Dependencies -- **pandoc**: Text extraction -- **docx**: `npm install -g docx` (new documents) -- **LibreOffice**: PDF conversion (auto-configured for sandboxed environments via `scripts/office/soffice.py`) -- **Poppler**: `pdftoppm` for images +`docx` (npm, preinstalled — install only if `require('docx')` fails) · `pandoc` · LibreOffice (`soffice`) · `pdftoppm` (Poppler) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/comment.py b/src/crates/assembly/core/builtin_skills/docx/scripts/comment.py index 36e1c935f2..7e16100192 100755 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/comment.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/comment.py @@ -1,26 +1,41 @@ -"""Add comments to DOCX documents. +"""Add comments to a DOCX document. + +Accepts either an unpacked directory OR a .docx/.dotx file directly. Usage: - python comment.py unpacked/ 0 "Comment text" - python comment.py unpacked/ 1 "Reply text" --parent 0 + # Against an unpacked directory (writes satellite files in place) + python comment.py unpacked/ "Comment text" + python comment.py unpacked/ "Reply text" --parent 0 + + # Against a .docx directly (extracts, writes satellite files, rezips) + python comment.py contract.docx "This cap is too low" -o annotated.docx + python comment.py contract.docx "Comment" --id 5 # explicit ID -Text should be pre-escaped XML (e.g., & for &, ’ for smart quotes). +The comment ID is auto-assigned (max existing + 1) unless --id is given. +Plain text is XML-escaped automatically; if you pass already-escaped text +(e.g. &, ’) use --raw to skip escaping. -After running, add markers to document.xml: - +After running, add markers to word/document.xml so the comment is visible: + ... commented content ... - - + + """ import argparse import random import shutil import sys +import tempfile +import zipfile from datetime import datetime, timezone from pathlib import Path import defusedxml.minidom +from xml.parsers.expat import ExpatError +from xml.sax.saxutils import escape as xml_escape + +from office.helpers import opc_target, rezip as _rezip, safe_extract as _safe_extract TEMPLATE_DIR = Path(__file__).parent / "templates" NS = { @@ -44,39 +59,38 @@ - {text} + {text} """ COMMENT_MARKER_TEMPLATE = """ -Add to document.xml (markers must be direct children of w:p, never inside w:r): +Add to word/document.xml (markers must be direct children of w:p, never inside w:r): ... """ REPLY_MARKER_TEMPLATE = """ -Nest markers inside parent {pid}'s markers (markers must be direct children of w:p, never inside w:r): +Nest markers inside parent {pid}'s markers (direct children of w:p, never inside w:r): ... """ +SMART_QUOTE_ENTITIES = { + "“": "“", + "”": "”", + "‘": "‘", + "’": "’", +} + def _generate_hex_id() -> str: return f"{random.randint(0, 0x7FFFFFFE):08X}" -SMART_QUOTE_ENTITIES = { - "\u201c": "“", - "\u201d": "”", - "\u2018": "‘", - "\u2019": "’", -} - - def _encode_smart_quotes(text: str) -> str: for char, entity in SMART_QUOTE_ENTITIES.items(): text = text.replace(char, entity) @@ -105,6 +119,19 @@ def _find_para_id(comments_path: Path, comment_id: int) -> str | None: return None +def _next_comment_id(comments_path: Path) -> int: + if not comments_path.exists(): + return 0 + dom = defusedxml.minidom.parseString(comments_path.read_text(encoding="utf-8")) + ids = [] + for c in dom.getElementsByTagName("w:comment"): + try: + ids.append(int(c.getAttribute("w:id"))) + except ValueError: + pass + return (max(ids) + 1) if ids else 0 + + def _get_next_rid(rels_path: Path) -> int: dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8")) max_rid = 0 @@ -120,151 +147,146 @@ def _get_next_rid(rels_path: Path) -> int: def _has_relationship(rels_path: Path, target: str) -> bool: dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8")) - for rel in dom.getElementsByTagName("Relationship"): - if rel.getAttribute("Target") == target: - return True - return False + return any( + rel.getAttribute("Target") == target + for rel in dom.getElementsByTagName("Relationship") + ) def _has_content_type(ct_path: Path, part_name: str) -> bool: dom = defusedxml.minidom.parseString(ct_path.read_text(encoding="utf-8")) - for override in dom.getElementsByTagName("Override"): - if override.getAttribute("PartName") == part_name: - return True - return False + return any( + o.getAttribute("PartName") == part_name + for o in dom.getElementsByTagName("Override") + ) + + +_COMMENT_RELS = [ + ("http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments", "comments.xml"), + ("http://schemas.microsoft.com/office/2011/relationships/commentsExtended", "commentsExtended.xml"), + ("http://schemas.microsoft.com/office/2016/09/relationships/commentsIds", "commentsIds.xml"), + ("http://schemas.microsoft.com/office/2018/08/relationships/commentsExtensible", "commentsExtensible.xml"), +] +_COMMENT_OVERRIDES = [ + ("/word/comments.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"), + ("/word/commentsExtended.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtended+xml"), + ("/word/commentsIds.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsIds+xml"), + ("/word/commentsExtensible.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtensible+xml"), +] def _ensure_comment_relationships(unpacked_dir: Path) -> None: rels_path = unpacked_dir / "word" / "_rels" / "document.xml.rels" if not rels_path.exists(): return - - if _has_relationship(rels_path, "comments.xml"): - return - dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8")) root = dom.documentElement + comment_types = {rel_type for rel_type, _ in _COMMENT_RELS} + existing = set() + for rel in dom.getElementsByTagName("Relationship"): + if rel.getAttribute("Type") not in comment_types: + continue + part = opc_target( + rel.getAttribute("Target"), + "word/document.xml", + rel.getAttribute("TargetMode"), + ) + if part is not None: + existing.add(part) next_rid = _get_next_rid(rels_path) - - rels = [ - ( - "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments", - "comments.xml", - ), - ( - "http://schemas.microsoft.com/office/2011/relationships/commentsExtended", - "commentsExtended.xml", - ), - ( - "http://schemas.microsoft.com/office/2016/09/relationships/commentsIds", - "commentsIds.xml", - ), - ( - "http://schemas.microsoft.com/office/2018/08/relationships/commentsExtensible", - "commentsExtensible.xml", - ), - ] - - for rel_type, target in rels: + changed = False + for rel_type, target in _COMMENT_RELS: + if opc_target(target, "word/document.xml") in existing: + continue rel = dom.createElement("Relationship") rel.setAttribute("Id", f"rId{next_rid}") rel.setAttribute("Type", rel_type) rel.setAttribute("Target", target) root.appendChild(rel) next_rid += 1 - - rels_path.write_bytes(dom.toxml(encoding="UTF-8")) + changed = True + if changed: + rels_path.write_bytes(dom.toxml(encoding="UTF-8")) def _ensure_comment_content_types(unpacked_dir: Path) -> None: ct_path = unpacked_dir / "[Content_Types].xml" if not ct_path.exists(): return - - if _has_content_type(ct_path, "/word/comments.xml"): - return - dom = defusedxml.minidom.parseString(ct_path.read_text(encoding="utf-8")) root = dom.documentElement - - overrides = [ - ( - "/word/comments.xml", - "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml", - ), - ( - "/word/commentsExtended.xml", - "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtended+xml", - ), - ( - "/word/commentsIds.xml", - "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsIds+xml", - ), - ( - "/word/commentsExtensible.xml", - "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtensible+xml", - ), - ] - - for part_name, content_type in overrides: + existing = { + o.getAttribute("PartName") + for o in dom.getElementsByTagName("Override") + } + changed = False + for part_name, content_type in _COMMENT_OVERRIDES: + if part_name in existing: + continue override = dom.createElement("Override") override.setAttribute("PartName", part_name) override.setAttribute("ContentType", content_type) root.appendChild(override) - - ct_path.write_bytes(dom.toxml(encoding="UTF-8")) + changed = True + if changed: + ct_path.write_bytes(dom.toxml(encoding="UTF-8")) def add_comment( - unpacked_dir: str, - comment_id: int, + unpacked_dir: Path | str, text: str, - author: str = "Claude", - initials: str = "C", + comment_id: int | None = None, + author: str = "BitFun", + initials: str = "B", parent_id: int | None = None, -) -> tuple[str, str]: - word = Path(unpacked_dir) / "word" + raw: bool = False, +) -> tuple[int, str, str]: + unpacked_dir = Path(unpacked_dir) + if not raw: + text = xml_escape(text) + author = xml_escape(author, {'"': """}) + initials = xml_escape(initials, {'"': """}) + word = unpacked_dir / "word" if not word.exists(): - return "", f"Error: {word} not found" + raise FileNotFoundError(f"{word} not found (not an unpacked .docx?)") + + comments = word / "comments.xml" + if comment_id is None: + comment_id = _next_comment_id(comments) + + parent_para = None + if parent_id is not None: + parent_para = _find_para_id(comments, parent_id) if comments.exists() else None + if not parent_para: + raise ValueError(f"parent comment {parent_id} not found") para_id, durable_id = _generate_hex_id(), _generate_hex_id() ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - comments = word / "comments.xml" - first_comment = not comments.exists() - if first_comment: + if not comments.exists(): shutil.copy(TEMPLATE_DIR / "comments.xml", comments) - _ensure_comment_relationships(Path(unpacked_dir)) - _ensure_comment_content_types(Path(unpacked_dir)) + _ensure_comment_relationships(unpacked_dir) + _ensure_comment_content_types(unpacked_dir) _append_xml( comments, "w:comments", COMMENT_XML.format( - id=comment_id, - author=author, - date=ts, - initials=initials, - para_id=para_id, - text=text, + id=comment_id, author=author, date=ts, initials=initials, + para_id=para_id, text=text, ), ) ext = word / "commentsExtended.xml" if not ext.exists(): shutil.copy(TEMPLATE_DIR / "commentsExtended.xml", ext) - if parent_id is not None: - parent_para = _find_para_id(comments, parent_id) - if not parent_para: - return "", f"Error: Parent comment {parent_id} not found" + if parent_para is not None: _append_xml( - ext, - "w15:commentsEx", + ext, "w15:commentsEx", f'', ) else: _append_xml( - ext, - "w15:commentsEx", + ext, "w15:commentsEx", f'', ) @@ -272,8 +294,7 @@ def add_comment( if not ids.exists(): shutil.copy(TEMPLATE_DIR / "commentsIds.xml", ids) _append_xml( - ids, - "w16cid:commentsIds", + ids, "w16cid:commentsIds", f'', ) @@ -281,38 +302,67 @@ def add_comment( if not extensible.exists(): shutil.copy(TEMPLATE_DIR / "commentsExtensible.xml", extensible) _append_xml( - extensible, - "w16cex:commentsExtensible", + extensible, "w16cex:commentsExtensible", f'', ) action = "reply" if parent_id is not None else "comment" - return para_id, f"Added {action} {comment_id} (para_id={para_id})" - - -if __name__ == "__main__": - p = argparse.ArgumentParser(description="Add comments to DOCX documents") - p.add_argument("unpacked_dir", help="Unpacked DOCX directory") - p.add_argument("comment_id", type=int, help="Comment ID (must be unique)") - p.add_argument("text", help="Comment text") - p.add_argument("--author", default="Claude", help="Author name") - p.add_argument("--initials", default="C", help="Author initials") - p.add_argument("--parent", type=int, help="Parent comment ID (for replies)") + return comment_id, para_id, f"Added {action} id={comment_id} (paraId={para_id})" + + +def main() -> None: + p = argparse.ArgumentParser(description="Add a comment to a DOCX (directory or .docx file).") + p.add_argument("input", help="Unpacked DOCX directory OR a .docx/.dotx file") + p.add_argument("text", help="Comment text (plain text; XML-escaped automatically)") + p.add_argument("--raw", action="store_true", + help="Treat text as pre-escaped XML (skip automatic escaping)") + p.add_argument("--id", type=int, dest="comment_id", + help="Comment ID (default: auto-assign as max existing + 1)") + p.add_argument("--author", default="BitFun", help="Author name") + p.add_argument("--initials", default="B", help="Author initials") + p.add_argument("--parent", type=int, help="Parent comment ID (makes this a reply)") + p.add_argument("-o", "--output", + help="Output .docx path (only used when input is a .docx; default: overwrite input)") args = p.parse_args() - para_id, msg = add_comment( - args.unpacked_dir, - args.comment_id, - args.text, - args.author, - args.initials, - args.parent, - ) - print(msg) - if "Error" in msg: + src = Path(args.input) + + try: + if src.is_dir(): + if args.output: + print("Warning: --output ignored for directory input", file=sys.stderr) + cid, _, msg = add_comment( + src, args.text, comment_id=args.comment_id, + author=args.author, initials=args.initials, + parent_id=args.parent, raw=args.raw, + ) + print(msg) + elif src.is_file() and src.suffix.lower() in (".docx", ".dotx"): + out = Path(args.output) if args.output else src + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + with zipfile.ZipFile(src) as zf: + _safe_extract(zf, tmp_path) + cid, _, msg = add_comment( + tmp_path, args.text, comment_id=args.comment_id, + author=args.author, initials=args.initials, + parent_id=args.parent, raw=args.raw, + ) + _rezip(tmp_path, out) + print(msg) + print(f"Wrote {out} (comment defined; add markers to word/document.xml to make it visible)") + else: + print(f"Error: {src} is neither a directory nor a .docx/.dotx file", file=sys.stderr) + sys.exit(1) + except (FileNotFoundError, ValueError, zipfile.BadZipFile, ExpatError) as e: + print(f"Error: {e}", file=sys.stderr) sys.exit(1) - cid = args.comment_id + if args.parent is not None: print(REPLY_MARKER_TEMPLATE.format(pid=args.parent, cid=cid)) else: print(COMMENT_MARKER_TEMPLATE.format(cid=cid)) + + +if __name__ == "__main__": + main() diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/merge_runs.py b/src/crates/assembly/core/builtin_skills/docx/scripts/merge_runs.py new file mode 100755 index 0000000000..977822929c --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/merge_runs.py @@ -0,0 +1,310 @@ +"""Merge adjacent identically-formatted runs in a DOCX. + +Word fragments paragraph text across many elements (revision ids, +spell-check markers, editing history), which makes find-and-replace on +word/document.xml unreliable — the string you're looking for is split +across runs. This coalesces adjacent runs whose formatting () is +identical, strips rsid attributes and proofErr markers, and consolidates the +text elements — , and for text inside a tracked deletion. + +Rendering is unchanged. The text you search is what Word draws, which is not +always the bytes in the file: an element without xml:space="preserve" has its +edge whitespace trimmed before it reaches the page, so `Hello ` +followed by `world` reads "Helloworld" and merges to exactly that. + +Runs in two different / wrappers are never merged: that would +rewrite tracked-change structure, collapsing separate revisions into one. + +Only word/document.xml is processed (not headers, footers, or footnotes). + +Usage: + python merge_runs.py unpacked/ # after unzip, before editing + python merge_runs.py document.docx # rewrite in place + python merge_runs.py document.docx -o out.docx +""" + + +import argparse +import sys +import tempfile +import zipfile +from pathlib import Path + +import defusedxml.minidom + +from office.helpers import XML_SPACE, rendered_text, rezip, safe_extract + +WORDML_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + +def merge_runs(input_dir: str) -> tuple[int, str]: + doc_xml = Path(input_dir) / "word" / "document.xml" + + if not doc_xml.exists(): + return 0, f"Error: {doc_xml} not found" + + try: + dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) + root = dom.documentElement + run_names = _run_tag_names(root) + + _remove_elements(root, "proofErr") + + runs = _find_runs(root, run_names) + _strip_rsid_attrs(runs) + + merge_count = 0 + for container in {run.parentNode for run in runs}: + merge_count += _merge_runs_in(container, run_names) + + doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) + return merge_count, f"Merged {merge_count} runs" + + except Exception as e: + return 0, f"Error: {e}" + + + + +def _is_element(node, tag: str) -> bool: + name = node.localName or node.tagName + return name == tag or name.endswith(f":{tag}") + + +def _run_tag_names(root) -> set[str]: + names = set() + for attr in root.attributes.values(): + if attr.value == WORDML_NS: + if attr.name == "xmlns": + names.add("r") + elif attr.name.startswith("xmlns:"): + names.add(attr.name.split(":", 1)[1] + ":r") + return names or {"w:r", "r"} + + +def _find_elements(root, tag: str) -> list: + results = [] + + def traverse(node): + if node.nodeType == node.ELEMENT_NODE: + if _is_element(node, tag): + results.append(node) + for child in node.childNodes: + traverse(child) + + traverse(root) + return results + + +def _find_runs(root, run_names: set[str]) -> list: + return [e for e in _find_elements(root, "r") if _is_run(e, run_names)] + + +def _get_child(parent, tag: str): + return next(iter(_get_children(parent, tag)), None) + + +def _get_children(parent, tag: str) -> list: + return [ + child + for child in parent.childNodes + if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag) + ] + + +def _is_adjacent(elem1, elem2) -> bool: + node = elem1.nextSibling + while node: + if node == elem2: + return True + if node.nodeType == node.ELEMENT_NODE: + return False + if node.nodeType == node.TEXT_NODE and node.data.strip(XML_SPACE): + return False + node = node.nextSibling + return False + + + + +def _remove_elements(root, tag: str): + for elem in _find_elements(root, tag): + if elem.parentNode: + elem.parentNode.removeChild(elem) + + +def _strip_rsid_attrs(runs: list): + for run in runs: + for attr in list(run.attributes.values()): + if "rsid" in attr.name.lower(): + run.removeAttribute(attr.name) + + + + +def _merge_runs_in(container, run_names: set[str]) -> int: + merge_count = 0 + run = _first_child_run(container, run_names) + + while run: + while True: + next_elem = _next_element_sibling(run) + if next_elem and _is_run(next_elem, run_names) and _can_merge(run, next_elem): + _merge_run_content(run, next_elem) + container.removeChild(next_elem) + merge_count += 1 + else: + break + + _consolidate_text(run) + run = _next_sibling_run(run, run_names) + + return merge_count + + +def _first_child_run(container, run_names: set[str]): + for child in container.childNodes: + if child.nodeType == child.ELEMENT_NODE and _is_run(child, run_names): + return child + return None + + +def _next_element_sibling(node): + sibling = node.nextSibling + while sibling: + if sibling.nodeType == sibling.ELEMENT_NODE: + return sibling + sibling = sibling.nextSibling + return None + + +def _next_sibling_run(node, run_names: set[str]): + sibling = node.nextSibling + while sibling: + if sibling.nodeType == sibling.ELEMENT_NODE: + if _is_run(sibling, run_names): + return sibling + sibling = sibling.nextSibling + return None + + +def _is_run(node, run_names: set[str]) -> bool: + return node.tagName in run_names + + +def _can_merge(run1, run2) -> bool: + rpr1 = _get_child(run1, "rPr") + rpr2 = _get_child(run2, "rPr") + + if (rpr1 is None) != (rpr2 is None): + return False + if rpr1 is None: + return True + return rpr1.toxml() == rpr2.toxml() + + +def _merge_run_content(target, source): + for child in list(source.childNodes): + if child.nodeType == child.ELEMENT_NODE: + name = child.localName or child.tagName + if name != "rPr" and not name.endswith(":rPr"): + target.appendChild(child) + + +def _element_text(elem) -> str: + return "".join( + child.data + for child in elem.childNodes + if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE) + ) + + +def _has_preserve(elem) -> bool: + return elem.getAttribute("xml:space") == "preserve" + + +def _rendered_text(elem) -> str: + return rendered_text(_element_text(elem), _has_preserve(elem)) + + +def _consolidate_text(run): + for tag in ("t", "delText"): + _consolidate_text_elements(run, tag) + + +def _consolidate_text_elements(run, tag: str): + t_elements = _get_children(run, tag) + + for i in range(len(t_elements) - 1, 0, -1): + curr, prev = t_elements[i], t_elements[i - 1] + + if _is_adjacent(prev, curr): + merged = _rendered_text(prev) + _rendered_text(curr) + had_preserve = _has_preserve(prev) or _has_preserve(curr) + + new_text = run.ownerDocument.createTextNode(merged) + for node in list(prev.childNodes): + if node.nodeType in (node.TEXT_NODE, node.CDATA_SECTION_NODE): + prev.removeChild(node) + else: + run.insertBefore(node, curr) + prev.appendChild(new_text) + for node in list(curr.childNodes): + if node.nodeType not in (node.TEXT_NODE, node.CDATA_SECTION_NODE): + run.insertBefore(node, curr) + + if merged != merged.strip(XML_SPACE) or had_preserve: + prev.setAttribute("xml:space", "preserve") + elif prev.hasAttribute("xml:space"): + prev.removeAttribute("xml:space") + + run.removeChild(curr) + + + + +def _merge_or_die(path: Path) -> str: + _, msg = merge_runs(str(path)) + if msg.startswith("Error"): + print(msg, file=sys.stderr) + sys.exit(1) + return msg + + +def main() -> None: + p = argparse.ArgumentParser( + description="Merge adjacent identically-formatted runs in a DOCX (directory or .docx file)." + ) + p.add_argument("input", help="Unpacked DOCX directory OR a .docx/.dotx file") + p.add_argument( + "-o", "--output", + help="Output .docx path (only valid when input is a .docx; default: overwrite input)", + ) + args = p.parse_args() + + src = Path(args.input) + + try: + if src.is_dir(): + if args.output: + p.error("--output is only valid for .docx input; directory input is modified in place") + print(_merge_or_die(src)) + elif src.is_file() and src.suffix.lower() in (".docx", ".dotx"): + out = Path(args.output) if args.output else src + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + with zipfile.ZipFile(src) as zf: + safe_extract(zf, tmp_path) + msg = _merge_or_die(tmp_path) + rezip(tmp_path, out) + print(f"{msg}; wrote {out}") + else: + print(f"Error: {src} is neither a directory nor a .docx/.dotx file", file=sys.stderr) + sys.exit(1) + except (OSError, ValueError, zipfile.BadZipFile) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/__init__.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/__init__.py index e69de29bb2..188b00aff4 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/__init__.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/__init__.py @@ -0,0 +1,150 @@ +import os +import posixpath +import re +import stat +import tempfile +import urllib.parse +import zipfile +from pathlib import Path + +OOXML_FAMILY = { + ".docx": "docx", + ".dotx": "docx", + ".pptx": "pptx", + ".potx": "pptx", + ".xlsx": "xlsx", + ".xltx": "xlsx", +} + +_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:") + +SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" + +MAX_ARCHIVE_MEMBERS = 10_000 +MAX_ARCHIVE_MEMBER_SIZE = 1 * 1024 * 1024 * 1024 +MAX_ARCHIVE_TOTAL_SIZE = 4 * 1024 * 1024 * 1024 +MAX_ARCHIVE_COMPRESSION_RATIO = 1_000 + + +def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None: + if not target: + return None + if target_mode.lower() == "external": + return None + if _SCHEME_RE.match(target): + return None + + target = urllib.parse.unquote(target) + + if "\\" in target: + raise ValueError(f"relationship target is not a POSIX part name: {target!r}") + + if target.startswith("/"): + joined = target.lstrip("/") + else: + joined = posixpath.join(posixpath.dirname(source_part), target) + + parts: list[str] = [] + for segment in posixpath.normpath(joined).split("/"): + if segment in ("", "."): + continue + if segment == "..": + if not parts: + raise ValueError(f"relationship target escapes the package: {target!r}") + parts.pop() + else: + parts.append(segment) + + if not parts: + raise ValueError(f"relationship target resolves to nothing: {target!r}") + return "/".join(parts) + + +def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str: + owner_dir = rels_file.parent.parent.relative_to(unpacked_dir) + return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./") + + +def part_text(data: bytes) -> str: + return data.decode("utf-8", "surrogateescape") + + +XML_SPACE = " \t\r\n" + + +def rendered_text(text: str, preserve: bool) -> str: + return text if preserve else text.strip(XML_SPACE) + + +def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None: + dest = dest.resolve() + members = zf.infolist() + if len(members) > MAX_ARCHIVE_MEMBERS: + raise ValueError(f"archive has too many entries: {len(members)}") + + total_size = 0 + targets: set[str] = set() + file_targets: set[str] = set() + validated: list[tuple[zipfile.ZipInfo, Path]] = [] + for m in members: + if stat.S_ISLNK(m.external_attr >> 16): + raise ValueError(f"symlink archive entry not allowed: {m.filename!r}") + target = (dest / m.filename).resolve() + if target == dest or not target.is_relative_to(dest): + raise ValueError(f"unsafe archive entry: {m.filename!r}") + target_key = os.path.normcase(str(target)) + if target_key in targets: + raise ValueError(f"duplicate archive entry: {m.filename!r}") + targets.add(target_key) + if not m.is_dir(): + file_targets.add(target_key) + validated.append((m, target)) + if m.file_size > MAX_ARCHIVE_MEMBER_SIZE: + raise ValueError(f"archive entry is too large: {m.filename!r}") + total_size += m.file_size + if total_size > MAX_ARCHIVE_TOTAL_SIZE: + raise ValueError("archive expands beyond the allowed total size") + if m.file_size and ( + m.compress_size == 0 + or m.file_size > m.compress_size * MAX_ARCHIVE_COMPRESSION_RATIO + ): + raise ValueError(f"archive entry has an unsafe compression ratio: {m.filename!r}") + + for m, target in validated: + for parent in target.parents: + if parent == dest: + break + if os.path.normcase(str(parent)) in file_targets: + raise ValueError(f"archive file entry conflicts with child path: {m.filename!r}") + + for m, _ in validated: + zf.extract(m, dest) + + +def rezip(src_dir: Path, out_path: Path) -> None: + files = sorted(p for p in src_dir.rglob("*") if p.is_file()) + ct = src_dir / "[Content_Types].xml" + fd, tmp_name = tempfile.mkstemp( + prefix=out_path.name + ".", suffix=".tmp", dir=out_path.parent + ) + tmp_out = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as fh: + with zipfile.ZipFile(fh, "w", zipfile.ZIP_DEFLATED) as zf: + if ct.exists(): + zf.write(ct, ct.relative_to(src_dir), compress_type=zipfile.ZIP_STORED) + for f in files: + if f == ct: + continue + zf.write(f, f.relative_to(src_dir)) + if out_path.exists(): + mode = out_path.stat().st_mode & 0o777 + else: + umask = os.umask(0) + os.umask(umask) + mode = 0o666 & ~umask + os.chmod(tmp_out, mode) + os.replace(tmp_out, out_path) + finally: + if tmp_out.exists(): + tmp_out.unlink() diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/merge_runs.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/merge_runs.py deleted file mode 100644 index ad7c25eec0..0000000000 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/merge_runs.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Merge adjacent runs with identical formatting in DOCX. - -Merges adjacent elements that have identical properties. -Works on runs in paragraphs and inside tracked changes (, ). - -Also: -- Removes rsid attributes from runs (revision metadata that doesn't affect rendering) -- Removes proofErr elements (spell/grammar markers that block merging) -""" - -from pathlib import Path - -import defusedxml.minidom - - -def merge_runs(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - _remove_elements(root, "proofErr") - _strip_run_rsid_attrs(root) - - containers = {run.parentNode for run in _find_elements(root, "r")} - - merge_count = 0 - for container in containers: - merge_count += _merge_runs_in(container) - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Merged {merge_count} runs" - - except Exception as e: - return 0, f"Error: {e}" - - - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def _get_child(parent, tag: str): - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - return child - return None - - -def _get_children(parent, tag: str) -> list: - results = [] - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(child) - return results - - -def _is_adjacent(elem1, elem2) -> bool: - node = elem1.nextSibling - while node: - if node == elem2: - return True - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - return False - - - - -def _remove_elements(root, tag: str): - for elem in _find_elements(root, tag): - if elem.parentNode: - elem.parentNode.removeChild(elem) - - -def _strip_run_rsid_attrs(root): - for run in _find_elements(root, "r"): - for attr in list(run.attributes.values()): - if "rsid" in attr.name.lower(): - run.removeAttribute(attr.name) - - - - -def _merge_runs_in(container) -> int: - merge_count = 0 - run = _first_child_run(container) - - while run: - while True: - next_elem = _next_element_sibling(run) - if next_elem and _is_run(next_elem) and _can_merge(run, next_elem): - _merge_run_content(run, next_elem) - container.removeChild(next_elem) - merge_count += 1 - else: - break - - _consolidate_text(run) - run = _next_sibling_run(run) - - return merge_count - - -def _first_child_run(container): - for child in container.childNodes: - if child.nodeType == child.ELEMENT_NODE and _is_run(child): - return child - return None - - -def _next_element_sibling(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - return sibling - sibling = sibling.nextSibling - return None - - -def _next_sibling_run(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - if _is_run(sibling): - return sibling - sibling = sibling.nextSibling - return None - - -def _is_run(node) -> bool: - name = node.localName or node.tagName - return name == "r" or name.endswith(":r") - - -def _can_merge(run1, run2) -> bool: - rpr1 = _get_child(run1, "rPr") - rpr2 = _get_child(run2, "rPr") - - if (rpr1 is None) != (rpr2 is None): - return False - if rpr1 is None: - return True - return rpr1.toxml() == rpr2.toxml() - - -def _merge_run_content(target, source): - for child in list(source.childNodes): - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name != "rPr" and not name.endswith(":rPr"): - target.appendChild(child) - - -def _consolidate_text(run): - t_elements = _get_children(run, "t") - - for i in range(len(t_elements) - 1, 0, -1): - curr, prev = t_elements[i], t_elements[i - 1] - - if _is_adjacent(prev, curr): - prev_text = prev.firstChild.data if prev.firstChild else "" - curr_text = curr.firstChild.data if curr.firstChild else "" - merged = prev_text + curr_text - - if prev.firstChild: - prev.firstChild.data = merged - else: - prev.appendChild(run.ownerDocument.createTextNode(merged)) - - if merged.startswith(" ") or merged.endswith(" "): - prev.setAttribute("xml:space", "preserve") - elif prev.hasAttribute("xml:space"): - prev.removeAttribute("xml:space") - - run.removeChild(curr) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_chart.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_chart.py new file mode 100644 index 0000000000..209cb7c58b --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_chart.py @@ -0,0 +1,170 @@ +"""Find chart XML that PowerPoint refuses but the schema accepts. + +Detection only: for either fault more than one repair is valid, and only the +author knows which was meant. +""" + + +from __future__ import annotations + +import re +from typing import Mapping + +from . import part_text + + +_CHART_PART_RE = re.compile(r"ppt/charts/chart\d+\.xml") + +_GROUPING_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") +_DLBL_POS_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") + +def _strip_ext_lst(text: str) -> str: + out, cursor = [], 0 + for lo, hi in _ext_lst_spans(text): + out.append(text[cursor:lo]) + cursor = hi + out.append(text[cursor:]) + return "".join(out) + +_BAR_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +STACKED_GROUPINGS = frozenset({"stacked", "percentStacked"}) +ILLEGAL_ON_STACKED = frozenset({"outEnd"}) +LEGAL_ON_STACKED = ("ctr", "inEnd", "inBase") + + +def _check_stacked_label_positions(part: str, xml: str) -> list[str]: + problems: list[str] = [] + for match in _BAR_GROUP_RE.finditer(xml): + block = _strip_ext_lst(match.group(0)) + group = match.group(1) + + grouping = _GROUPING_RE.search(block) + if grouping is None or grouping.group(1) not in STACKED_GROUPINGS: + continue + + bad = [p for p in _DLBL_POS_RE.findall(block) if p in ILLEGAL_ON_STACKED] + for pos in sorted(set(bad)): + problems.append( + f'{part}: {bad.count(pos)} data label(s) use dLblPos="{pos}" on a ' + f"{grouping.group(1)} {group}; PowerPoint allows only " + f"{', '.join(LEGAL_ON_STACKED)} there" + ) + return problems + + + +_ANY_CHART_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +_AXID_RE = re.compile( + r"""\s*]*?\bval=["'](-?\d+)["']\s*(?:/>|>\s*)""" +) + +_AXIS_DECL_RE = re.compile( + r"""]*(?\s*]*?\bval=["'](-?\d+)["']""" +) + +AXID_LIMIT = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 3, "line3DChart": 3, "area3DChart": 3, + "surfaceChart": 3, "surface3DChart": 3, +} + +AXID_MINIMUM = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 2, "area3DChart": 2, "surfaceChart": 2, + "line3DChart": 3, "surface3DChart": 3, +} + + +def _declared_axes(xml: str) -> dict[str, list[str]]: + axes: dict[str, list[str]] = {} + for kind, axid in _AXIS_DECL_RE.findall(xml): + axes.setdefault(kind, []).append(axid) + return axes + + +def _canonical_ids(axes: dict[str, list[str]], limit: int) -> list[str] | None: + category = axes.get("catAx", []) + axes.get("dateAx", []) + value = axes.get("valAx", []) + series = axes.get("serAx", []) + if len(category) != 1 or len(value) != 1 or len(series) > 1: + return None + ids = [category[0], value[0]] + if limit >= 3 and series: + ids.append(series[0]) + return ids + + +def _undeclared_axes(kind: str, block: str, axes: dict[str, list[str]]) -> list[str] | None: + if kind not in AXID_LIMIT: + return None + ids = _AXID_RE.findall(block) + declared = {i for group in axes.values() for i in group} + if len([i for i in ids if i in declared]) >= 2: + return None + return ids + + +def _check_chart_axis_references(part: str, xml: str) -> list[str]: + axes = _declared_axes(xml) + problems: list[str] = [] + declared = {i for group in axes.values() for i in group} + for match in _ANY_CHART_GROUP_RE.finditer(xml): + kind, block = match.group(1), match.group(0) + ids = _undeclared_axes(kind, block, axes) + if ids is None: + continue + if not ids: + problems.append( + f"{part}: declares no this part can resolve; a chart " + f"group needs {AXID_MINIMUM[kind]}, and PowerPoint discards one with fewer" + ) + continue + dead = [i for i in ids if i not in declared] + canonical = _canonical_ids(axes, AXID_LIMIT[kind]) + if canonical is not None and len(canonical) >= AXID_MINIMUM[kind]: + hint = f"Fix: point them at the axes this part declares ({', '.join(canonical)})" + else: + hint = ("Fix: the part declares several axes of a kind -- declare the " + "secondary axes the series expects, or drop them") + detail = (f"of which {', '.join(dead)} name no declared axis" + if dead else f"only {len(ids)} of which this part declares") + problems.append( + f"{part}: references axId {', '.join(ids)}, {detail}, " + f"leaving fewer than two live axes; PowerPoint discards the chart. {hint}" + ) + return problems + + +def _ext_lst_spans(text: str) -> list[tuple[int, int]]: + spans: list[tuple[int, int]] = [] + depth = 0 + start = 0 + for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text): + closing, self_closing = match.group(1), match.group(2) + if self_closing: + continue + if closing: + depth -= 1 + if depth == 0: + spans.append((start, match.end())) + else: + if depth == 0: + start = match.start() + depth += 1 + return spans + + +CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references) + + +def find_chart_problems(files: Mapping[str, bytes]) -> list[str]: + problems: list[str] = [] + for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)): + xml = part_text(files[part]) + for check in CHART_CHECKS: + problems.extend(check(part, xml)) + return problems diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_slide.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_slide.py new file mode 100644 index 0000000000..22f9aee0ff --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_slide.py @@ -0,0 +1,60 @@ +"""Pick the slide-XML schema errors PowerPoint refuses the file over. + +A denylist over lxml's messages, so an unrecognised error class is a miss rather +than a false alarm. +""" + + +from __future__ import annotations + +import re + +SLIDE_PART_RE = re.compile( + r"ppt/(slides|slideLayouts|slideMasters|notesSlides|notesMasters|handoutMasters)" + r"/[^/]+\.xml" +) + +FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"\}tableStyleId': This element is not expected"), + "two in one (the schema allows one)", + ), + ( + re.compile(r"\}srgbClr', attribute 'val'"), + "a colour that is not six hex digits", + ), + ( + re.compile(r"\}txBody': Missing child element"), + "a with no children", + ), + ( + re.compile(r"\}miter', attribute 'lim'"), + 'a line join with lim="NaN"', + ), + ( + re.compile(r"\}uLnTx': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}overrideClrMapping': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}nvGrpSpPr': Missing child element"), + "a with no children", + ), +) + + +def is_schema_verdict(error: str) -> bool: + return error.startswith("Element ") + + +def fatal_slide_errors(errors: set[str]) -> list[str]: + out = [] + for error in sorted(errors): + for pattern, meaning in FATAL_SLIDE_ERRORS: + if pattern.search(error): + out.append(f"{meaning}: {error}") + break + return out diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_theme.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_theme.py new file mode 100644 index 0000000000..5ef4c3e835 --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/pptx_theme.py @@ -0,0 +1,114 @@ +"""Find masters sharing a theme part in the way PowerPoint refuses to open. + +Reports only; the fix is to move back to directly after + in ppt/presentation.xml. +""" + + +from __future__ import annotations + +import posixpath +import re +from typing import Mapping + +from . import part_text + +THEME_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" + +_MASTER_RE = re.compile( + r"^ppt/(?PslideMasters|notesMasters|handoutMasters)/" + r"(?:slide|notes|handout)Master(?P\d+)\.xml$" +) +_GROUP_ORDER = {"slideMasters": 0, "notesMasters": 1, "handoutMasters": 2} + +_RELATIONSHIP_RE = re.compile( + r"]*?(?:/>|>.*?)", re.DOTALL +) + + +def _sort_key(name: str) -> tuple[int, int]: + m = _MASTER_RE.match(name) + assert m is not None + return (_GROUP_ORDER[m.group("group")], int(m.group("num"))) + + +def _rels_path(part: str) -> str: + directory, base = posixpath.split(part) + return f"{directory}/_rels/{base}.rels" + + +def _resolve(rels_path: str, target: str) -> str: + if target.startswith("/"): + return target.lstrip("/") + part_dir = posixpath.dirname(posixpath.dirname(rels_path)) + return posixpath.normpath(posixpath.join(part_dir, target)) + + +def _theme_rel(files: Mapping[str, bytes], master: str): + rels_path = _rels_path(master) + rels = files.get(rels_path) + if rels is None: + return None + for element in _RELATIONSHIP_RE.findall(part_text(rels)): + if f'Type="{THEME_REL_TYPE}"' not in element: + continue + target = re.search(r'\bTarget="([^"]+)"', element) + if target is None: + continue + return rels_path, element, _resolve(rels_path, target.group(1)) + return None + + +def _masters(files: Mapping[str, bytes]) -> list[str]: + return sorted((n for n in files if _MASTER_RE.match(n)), key=_sort_key) + + +_PRESENTATION = "ppt/presentation.xml" +_NOTES_MASTERS = "ppt/notesMasters/" +_IGNORABLE_RE = re.compile(r"|<\?.*?\?>", re.DOTALL) +_AFTER_SLDIDLST_RE = re.compile( + r"]*/>|[^>]*>.*?)\s*(<[^>\s/]+)", re.DOTALL +) + + +def _notes_master_share_is_inert(files: Mapping[str, bytes]) -> bool: + data = files.get(_PRESENTATION) + if data is None: + return False + match = _AFTER_SLDIDLST_RE.search(_IGNORABLE_RE.sub("", part_text(data))) + return match is not None and match.group(1) == " bool: + return inert_notes and master.startswith(_NOTES_MASTERS) + + +def find_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + ] + + +def live_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + inert_notes = _notes_master_share_is_inert(files) + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + if not _is_inert(master, inert_notes) + ] diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/simplify_redlines.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/simplify_redlines.py deleted file mode 100644 index db963bb998..0000000000 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/helpers/simplify_redlines.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Simplify tracked changes by merging adjacent w:ins or w:del elements. - -Merges adjacent elements from the same author into a single element. -Same for elements. This makes heavily-redlined documents easier to -work with by reducing the number of tracked change wrappers. - -Rules: -- Only merges w:ins with w:ins, w:del with w:del (same element type) -- Only merges if same author (ignores timestamp differences) -- Only merges if truly adjacent (only whitespace between them) -""" - -import xml.etree.ElementTree as ET -import zipfile -from pathlib import Path - -import defusedxml.minidom - -WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" - - -def simplify_redlines(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - merge_count = 0 - - containers = _find_elements(root, "p") + _find_elements(root, "tc") - - for container in containers: - merge_count += _merge_tracked_changes_in(container, "ins") - merge_count += _merge_tracked_changes_in(container, "del") - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Simplified {merge_count} tracked changes" - - except Exception as e: - return 0, f"Error: {e}" - - -def _merge_tracked_changes_in(container, tag: str) -> int: - merge_count = 0 - - tracked = [ - child - for child in container.childNodes - if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag) - ] - - if len(tracked) < 2: - return 0 - - i = 0 - while i < len(tracked) - 1: - curr = tracked[i] - next_elem = tracked[i + 1] - - if _can_merge_tracked(curr, next_elem): - _merge_tracked_content(curr, next_elem) - container.removeChild(next_elem) - tracked.pop(i + 1) - merge_count += 1 - else: - i += 1 - - return merge_count - - -def _is_element(node, tag: str) -> bool: - name = node.localName or node.tagName - return name == tag or name.endswith(f":{tag}") - - -def _get_author(elem) -> str: - author = elem.getAttribute("w:author") - if not author: - for attr in elem.attributes.values(): - if attr.localName == "author" or attr.name.endswith(":author"): - return attr.value - return author - - -def _can_merge_tracked(elem1, elem2) -> bool: - if _get_author(elem1) != _get_author(elem2): - return False - - node = elem1.nextSibling - while node and node != elem2: - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - - return True - - -def _merge_tracked_content(target, source): - while source.firstChild: - child = source.firstChild - source.removeChild(child) - target.appendChild(child) - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def get_tracked_change_authors(doc_xml_path: Path) -> dict[str, int]: - if not doc_xml_path.exists(): - return {} - - try: - tree = ET.parse(doc_xml_path) - root = tree.getroot() - except ET.ParseError: - return {} - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - - return authors - - -def _get_authors_from_docx(docx_path: Path) -> dict[str, int]: - try: - with zipfile.ZipFile(docx_path, "r") as zf: - if "word/document.xml" not in zf.namelist(): - return {} - with zf.open("word/document.xml") as f: - tree = ET.parse(f) - root = tree.getroot() - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - return authors - except (zipfile.BadZipFile, ET.ParseError): - return {} - - -def infer_author(modified_dir: Path, original_docx: Path, default: str = "Claude") -> str: - modified_xml = modified_dir / "word" / "document.xml" - modified_authors = get_tracked_change_authors(modified_xml) - - if not modified_authors: - return default - - original_authors = _get_authors_from_docx(original_docx) - - new_changes: dict[str, int] = {} - for author, count in modified_authors.items(): - original_count = original_authors.get(author, 0) - diff = count - original_count - if diff > 0: - new_changes[author] = diff - - if not new_changes: - return default - - if len(new_changes) == 1: - return next(iter(new_changes)) - - raise ValueError( - f"Multiple authors added new changes: {new_changes}. " - "Cannot infer which author to validate." - ) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/pack.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/pack.py deleted file mode 100755 index db29ed8b1c..0000000000 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/pack.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Pack a directory into a DOCX, PPTX, or XLSX file. - -Validates with auto-repair, condenses XML formatting, and creates the Office file. - -Usage: - python pack.py [--original ] [--validate true|false] - -Examples: - python pack.py unpacked/ output.docx --original input.docx - python pack.py unpacked/ output.pptx --validate false -""" - -import argparse -import sys -import shutil -import tempfile -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator - -def pack( - input_directory: str, - output_file: str, - original_file: str | None = None, - validate: bool = True, - infer_author_func=None, -) -> tuple[None, str]: - input_dir = Path(input_directory) - output_path = Path(output_file) - suffix = output_path.suffix.lower() - - if not input_dir.is_dir(): - return None, f"Error: {input_dir} is not a directory" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {output_file} must be a .docx, .pptx, or .xlsx file" - - if validate and original_file: - original_path = Path(original_file) - if original_path.exists(): - success, output = _run_validation( - input_dir, original_path, suffix, infer_author_func - ) - if output: - print(output) - if not success: - return None, f"Error: Validation failed for {input_dir}" - - with tempfile.TemporaryDirectory() as temp_dir: - temp_content_dir = Path(temp_dir) / "content" - shutil.copytree(input_dir, temp_content_dir) - - for pattern in ["*.xml", "*.rels"]: - for xml_file in temp_content_dir.rglob(pattern): - _condense_xml(xml_file) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf: - for f in temp_content_dir.rglob("*"): - if f.is_file(): - zf.write(f, f.relative_to(temp_content_dir)) - - return None, f"Successfully packed {input_dir} to {output_file}" - - -def _run_validation( - unpacked_dir: Path, - original_file: Path, - suffix: str, - infer_author_func=None, -) -> tuple[bool, str | None]: - output_lines = [] - validators = [] - - if suffix == ".docx": - author = "Claude" - if infer_author_func: - try: - author = infer_author_func(unpacked_dir, original_file) - except ValueError as e: - print(f"Warning: {e} Using default author 'Claude'.", file=sys.stderr) - - validators = [ - DOCXSchemaValidator(unpacked_dir, original_file), - RedliningValidator(unpacked_dir, original_file, author=author), - ] - elif suffix == ".pptx": - validators = [PPTXSchemaValidator(unpacked_dir, original_file)] - - if not validators: - return True, None - - total_repairs = sum(v.repair() for v in validators) - if total_repairs: - output_lines.append(f"Auto-repaired {total_repairs} issue(s)") - - success = all(v.validate() for v in validators) - - if success: - output_lines.append("All validations PASSED!") - - return success, "\n".join(output_lines) if output_lines else None - - -def _condense_xml(xml_file: Path) -> None: - try: - with open(xml_file, encoding="utf-8") as f: - dom = defusedxml.minidom.parse(f) - - for element in dom.getElementsByTagName("*"): - if element.tagName.endswith(":t"): - continue - - for child in list(element.childNodes): - if ( - child.nodeType == child.TEXT_NODE - and child.nodeValue - and child.nodeValue.strip() == "" - ) or child.nodeType == child.COMMENT_NODE: - element.removeChild(child) - - xml_file.write_bytes(dom.toxml(encoding="UTF-8")) - except Exception as e: - print(f"ERROR: Failed to parse {xml_file.name}: {e}", file=sys.stderr) - raise - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Pack a directory into a DOCX, PPTX, or XLSX file" - ) - parser.add_argument("input_directory", help="Unpacked Office document directory") - parser.add_argument("output_file", help="Output Office file (.docx/.pptx/.xlsx)") - parser.add_argument( - "--original", - help="Original file for validation comparison", - ) - parser.add_argument( - "--validate", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Run validation with auto-repair (default: true)", - ) - args = parser.parse_args() - - _, message = pack( - args.input_directory, - args.output_file, - original_file=args.original, - validate=args.validate, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/soffice.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/soffice.py index c7f7e3289f..0b4c99deca 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/soffice.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/soffice.py @@ -4,20 +4,23 @@ at runtime and applies an LD_PRELOAD shim if needed. Usage: - from office.soffice import run_soffice, get_soffice_env + from office.soffice import run_soffice - # Option 1 – run soffice directly result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"]) - # Option 2 – get env dict for your own subprocess calls - env = get_soffice_env() - subprocess.run(["soffice", ...], env=env) +Call soffice through run_soffice, not through subprocess with get_soffice_env(): +the env dict carries the shim but names no user profile, and a non-root sandbox +cannot bootstrap the default one -- soffice aborts with "User installation could +not be completed" and converts nothing. get_soffice_env() stays public for the +callers that build their own argv (they must pass -env:UserInstallation too). """ +import contextlib import os import socket import subprocess import tempfile +from collections.abc import Iterable from pathlib import Path @@ -32,9 +35,15 @@ def get_soffice_env() -> dict: return env -def run_soffice(args: list[str], **kwargs) -> subprocess.CompletedProcess: - env = get_soffice_env() - return subprocess.run(["soffice"] + args, env=env, **kwargs) +def run_soffice(args: Iterable[str], **kwargs) -> subprocess.CompletedProcess: + args = list(args) + with contextlib.ExitStack() as stack: + if not any(str(a).startswith("-env:UserInstallation") for a in args): + profile = stack.enter_context( + tempfile.TemporaryDirectory(prefix="lo_profile_", ignore_cleanup_errors=True) + ) + args = [f"-env:UserInstallation={Path(profile).as_uri()}"] + args + return subprocess.run(["soffice"] + args, env=get_soffice_env(), **kwargs) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/unpack.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/unpack.py deleted file mode 100755 index 00152533ac..0000000000 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/unpack.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Unpack Office files (DOCX, PPTX, XLSX) for editing. - -Extracts the ZIP archive, pretty-prints XML files, and optionally: -- Merges adjacent runs with identical formatting (DOCX only) -- Simplifies adjacent tracked changes from same author (DOCX only) - -Usage: - python unpack.py [options] - -Examples: - python unpack.py document.docx unpacked/ - python unpack.py presentation.pptx unpacked/ - python unpack.py document.docx unpacked/ --merge-runs false -""" - -import argparse -import sys -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from helpers.merge_runs import merge_runs as do_merge_runs -from helpers.simplify_redlines import simplify_redlines as do_simplify_redlines - -SMART_QUOTE_REPLACEMENTS = { - "\u201c": "“", - "\u201d": "”", - "\u2018": "‘", - "\u2019": "’", -} - - -def unpack( - input_file: str, - output_directory: str, - merge_runs: bool = True, - simplify_redlines: bool = True, -) -> tuple[None, str]: - input_path = Path(input_file) - output_path = Path(output_directory) - suffix = input_path.suffix.lower() - - if not input_path.exists(): - return None, f"Error: {input_file} does not exist" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {input_file} must be a .docx, .pptx, or .xlsx file" - - try: - output_path.mkdir(parents=True, exist_ok=True) - - with zipfile.ZipFile(input_path, "r") as zf: - zf.extractall(output_path) - - xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels")) - for xml_file in xml_files: - _pretty_print_xml(xml_file) - - message = f"Unpacked {input_file} ({len(xml_files)} XML files)" - - if suffix == ".docx": - if simplify_redlines: - simplify_count, _ = do_simplify_redlines(str(output_path)) - message += f", simplified {simplify_count} tracked changes" - - if merge_runs: - merge_count, _ = do_merge_runs(str(output_path)) - message += f", merged {merge_count} runs" - - for xml_file in xml_files: - _escape_smart_quotes(xml_file) - - return None, message - - except zipfile.BadZipFile: - return None, f"Error: {input_file} is not a valid Office file" - except Exception as e: - return None, f"Error unpacking: {e}" - - -def _pretty_print_xml(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - dom = defusedxml.minidom.parseString(content) - xml_file.write_bytes(dom.toprettyxml(indent=" ", encoding="utf-8")) - except Exception: - pass - - -def _escape_smart_quotes(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - for char, entity in SMART_QUOTE_REPLACEMENTS.items(): - content = content.replace(char, entity) - xml_file.write_text(content, encoding="utf-8") - except Exception: - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Unpack an Office file (DOCX, PPTX, XLSX) for editing" - ) - parser.add_argument("input_file", help="Office file to unpack") - parser.add_argument("output_directory", help="Output directory") - parser.add_argument( - "--merge-runs", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent runs with identical formatting (DOCX only, default: true)", - ) - parser.add_argument( - "--simplify-redlines", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent tracked changes from same author (DOCX only, default: true)", - ) - args = parser.parse_args() - - _, message = unpack( - args.input_file, - args.output_directory, - merge_runs=args.merge_runs, - simplify_redlines=args.simplify_redlines, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validate.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validate.py index 03b01f6e3b..8fbd2f71ca 100755 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validate.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validate.py @@ -6,7 +6,7 @@ The first argument can be either: - An unpacked directory containing the Office document XML files -- A packed Office file (.docx/.pptx/.xlsx) which will be unpacked to a temp directory +- A packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx template) which will be unpacked to a temp directory Auto-repair fixes: - paraId/durableId values that exceed OOXML limits @@ -19,20 +19,43 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import OOXML_FAMILY, rezip, safe_extract from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator +WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + +def _fail(message: str): + print(f"Error: {message}", file=sys.stderr) + sys.exit(2) + + +def _has_tracked_changes(unpacked_dir: Path) -> bool: + document = unpacked_dir / "word" / "document.xml" + if not document.is_file(): + return False + try: + root = ET.parse(document).getroot() + except (ET.ParseError, DefusedXmlException): + return False + tracked = {f"{{{WORD_NS}}}ins", f"{{{WORD_NS}}}del"} + return any(elem.tag in tracked for elem in root.iter()) + def main(): parser = argparse.ArgumentParser(description="Validate Office document XML files") parser.add_argument( "path", - help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx)", + help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx)", ) parser.add_argument( "--original", required=False, default=None, - help="Path to original file (.docx/.pptx/.xlsx). If omitted, all XSD errors are reported and redlining validation is skipped.", + help="Path to original file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx). If omitted, all XSD errors are reported and redlining validation is skipped.", ) parser.add_argument( "-v", @@ -43,63 +66,102 @@ def main(): parser.add_argument( "--auto-repair", action="store_true", - help="Automatically repair common issues (hex IDs, whitespace preservation)", + help="Automatically repair common issues (hex IDs, whitespace preservation). " + "Modifies the input in place: repairs to a packed file are written back to it.", ) parser.add_argument( "--author", - default="Claude", - help="Author name for redlining validation (default: Claude)", + default=None, + help="The name you are redlining under. Passing it turns on the " + "tracked-change check: any text differing from --original without a " + "/ recording it is reported. Untracked edits carry no " + "author, so the check covers them whoever made them — the name marks " + "the run as redlining work and is not used to filter. Requires " + "--original; docx only.", ) args = parser.parse_args() + if args.author is not None and not args.original: + _fail("--author requires --original") + path = Path(args.path) - assert path.exists(), f"Error: {path} does not exist" + if not path.exists(): + _fail(f"{path} does not exist") original_file = None if args.original: original_file = Path(args.original) - assert original_file.is_file(), f"Error: {original_file} is not a file" - assert original_file.suffix.lower() in [".docx", ".pptx", ".xlsx"], ( - f"Error: {original_file} must be a .docx, .pptx, or .xlsx file" + if not original_file.is_file(): + _fail(f"{original_file} is not a file") + if original_file.suffix.lower() not in OOXML_FAMILY: + _fail(f"{original_file} must be one of: {', '.join(sorted(OOXML_FAMILY))}") + + family = OOXML_FAMILY.get((original_file or path).suffix.lower()) + if family is None: + _fail( + f"Cannot determine file type from {path}. Use --original or provide one of: {', '.join(sorted(OOXML_FAMILY))}." ) - file_extension = (original_file or path).suffix.lower() - assert file_extension in [".docx", ".pptx", ".xlsx"], ( - f"Error: Cannot determine file type from {path}. Use --original or provide a .docx/.pptx/.xlsx file." - ) - - if path.is_file() and path.suffix.lower() in [".docx", ".pptx", ".xlsx"]: - temp_dir = tempfile.mkdtemp() - with zipfile.ZipFile(path, "r") as zf: - zf.extractall(temp_dir) - unpacked_dir = Path(temp_dir) + if args.author is not None and family != "docx": + _fail(f"--author only applies to docx files, not {family}") + + packed_file = None + temp_dir_ctx = None + if path.is_file() and path.suffix.lower() in OOXML_FAMILY: + packed_file = path + temp_dir_ctx = tempfile.TemporaryDirectory() + unpacked_dir = Path(temp_dir_ctx.name) + try: + with zipfile.ZipFile(path, "r") as zf: + safe_extract(zf, unpacked_dir) + except (zipfile.BadZipFile, ValueError, OSError) as e: + _fail(f"cannot unpack {path}: {e}") else: - assert path.is_dir(), f"Error: {path} is not a directory or Office file" + if not path.is_dir(): + _fail(f"{path} is not a directory or Office file") unpacked_dir = path - match file_extension: - case ".docx": + match family: + case "docx": validators = [ DOCXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] - if original_file: + if args.author is not None: validators.append( - RedliningValidator(unpacked_dir, original_file, verbose=args.verbose, author=args.author) + RedliningValidator(unpacked_dir, original_file, verbose=args.verbose) + ) + elif original_file and _has_tracked_changes(unpacked_dir): + print( + "Note: this document has tracked changes; they were not " + "checked against the original (pass --author to check)." ) - case ".pptx": + case "pptx": validators = [ PPTXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] + case "xlsx": + exts = ", ".join(k for k, v in sorted(OOXML_FAMILY.items()) if v == "xlsx") + print( + f"No XSD schema validation is performed for xlsx-family files ({exts}). " + "For formula-error checking, use scripts/recalc.py instead." + ) + sys.exit(0) case _: - print(f"Error: Validation not supported for file type {file_extension}") + print(f"Error: Validation not supported for file type {family}") sys.exit(1) if args.auto_repair: total_repairs = sum(v.repair() for v in validators) if total_repairs: print(f"Auto-repaired {total_repairs} issue(s)") + if packed_file is not None: + rezip(unpacked_dir, packed_file) + print(f"Wrote repaired file to {packed_file}") + + success = all([v.validate() for v in validators]) - success = all(v.validate() for v in validators) + if temp_dir_ctx is not None: + temp_dir_ctx.cleanup() if success: print("All validations PASSED!") diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/base.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/base.py index db4a06a229..19d52a7fe0 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/base.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/base.py @@ -6,8 +6,20 @@ from pathlib import Path import defusedxml.minidom +from functools import lru_cache + import lxml.etree +from helpers import safe_extract + + +@lru_cache(maxsize=None) +def _load_schema(schema_path: str): + with open(schema_path, "rb") as xsd_file: + xsd_doc = lxml.etree.parse( + xsd_file, parser=lxml.etree.XMLParser(), base_url=schema_path + ) + return lxml.etree.XMLSchema(xsd_doc) class BaseSchemaValidator: @@ -119,21 +131,28 @@ def repair_whitespace_preservation(self) -> int: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) - modified = False + pending = [] for elem in dom.getElementsByTagName("*"): - if elem.tagName.endswith(":t") and elem.firstChild: - text = elem.firstChild.nodeValue - if text and (text.startswith((' ', '\t')) or text.endswith((' ', '\t'))): + local_name = elem.tagName.rsplit(":", 1)[-1] + if local_name in ("t", "delText", "instrText", "delInstrText"): + text = "".join( + child.data + for child in elem.childNodes + if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE) + ) + ws = (" ", "\t", "\n", "\r") + if text and (text.startswith(ws) or text.endswith(ws)): if elem.getAttribute("xml:space") != "preserve": elem.setAttribute("xml:space", "preserve") text_preview = repr(text[:30]) + "..." if len(text) > 30 else repr(text) - print(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - repairs += 1 - modified = True + pending.append(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - if modified: + if pending: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass @@ -212,6 +231,8 @@ def validate_unique_ids(self): elem.getparent().remove(elem) for elem in root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue tag = ( elem.tag.split("}")[-1].lower() if "}" in elem.tag @@ -326,6 +347,8 @@ def validate_file_references(self): namespaces={"ns": self.PACKAGE_RELATIONSHIPS_NAMESPACE}, ): target = rel.get("Target") + if rel.get("TargetMode") == "External": + continue if target and not target.startswith( ("http", "mailto:") ): @@ -423,6 +446,8 @@ def validate_all_relationship_ids(self): r_ns = self.OFFICE_RELATIONSHIPS_NAMESPACE rid_attrs_to_check = ["id", "embed", "link"] for elem in xml_root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue for attr_name in rid_attrs_to_check: rid_attr = elem.get(f"{{{r_ns}}}{attr_name}") if not rid_attr: @@ -747,18 +772,16 @@ def _preprocess_for_mc_ignorable(self, xml_doc): return xml_doc - def _validate_single_file_xsd(self, xml_file, base_path): - schema_path = self._get_schema_path(xml_file) + def _preprocess_for_schema(self, xml_doc, relative_path): + return xml_doc + + def _validate_single_file_xsd(self, xml_file, base_path, schema_path=None): + schema_path = schema_path or self._get_schema_path(xml_file) if not schema_path: return None, None try: - with open(schema_path, "rb") as xsd_file: - parser = lxml.etree.XMLParser() - xsd_doc = lxml.etree.parse( - xsd_file, parser=parser, base_url=str(schema_path) - ) - schema = lxml.etree.XMLSchema(xsd_doc) + schema = _load_schema(str(schema_path)) with open(xml_file, "r") as f: xml_doc = lxml.etree.parse(f) @@ -773,6 +796,8 @@ def _validate_single_file_xsd(self, xml_file, base_path): ): xml_doc = self._clean_ignorable_namespaces(xml_doc) + xml_doc = self._preprocess_for_schema(xml_doc, relative_path) + if schema.validate(xml_doc): return True, set() else: @@ -784,7 +809,7 @@ def _validate_single_file_xsd(self, xml_file, base_path): except Exception as e: return False, {str(e)} - def _get_original_file_errors(self, xml_file): + def _get_original_file_errors(self, xml_file, schema_path=None): if self.original_file is None: return set() @@ -798,8 +823,11 @@ def _get_original_file_errors(self, xml_file): with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - with zipfile.ZipFile(self.original_file, "r") as zip_ref: - zip_ref.extractall(temp_path) + try: + with zipfile.ZipFile(self.original_file, "r") as zip_ref: + safe_extract(zip_ref, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() original_xml_file = temp_path / relative_path @@ -807,7 +835,7 @@ def _get_original_file_errors(self, xml_file): return set() is_valid, errors = self._validate_single_file_xsd( - original_xml_file, temp_path + original_xml_file, temp_path, schema_path=schema_path ) return errors if errors else set() diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/docx.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/docx.py index fec405e694..0d18b6979a 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/docx.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/docx.py @@ -6,10 +6,13 @@ import re import tempfile import zipfile +from pathlib import Path import defusedxml.minidom import lxml.etree +from helpers import safe_extract + from .base import BaseSchemaValidator @@ -186,7 +189,7 @@ def count_paragraphs_in_original(self): try: with tempfile.TemporaryDirectory() as temp_dir: with zipfile.ZipFile(original, "r") as zip_ref: - zip_ref.extractall(temp_dir) + safe_extract(zip_ref, Path(temp_dir)) doc_xml_path = temp_dir + "/word/document.xml" root = lxml.etree.parse(doc_xml_path).getroot() @@ -241,9 +244,12 @@ def validate_insertions(self): return True def compare_paragraph_counts(self): - original_count = self.count_paragraphs_in_original() new_count = self.count_paragraphs_in_unpacked() + if self.original_file is None: + print(f"\nParagraphs: {new_count}") + return + original_count = self.count_paragraphs_in_original() diff = new_count - original_count diff_str = f"+{diff}" if diff > 0 else str(diff) print(f"\nParagraphs: {original_count} → {new_count} ({diff_str})") @@ -260,9 +266,15 @@ def validate_id_constraints(self): try: for elem in lxml.etree.parse(str(xml_file)).iter(): if val := elem.get(para_id_attr): - if self._parse_id_value(val, base=16) >= 0x80000000: + try: + if self._parse_id_value(val, base=16) >= 0x80000000: + errors.append( + f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + ) + except ValueError: errors.append( - f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + f" {xml_file.name}:{elem.sourceline}: " + f"paraId={val} is not valid hex" ) if val := elem.get(durable_id_attr): @@ -279,13 +291,19 @@ def validate_id_constraints(self): f"durableId={val} must be decimal in numbering.xml" ) else: - if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + try: + if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + errors.append( + f" {xml_file.name}:{elem.sourceline}: " + f"durableId={val} >= 0x7FFFFFFF" + ) + except ValueError: errors.append( f" {xml_file.name}:{elem.sourceline}: " - f"durableId={val} >= 0x7FFFFFFF" + f"durableId={val} is not valid hex" ) - except Exception: - pass + except lxml.etree.XMLSyntaxError: + continue if errors: print(f"FAILED - {len(errors)} ID constraint violations:") @@ -389,52 +407,54 @@ def repair(self) -> int: return repairs def repair_durableId(self) -> int: + DURABLE_ID_ATTRS = ("w16cid:durableId", "w16cex:durableId") repairs = 0 + renames: dict = {} for xml_file in self.xml_files: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) + is_numbering = xml_file.name == "numbering.xml" + base = 10 if is_numbering else 16 + pending = [] + seen_in_file = set() modified = False for elem in dom.getElementsByTagName("*"): - if not elem.hasAttribute("w16cid:durableId"): - continue + for attr_name in DURABLE_ID_ATTRS: + if not elem.hasAttribute(attr_name): + continue - durable_id = elem.getAttribute("w16cid:durableId") - needs_repair = False - - if xml_file.name == "numbering.xml": + durable_id = elem.getAttribute(attr_name) try: - needs_repair = ( - self._parse_id_value(durable_id, base=10) >= 0x7FFFFFFF - ) + key = self._parse_id_value(durable_id, base=base) + needs_repair = key >= 0x7FFFFFFF except ValueError: + key = durable_id needs_repair = True - else: - try: - needs_repair = ( - self._parse_id_value(durable_id, base=16) >= 0x7FFFFFFF - ) - except ValueError: - needs_repair = True - - if needs_repair: - value = random.randint(1, 0x7FFFFFFE) - if xml_file.name == "numbering.xml": - new_id = str(value) - else: - new_id = f"{value:08X}" - elem.setAttribute("w16cid:durableId", new_id) - print( - f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" - ) - repairs += 1 - modified = True + if needs_repair: + if key in seen_in_file: + value = random.randint(1, 0x7FFFFFFE) + else: + seen_in_file.add(key) + if key not in renames: + renames[key] = random.randint(1, 0x7FFFFFFE) + value = renames[key] + new_id = str(value) if is_numbering else f"{value:08X}" + + elem.setAttribute(attr_name, new_id) + pending.append( + f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" + ) + modified = True if modified: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/pptx.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/pptx.py index 09842aa998..7b53d0d3e4 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/pptx.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/pptx.py @@ -3,6 +3,9 @@ """ import re +from pathlib import Path + +from helpers import opc_target, rels_source_part, safe_extract from .base import BaseSchemaValidator @@ -57,8 +60,171 @@ def validate(self): if not self.validate_no_duplicate_slide_layouts(): all_valid = False + if not self.validate_master_theme_uniqueness(): + all_valid = False + + if not self.validate_charts(): + all_valid = False + + if not self.validate_slides(): + all_valid = False + return all_valid + def _package_map(self) -> dict: + wanted = [] + wanted += list(self.unpacked_dir.glob("[[]Content_Types[]].xml")) + wanted += list(self.unpacked_dir.glob("ppt/presentation.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/*.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/_rels/*.rels")) + wanted += list(self.unpacked_dir.glob("ppt/charts/chart*.xml")) + for group in ("slideMasters", "notesMasters", "handoutMasters"): + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/*.xml")) + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/_rels/*.rels")) + return { + p.relative_to(self.unpacked_dir).as_posix(): p.read_bytes() + for p in wanted + if p.is_file() + } + + def validate_master_theme_uniqueness(self): + from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes + + shared = live_shared_master_themes(self._package_map()) + if shared: + print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:") + for message in shared: + print(f" {message}") + if any(m.startswith(_NOTES_MASTERS) for m in shared): + print(" Fix: in ppt/presentation.xml, move back to " + "directly after . PowerPoint reads that happily.") + else: + print(" Fix: give each master its own theme part.") + return False + + if self.verbose: + print("PASSED - No master shares a theme part in a way PowerPoint refuses") + return True + + def validate_charts(self): + from helpers.pptx_chart import find_chart_problems + + problems = find_chart_problems(self._package_map()) + if problems: + print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:") + for message in problems: + print(f" {message}") + return False + + if self.verbose: + print("PASSED - Charts satisfy the constraints PowerPoint enforces") + return True + + def _original_slide_defects(self, schema) -> set[str]: + import tempfile + import zipfile + + from helpers.pptx_slide import SLIDE_PART_RE, fatal_slide_errors + + if self.original_file is None: + return set() + + found: set[str] = set() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + try: + with zipfile.ZipFile(self.original_file, "r") as zf: + safe_extract(zf, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() + + for part in sorted(temp_path.rglob("*.xml")): + relative = part.relative_to(temp_path).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + part.resolve(), temp_path.resolve(), schema_path=schema + ) + if ok is None or ok or not errors: + continue + found |= set(fatal_slide_errors(set(errors))) + return found + + def validate_slides(self): + from helpers.pptx_slide import ( + SLIDE_PART_RE, + fatal_slide_errors, + is_schema_verdict, + ) + + schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"] + inherited = self._original_slide_defects(schema) + problems: list[str] = [] + broken: list[str] = [] + + for xml_file in self.xml_files: + relative = xml_file.relative_to(self.unpacked_dir).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema + ) + if ok is None or not errors: + continue + + unreadable = [f"{relative}: {e}" for e in errors if not is_schema_verdict(e)] + if unreadable: + broken.extend(unreadable) + continue + if ok: + continue + + for message in fatal_slide_errors(set(errors)): + if message in inherited: + continue + problems.append(f"{relative}: {message}") + + if broken: + print(f"FAILED - Could not check {len(broken)} slide part(s):") + for message in sorted(broken): + print(f" {message[:240]}") + + if problems: + print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:") + for message in sorted(problems): + print(f" {message[:240]}") + + if broken or problems: + return False + + if self.verbose: + print("PASSED - Slide XML has none of the defects PowerPoint refuses") + return True + + def _get_schema_path(self, xml_file): + if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"): + return None + return super()._get_schema_path(xml_file) + + def _preprocess_for_schema(self, xml_doc, relative_path): + if relative_path.as_posix() != "ppt/presentation.xml": + return xml_doc + + root = xml_doc.getroot() + ns = f"{{{self.PRESENTATIONML_NAMESPACE}}}" + notes = root.find(f"{ns}notesMasterIdLst") + slides = root.find(f"{ns}sldIdLst") + if notes is None or slides is None: + return xml_doc + + children = list(root) + if children.index(notes) < children.index(slides): + return xml_doc + + root.remove(notes) + root.insert(list(root).index(slides), notes) + return xml_doc + def validate_uuid_ids(self): import lxml.etree @@ -229,17 +395,17 @@ def validate_notes_slide_references(self): ): rel_type = rel.get("Type", "") if "notesSlide" in rel_type: - target = rel.get("Target", "") - if target: - normalized_target = target.replace("../", "") - + part = opc_target( + rel.get("Target", ""), + rels_source_part(rels_file, self.unpacked_dir), + rel.get("TargetMode", ""), + ) + if part: slide_name = rels_file.stem.replace( ".xml", "" ) - if normalized_target not in notes_slide_references: - notes_slide_references[normalized_target] = [] - notes_slide_references[normalized_target].append( + notes_slide_references.setdefault(part, []).append( (slide_name, rels_file) ) diff --git a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/redlining.py b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/redlining.py index 71c81b6bf4..18d0c68be9 100644 --- a/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/redlining.py +++ b/src/crates/assembly/core/builtin_skills/docx/scripts/office/validators/redlining.py @@ -1,5 +1,14 @@ """ Validator for tracked changes in Word documents. + +Detects untracked edits in word/document.xml: text that differs from the +original without a / wrapper recording it. The tracked changes +that are new relative to the original are undone, and the result is compared +against the original; whatever text still differs was edited without being +tracked. + +Only the document body is compared. Headers, footers, footnotes and endnotes +are separate parts and are not checked. """ import subprocess @@ -7,14 +16,18 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import rendered_text, safe_extract + class RedliningValidator: - def __init__(self, unpacked_dir, original_docx, verbose=False, author="Claude"): + def __init__(self, unpacked_dir, original_docx, verbose=False): self.unpacked_dir = Path(unpacked_dir) self.original_docx = Path(original_docx) self.verbose = verbose - self.author = author self.namespaces = { "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main" } @@ -28,40 +41,12 @@ def validate(self): print(f"FAILED - Modified document.xml not found at {modified_file}") return False - try: - import xml.etree.ElementTree as ET - - tree = ET.parse(modified_file) - root = tree.getroot() - - del_elements = root.findall(".//w:del", self.namespaces) - ins_elements = root.findall(".//w:ins", self.namespaces) - - author_del_elements = [ - elem - for elem in del_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - author_ins_elements = [ - elem - for elem in ins_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - - if not author_del_elements and not author_ins_elements: - if self.verbose: - print(f"PASSED - No tracked changes by {self.author} found.") - return True - - except Exception: - pass - with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) try: with zipfile.ZipFile(self.original_docx, "r") as zip_ref: - zip_ref.extractall(temp_path) + safe_extract(zip_ref, temp_path) except Exception as e: print(f"FAILED - Error unpacking original docx: {e}") return False @@ -74,18 +59,16 @@ def validate(self): return False try: - import xml.etree.ElementTree as ET - modified_tree = ET.parse(modified_file) modified_root = modified_tree.getroot() original_tree = ET.parse(original_file) original_root = original_tree.getroot() - except ET.ParseError as e: + except (ET.ParseError, DefusedXmlException) as e: print(f"FAILED - Error parsing XML files: {e}") return False - self._remove_author_tracked_changes(original_root) - self._remove_author_tracked_changes(modified_root) + new_changes = self._new_tracked_changes(original_root, modified_root) + self._remove_tracked_changes(modified_root, new_changes) modified_text = self._extract_text_content(modified_root) original_text = self._extract_text_content(original_root) @@ -98,20 +81,91 @@ def validate(self): return False if self.verbose: - print(f"PASSED - All changes by {self.author} are properly tracked") + print( + f"PASSED - All {len(new_changes)} change(s) against the original " + "are properly tracked" + ) return True + def _tracked_change_elements(self, root): + ins_tag = f"{{{self.namespaces['w']}}}ins" + del_tag = f"{{{self.namespaces['w']}}}del" + return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)] + + def _rendered_text(self, elem): + preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" + return rendered_text(elem.text or "", preserve) + + def _text_elements(self, elem): + w = self.namespaces["w"] + return [ + node + for node in elem.iter() + if node.tag in (f"{{{w}}}t", f"{{{w}}}delText") + ] + + def _tracked_change_key(self, elem): + w = self.namespaces["w"] + text = "".join(self._rendered_text(node) for node in self._text_elements(elem)) + return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text) + + def _new_tracked_changes(self, original_root, modified_root): + original = self._tracked_change_elements(original_root) + modified = self._tracked_change_elements(modified_root) + + pool = {} + for elem in original: + pool.setdefault(self._tracked_change_key(elem), []).append(elem) + + matched, leftover = set(), [] + for elem in modified: + bucket = pool.get(self._tracked_change_key(elem)) + if bucket: + matched.add(bucket.pop()) + else: + leftover.append(elem) + + def group(elem): + return self._tracked_change_key(elem)[:3] + + def text_of(elems): + return "".join(self._tracked_change_key(e)[3] for e in elems) + + unmatched_original = {} + for elem in original: + if elem not in matched: + unmatched_original.setdefault(group(elem), []).append(elem) + + by_group = {} + for elem in leftover: + by_group.setdefault(group(elem), []).append(elem) + + new = set() + for key, elems in by_group.items(): + rebuilt = text_of(elems) + if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])): + continue + new.update(elems) + return new + def _generate_detailed_diff(self, original_text, modified_text): error_parts = [ - f"FAILED - Document text doesn't match after removing {self.author}'s tracked changes", + "FAILED - Document text doesn't match after removing the tracked changes", "", "Likely causes:", " 1. Modified text inside another author's or tags", " 2. Made edits without proper tracked changes", " 3. Didn't nest inside when deleting another's insertion", + " 4. Rewrote another author's / and changed its text on", + " the way. A tracked change from the original is recognised by its", + " author, date and text; anything that doesn't reproduce one exactly", + " reads as new, and the text it carried is reported missing.", "", "For pre-redlined documents, use correct patterns:", " - To reject another's INSERTION: Nest inside their ", + " - To reject PART of one: nest around only the runs you reject.", + " Their may be split around it, so long as the pieces keep", + " their author and date and still spell out the same text.", " - To restore another's DELETION: Add new AFTER their ", "", ] @@ -195,15 +249,14 @@ def _get_git_word_diff(self, original_text, modified_text): return None - def _remove_author_tracked_changes(self, root): + def _remove_tracked_changes(self, root, targets): ins_tag = f"{{{self.namespaces['w']}}}ins" del_tag = f"{{{self.namespaces['w']}}}del" - author_attr = f"{{{self.namespaces['w']}}}author" for parent in root.iter(): to_remove = [] for child in parent: - if child.tag == ins_tag and child.get(author_attr) == self.author: + if child.tag == ins_tag and child in targets: to_remove.append(child) for elem in to_remove: parent.remove(elem) @@ -214,7 +267,7 @@ def _remove_author_tracked_changes(self, root): for parent in root.iter(): to_process = [] for child in parent: - if child.tag == del_tag and child.get(author_attr) == self.author: + if child.tag == del_tag and child in targets: to_process.append((child, list(parent).index(child))) for del_elem, del_index in reversed(to_process): @@ -234,8 +287,7 @@ def _extract_text_content(self, root): for p_elem in root.findall(f".//{p_tag}"): text_parts = [] for t_elem in p_elem.findall(f".//{t_tag}"): - if t_elem.text: - text_parts.append(t_elem.text) + text_parts.append(self._rendered_text(t_elem)) paragraph_text = "".join(text_parts) if paragraph_text: paragraphs.append(paragraph_text) diff --git a/src/crates/assembly/core/builtin_skills/gstack-autoplan/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-autoplan/SKILL.md index 49fa6f4ca3..34463c2424 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-autoplan/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-autoplan/SKILL.md @@ -180,12 +180,12 @@ Then prepend a one-line HTML comment to the plan file: on top of) or if an AI agent is the primary user (OpenClaw actions, BitFun skills, MCP servers). -### Step 3: Load skill files from disk +### Step 3: Load review skills -Read each file using the Read tool: -- `the bundled plan-ceo-review skill via the Skill tool` -- `the bundled plan-design-review skill via the Skill tool` (only if UI scope detected) -- `the bundled plan-eng-review skill via the Skill tool` +Load each stable key with the Skill tool: +- `user::bitfun-system::gstack-plan-ceo-review` +- `user::bitfun-system::gstack-plan-design-review` (only if UI scope detected) +- `user::bitfun-system::gstack-plan-eng-review` - `the relevant built-in developer-experience review methodology, if present` (only if DX scope detected) **Section skip list — when following a loaded skill file, SKIP these sections @@ -212,7 +212,7 @@ Loaded review skills from disk. Starting full review pipeline with auto-decision ## Phase 1: CEO Review (Strategy & Scope) -Follow plan-ceo-review/SKILL.md — all sections, full depth. +Load `user::bitfun-system::gstack-plan-ceo-review` with the Skill tool and follow all sections at full depth. Override: every AskUserQuestion → auto-decide using the 6 principles. **Override rules:** @@ -331,7 +331,7 @@ and the premise gate has been passed. ## Phase 2: Design Review (conditional — skip if no UI scope) -Follow plan-design-review/SKILL.md — all 7 dimensions, full depth. +Load `user::bitfun-system::gstack-plan-design-review` with the Skill tool and follow all 7 dimensions at full depth. Override: every AskUserQuestion → auto-decide using the 6 principles. **Override rules:** @@ -409,7 +409,7 @@ Do NOT begin Phase 3 until all Phase 2 outputs (if run) are written to the plan ## Phase 3: Eng Review + Dual Voices -Follow plan-eng-review/SKILL.md — all sections, full depth. +Load `user::bitfun-system::gstack-plan-eng-review` with the Skill tool and follow all sections at full depth. Override: every AskUserQuestion → auto-decide using the 6 principles. **Override rules:** @@ -514,7 +514,7 @@ Missing voice = N/A (not CONFIRMED). Single critical finding from one voice = fl ## Phase 3.5: DX Review (conditional — skip if no developer-facing scope) -Follow plan-devex-review/SKILL.md — all 8 DX dimensions, full depth. +Use the self-contained checklist below — all 8 DX dimensions, full depth. Override: every AskUserQuestion → auto-decide using the 6 principles. **Skip condition:** If DX scope was NOT detected in Phase 0, skip this phase entirely. diff --git a/src/crates/assembly/core/builtin_skills/gstack-design-consultation/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-design-consultation/SKILL.md index 983b796343..80fd63a21b 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-design-consultation/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-design-consultation/SKILL.md @@ -59,7 +59,9 @@ If office-hours output exists, read it — the product context is pre-filled. If the codebase is empty and purpose is unclear, say: *"I don't have a clear picture of what you're building yet. Want to explore first with `/office-hours`? Once we know the product direction, we can set up the design system."* -**Visual research tooling:** Use BitFun built-in browser/computer-use capability for screenshots and live-page inspection. Do not install, build, or call any external browse binary. If browser tooling is unavailable, continue with code inspection, WebSearch when allowed, and static visual analysis. +**Visual research tooling:** Use agent-browser for screenshots and live-page inspection. If it is unavailable, continue with code inspection, WebSearch when allowed, and static visual analysis. + +Once per skill invocation, before the first browser command, run `agent-browser --version` (require 0.32.3 or newer) and load `agent-browser skills get core`; reuse that guidance for the rest of this invocation. If either step fails, do not install automatically; use the static fallback above or ask whether the user wants the pinned install from the bundled agent-browser skill. If browse is not available, that's fine — visual research is optional. The skill works without it using WebSearch and your built-in design knowledge. @@ -67,7 +69,7 @@ If browse is not available, that's fine — visual research is optional. The ski ## DESIGN SETUP -Use BitFun built-in image/design and browser/computer-use capabilities. Do not install, build, or call external `design` or `browse` binaries. Generate mockups, comparison boards, screenshots, and visual QA artifacts through BitFun tools; if a visual generation capability is not available in the current session, fall back to HTML wireframes and code-level design review. +Use BitFun's built-in image/design capabilities and agent-browser for live-page work. Do not install or build external `design` binaries. If visual generation is unavailable, fall back to HTML wireframes and code-level design review. **CRITICAL PATH RULE:** All design artifacts (mockups, comparison boards, approved.json) MUST be saved to `$HOME/.bitfun/team/projects/$SLUG/designs/`, NEVER to `.context/`, @@ -111,12 +113,12 @@ Use WebSearch to find 5-10 products in their space. Search for: **Step 2: Visual research via browse (if available)** -If the BitFun browser/computer-use tooling is available (`BitFun browser/computer-use` is set), visit the top 3-5 sites in the space and capture visual evidence: +If the agent-browser CLI is available, visit the top 3-5 sites in the space and capture visual evidence: ```bash -BitFun browser/computer-use goto "https://example-site.com" -BitFun browser/computer-use screenshot "/tmp/design-research-site-name.png" -BitFun browser/computer-use snapshot +agent-browser open "https://example-site.com" +agent-browser screenshot "/tmp/design-research-site-name.png" +agent-browser snapshot ``` For each site, analyze: fonts actually used, color palette, layout approach, spacing density, aesthetic direction. The screenshot gives you the feel; the snapshot gives you structural data. @@ -584,7 +586,7 @@ List all decisions. Flag any that used agent defaults without explicit user conf After shipping DESIGN.md, if the session produced screen-level mockups or page layouts (not just system-level tokens), suggest: -"Want to see this design system as working Pretext-native HTML? Run /design-html." +"Want to turn this design system into working HTML with an available prototyping capability?" --- diff --git a/src/crates/assembly/core/builtin_skills/gstack-design-review/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-design-review/SKILL.md index 4b7328c1d2..6331bcbbdc 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-design-review/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-design-review/SKILL.md @@ -19,7 +19,7 @@ You are a senior product designer AND a frontend engineer. Review live sites wit When this skill is invoked by BitFun Team Mode, this skill supplies the live design-audit methodology. Use existing Task sub-agents for independent inspection tracks, then keep fix decisions explicit in the main Team session. - Do not assume a Designer sub-agent exists. Choose only from the Task tool's available agents. -- Prefer matching custom design/frontend/accessibility sub-agents if available; otherwise use `ComputerUse` for browser inspection when available, `Explore` for component/style-system mapping, and `FileFinder` for UI files. +- Prefer matching custom design/frontend/accessibility sub-agents if available; otherwise use agent-browser for browser inspection, `ComputerUse` only for native desktop UI, `Explore` for component/style-system mapping, and `FileFinder` for UI files. - Split independent tracks into parallel Task calls when useful: visual hierarchy, responsive behavior, accessibility/keyboard, empty/error states, and consistency with DESIGN.md. - Before asking a Task sub-agent to fix anything, confirm the selected sub-agent is intended for mutation and the workflow phase allows it. Otherwise request report-only output. - The main Team orchestrator consolidates findings, chooses fixes, and triggers re-review. @@ -39,8 +39,9 @@ When this skill is invoked by BitFun Team Mode, this skill supplies the live des **If no URL is given and you're on main/master:** Ask the user for a URL. -**Browser session detection:** Use BitFun browser/computer-use state to detect whether an existing user browser session is available. -If `CDP_MODE=true`: skip cookie import steps — the real browser already has cookies and auth sessions. Skip headless detection workarounds. +**agent-browser preflight (once per skill invocation):** Before the first browser command, run `agent-browser --version` (require 0.32.3 or newer) and load `agent-browser skills get core`. Reuse that guidance for the rest of this invocation. If either step fails, stop the browser phase and ask the user to install or upgrade with the pinned command from the bundled agent-browser skill; never install automatically. + +**Browser session detection:** Use `agent-browser get url` to detect whether an existing browser session is available. Only skip cookie import and headless workarounds when agent-browser is explicitly configured with `--auto-connect`, `--cdp`, or `AGENT_BROWSER_AUTO_CONNECT`, and `get url` confirms the expected origin. **Check for DESIGN.md:** @@ -64,7 +65,7 @@ RECOMMENDATION: Choose A because uncommitted work should be preserved as a commi After the user chooses, execute their choice (commit or stash), then continue with setup. -**Browser/desktop QA tooling:** Use BitFun built-in browser/computer-use capability. Do not install, build, or call any external browse binary. Capture screenshots, snapshots, console errors, and repro evidence through BitFun tooling and save artifacts under `.bitfun/team/qa-reports/`. +**Browser/desktop QA tooling:** Use agent-browser for browser QA and BitFun ComputerUse only for native desktop surfaces it cannot reach. Save QA artifacts under `.bitfun/team/qa-reports/`. **Check test framework (bootstrap if needed):** @@ -226,7 +227,7 @@ Only commit if there are changes. Stage all bootstrap files (config, test direct ## DESIGN SETUP -Use BitFun built-in image/design and browser/computer-use capabilities. Do not install, build, or call external `design` or `browse` binaries. Generate mockups, comparison boards, screenshots, and visual QA artifacts through BitFun tools; if a visual generation capability is not available in the current session, fall back to HTML wireframes and code-level design review. +Use BitFun's built-in image/design capabilities and agent-browser for live-page work. Do not install or build external `design` binaries. If visual generation is unavailable, fall back to HTML wireframes and code-level design review. **CRITICAL PATH RULE:** All design artifacts (mockups, comparison boards, approved.json) MUST be saved to `$HOME/.bitfun/team/projects/$SLUG/designs/`, NEVER to `.context/`, @@ -282,7 +283,7 @@ Run full audit, then load previous `design-baseline.json`. Compare: per-category The most uniquely designer-like output. Form a gut reaction before analyzing anything. 1. Navigate to the target URL -2. Take a full-page desktop screenshot: `BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/first-impression.png"` +2. Take a full-page desktop screenshot: `agent-browser screenshot "$REPORT_DIR/screenshots/first-impression.png"` 3. Write the **First Impression** using this structured critique format: - "The site communicates **[what]**." (what it says at a glance — competence? playfulness? confusion?) - "I notice **[observation]**." (what stands out, positive or negative — be specific) @@ -299,19 +300,19 @@ Extract the actual design system the site uses (not what a DESIGN.md says, but w ```bash # Fonts in use (capped at 500 elements to avoid timeout) -BitFun browser/computer-use js "JSON.stringify([...new Set([...document.querySelectorAll('*')].slice(0,500).map(e => getComputedStyle(e).fontFamily))])" +agent-browser eval "JSON.stringify([...new Set([...document.querySelectorAll('*')].slice(0,500).map(e => getComputedStyle(e).fontFamily))])" # Color palette in use -BitFun browser/computer-use js "JSON.stringify([...new Set([...document.querySelectorAll('*')].slice(0,500).flatMap(e => [getComputedStyle(e).color, getComputedStyle(e).backgroundColor]).filter(c => c !== 'rgba(0, 0, 0, 0)'))])" +agent-browser eval "JSON.stringify([...new Set([...document.querySelectorAll('*')].slice(0,500).flatMap(e => [getComputedStyle(e).color, getComputedStyle(e).backgroundColor]).filter(c => c !== 'rgba(0, 0, 0, 0)'))])" # Heading hierarchy -BitFun browser/computer-use js "JSON.stringify([...document.querySelectorAll('h1,h2,h3,h4,h5,h6')].map(h => ({tag:h.tagName, text:h.textContent.trim().slice(0,50), size:getComputedStyle(h).fontSize, weight:getComputedStyle(h).fontWeight})))" +agent-browser eval "JSON.stringify([...document.querySelectorAll('h1,h2,h3,h4,h5,h6')].map(h => ({tag:h.tagName, text:h.textContent.trim().slice(0,50), size:getComputedStyle(h).fontSize, weight:getComputedStyle(h).fontWeight})))" # Touch target audit (find undersized interactive elements) -BitFun browser/computer-use js "JSON.stringify([...document.querySelectorAll('a,button,input,[role=button]')].filter(e => {const r=e.getBoundingClientRect(); return r.width>0 && (r.width<44||r.height<44)}).map(e => ({tag:e.tagName, text:(e.textContent||'').trim().slice(0,30), w:Math.round(e.getBoundingClientRect().width), h:Math.round(e.getBoundingClientRect().height)})).slice(0,20))" +agent-browser eval "JSON.stringify([...document.querySelectorAll('a,button,input,[role=button]')].filter(e => {const r=e.getBoundingClientRect(); return r.width>0 && (r.width<44||r.height<44)}).map(e => ({tag:e.tagName, text:(e.textContent||'').trim().slice(0,30), w:Math.round(e.getBoundingClientRect().width), h:Math.round(e.getBoundingClientRect().height)})).slice(0,20))" # Performance baseline -BitFun browser/computer-use perf +agent-browser vitals ``` Structure findings as an **Inferred Design System**: @@ -329,20 +330,23 @@ After extraction, offer: *"Want me to save this as your DESIGN.md? I can lock in For each page in scope: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/{page}-annotated.png" -BitFun browser/computer-use responsive "$REPORT_DIR/screenshots/{page}" -BitFun browser/computer-use console --errors -BitFun browser/computer-use perf +agent-browser open +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/{page}-annotated.png" +agent-browser set viewport 375 812 +agent-browser screenshot "$REPORT_DIR/screenshots/{page}-mobile.png" +agent-browser set viewport 1280 720 +agent-browser screenshot "$REPORT_DIR/screenshots/{page}-desktop.png" +agent-browser errors +agent-browser vitals ``` ### Auth Detection After the first navigation, check if the URL changed to a login-like path: ```bash -BitFun browser/computer-use url +agent-browser get url ``` -If URL contains `/login`, `/signin`, `/auth`, or `/sso`: the site requires authentication. AskUserQuestion: "This site requires authentication. Want to import cookies from your browser? Run `/setup-browser-cookies` first if needed." +If URL contains `/login`, `/signin`, `/auth`, or `/sso`: ask the user for an existing agent-browser auth state or cookie-file path, or ask them to log in interactively. ### Design Audit Checklist (10 categories, ~80 items) @@ -367,7 +371,7 @@ Apply these at each page. Each finding gets an impact rating (high/medium/polish - Weight contrast: >=2 weights used for hierarchy - No blacklisted fonts (Papyrus, Comic Sans, Lobster, Impact, Jokerman) - If primary font is Inter/Roboto/Open Sans/Poppins → flag as potentially generic -- `text-wrap: balance` or `text-pretty` on headings (check via `BitFun browser/computer-use css text-wrap`) +- `text-wrap: balance` or `text-pretty` on headings (inspect `text-wrap` via `agent-browser get styles `) - Curly quotes used, not straight quotes - Ellipsis character (`…`) not three dots (`...`) - `font-variant-numeric: tabular-nums` on number columns @@ -427,7 +431,7 @@ Apply these at each page. Each finding gets an impact rating (high/medium/polish - Easing: ease-out for entering, ease-in for exiting, ease-in-out for moving - Duration: 50-700ms range (nothing slower unless page transition) - Purpose: every animation communicates something (state change, attention, spatial relationship) -- `prefers-reduced-motion` respected (check: `BitFun browser/computer-use js "matchMedia('(prefers-reduced-motion: reduce)').matches"`) +- `prefers-reduced-motion` respected (check: `agent-browser eval "matchMedia('(prefers-reduced-motion: reduce)').matches"`) - No `transition: all` — properties listed explicitly - Only `transform` and `opacity` animated (not layout properties like width, height, top, left) @@ -471,9 +475,9 @@ The test: would a human designer at a respected studio ever ship this? Walk 2-3 key user flows and evaluate the *feel*, not just the function: ```bash -BitFun browser/computer-use snapshot -i -BitFun browser/computer-use click @e3 # perform action -BitFun browser/computer-use snapshot -D # diff to see what changed +agent-browser snapshot -i +agent-browser click @e3 # perform action +agent-browser diff snapshot # compare with the prior snapshot ``` Evaluate: @@ -579,11 +583,11 @@ Tie everything to user goals and product objectives. Always suggest specific imp 4. **Never read source code.** Evaluate the rendered site, not the implementation. (Exception: offer to write DESIGN.md from extracted observations.) 5. **AI Slop detection is your superpower.** Most developers can't evaluate whether their site looks AI-generated. You can. Be direct about it. 6. **Quick wins matter.** Always include a "Quick Wins" section — the 3-5 highest-impact fixes that take <30 minutes each. -7. **Use `snapshot -C` for tricky UIs.** Finds clickable divs that the accessibility tree misses. +7. **Use `screenshot --annotate` for tricky UIs.** It labels interactive targets that need visual inspection. 8. **Responsive is design, not just "not broken."** A stacked desktop layout on mobile is not responsive design — it's lazy. Evaluate whether the mobile layout makes *design* sense. 9. **Document incrementally.** Write each finding to the report as you find it. Don't batch. 10. **Depth over breadth.** 5-10 well-documented findings with screenshots and specific suggestions > 20 vague observations. -11. **Show screenshots to the user.** After every `BitFun browser/computer-use screenshot`, `BitFun browser/computer-use snapshot -a -o`, or `BitFun browser/computer-use responsive` command, use the Read tool on the output file(s) so the user can see them inline. For `responsive` (3 files), Read all three. This is critical — without it, screenshots are invisible to the user. +11. **Show screenshots to the user.** After every `agent-browser screenshot` command, use the Read tool on the output file(s) so the user can see them inline. Read every viewport capture. This is critical — without it, screenshots are invisible to the user. ### Design Hard Rules @@ -822,10 +826,10 @@ git commit -m "style(design): FINDING-NNN — short description" Navigate back to the affected page and verify the fix: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/finding-NNN-after.png" -BitFun browser/computer-use console --errors -BitFun browser/computer-use snapshot -D +agent-browser open +agent-browser screenshot "$REPORT_DIR/screenshots/finding-NNN-after.png" +agent-browser errors +agent-browser diff snapshot ``` Take **before/after screenshot pair** for every fix. diff --git a/src/crates/assembly/core/builtin_skills/gstack-office-hours/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-office-hours/SKILL.md index 621b349665..e373fb1068 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-office-hours/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-office-hours/SKILL.md @@ -586,20 +586,24 @@ Generate a single-page HTML file with these constraints: matches the actual use case) - Add HTML comments explaining design decisions -Write to a temp file: +Create a cross-platform temp file: ```bash -SKETCH_FILE="/tmp/gstack-sketch-$(date +%s).html" +SKETCH_FILE=$(python -c "import os,tempfile; fd,path=tempfile.mkstemp(prefix='gstack-sketch-',suffix='.html'); os.close(fd); print(path)") ``` **Step 3: Render and capture** +Once per skill invocation, before the first browser command used for rendering, run `agent-browser --version` (require 0.32.3 or newer) and load `agent-browser skills get core`; reuse that guidance for the rest of this invocation. If either step fails, do not install automatically; keep the HTML artifact and use the fallback below. + ```bash -BitFun browser/computer-use goto "file://$SKETCH_FILE" -BitFun browser/computer-use screenshot /tmp/gstack-sketch.png +SKETCH_URI=$(python -c "import sys; from pathlib import Path; print(Path(sys.argv[1]).resolve().as_uri())" "$SKETCH_FILE") +SKETCH_SCREENSHOT="${SKETCH_FILE%.html}.png" +agent-browser --allow-file-access open "$SKETCH_URI" +agent-browser screenshot "$SKETCH_SCREENSHOT" ``` -If `BitFun browser/computer-use` is not available (BitFun browser/computer-use tooling not set up), skip the render step. Tell the -user: "Use BitFun browser/computer-use tooling for the visual sketch when it is available. If unavailable, skip the render step and keep the HTML sketch artifact." +If `agent-browser` is not available (agent-browser CLI not set up), skip the render step. Tell the +user: "Use the agent-browser CLI for the visual sketch when it is available. If unavailable, skip the render step and keep the HTML sketch artifact." **Step 4: Present and iterate** @@ -611,7 +615,7 @@ If they approve or say "good enough," proceed. **Step 5: Include in design doc** Reference the wireframe screenshot in the design doc's "Recommended Approach" section. -The screenshot file at `/tmp/gstack-sketch.png` can be referenced by downstream skills +The generated `$SKETCH_SCREENSHOT` path can be referenced by downstream skills (`/plan-design-review`, `/design-review`) to see what was originally envisioned. **Step 6: Outside design voices** (optional) diff --git a/src/crates/assembly/core/builtin_skills/gstack-plan-ceo-review/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-plan-ceo-review/SKILL.md index 81140a42e1..606055754a 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-plan-ceo-review/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-plan-ceo-review/SKILL.md @@ -1083,8 +1083,6 @@ Parse each JSONL entry. Each skill logs different fields: → Findings: "{issues_found} issues, {critical_gaps} critical gaps" - **plan-design-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`unresolved\`, \`decisions_made\`, \`commit\` → Findings: "score: {initial_score}/10 → {overall_score}/10, {decisions_made} decisions" -- **plan-devex-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`product_type\`, \`tthw_current\`, \`tthw_target\`, \`mode\`, \`persona\`, \`competitive_tier\`, \`unresolved\`, \`commit\` - → Findings: "score: {initial_score}/10 → {overall_score}/10, TTHW: {tthw_current} → {tthw_target}" - **devex-review**: \`status\`, \`overall_score\`, \`product_type\`, \`tthw_measured\`, \`dimensions_tested\`, \`dimensions_inferred\`, \`boomerang\`, \`commit\` → Findings: "score: {overall_score}/10, TTHW: {tthw_measured}, {dimensions_tested} tested/{dimensions_inferred} inferred" - **outside-voice-review**: \`status\`, \`gate\`, \`findings\`, \`findings_fixed\` @@ -1105,7 +1103,6 @@ Produce this markdown table: | outside-voice sub-agent Review | \`BitFun Task outside-voice review\` | Independent 2nd opinion | {runs} | {status} | {findings} | | Eng Review | \`/plan-eng-review\` | Architecture & tests (required) | {runs} | {status} | {findings} | | Design Review | \`/plan-design-review\` | UI/UX gaps | {runs} | {status} | {findings} | -| DX Review | \`/plan-devex-review\` | Developer experience gaps | {runs} | {status} | {findings} | \`\`\` Below the table, add these lines (omit any that are empty/not applicable): diff --git a/src/crates/assembly/core/builtin_skills/gstack-plan-design-review/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-plan-design-review/SKILL.md index 225758a3db..95283be665 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-plan-design-review/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-plan-design-review/SKILL.md @@ -180,7 +180,7 @@ planning phase. Generating mockups during planning is the whole point. Allowed commands under this exception: - `mkdir -p $HOME/.bitfun/team/projects/$SLUG/designs/...` - `BitFun image/design capability generate`, `BitFun image/design capability variants`, `BitFun image/design capability compare`, `BitFun image/design capability iterate`, `BitFun image/design capability evolve`, `BitFun image/design capability check` -- `open` (fallback for viewing boards when `BitFun browser/computer-use` is not available) +- `open` (fallback for viewing boards when `agent-browser` is not available) First, set up the output directory. Name it after the screen/feature being designed and today's date: @@ -194,7 +194,7 @@ echo "DESIGN_DIR: $_DESIGN_DIR" Replace `` with a descriptive kebab-case name (e.g., `homepage-variants`, `settings-page`, `onboarding-flow`). **Generate mockups ONE AT A TIME in this skill.** The inline review flow generates -fewer variants and benefits from sequential control. Note: /design-shotgun uses +fewer variants and benefits from sequential control. Note: parallel design exploration uses parallel Agent subagents for variant generation, which works at Tier 2+ (15+ RPM). The sequential constraint here is specific to plan-design-review's inline pattern. @@ -784,8 +784,6 @@ Parse each JSONL entry. Each skill logs different fields: → Findings: "{issues_found} issues, {critical_gaps} critical gaps" - **plan-design-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`unresolved\`, \`decisions_made\`, \`commit\` → Findings: "score: {initial_score}/10 → {overall_score}/10, {decisions_made} decisions" -- **plan-devex-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`product_type\`, \`tthw_current\`, \`tthw_target\`, \`mode\`, \`persona\`, \`competitive_tier\`, \`unresolved\`, \`commit\` - → Findings: "score: {initial_score}/10 → {overall_score}/10, TTHW: {tthw_current} → {tthw_target}" - **devex-review**: \`status\`, \`overall_score\`, \`product_type\`, \`tthw_measured\`, \`dimensions_tested\`, \`dimensions_inferred\`, \`boomerang\`, \`commit\` → Findings: "score: {overall_score}/10, TTHW: {tthw_measured}, {dimensions_tested} tested/{dimensions_inferred} inferred" - **outside-voice-review**: \`status\`, \`gate\`, \`findings\`, \`findings_fixed\` @@ -806,7 +804,6 @@ Produce this markdown table: | outside-voice sub-agent Review | \`BitFun Task outside-voice review\` | Independent 2nd opinion | {runs} | {status} | {findings} | | Eng Review | \`/plan-eng-review\` | Architecture & tests (required) | {runs} | {status} | {findings} | | Design Review | \`/plan-design-review\` | UI/UX gaps | {runs} | {status} | {findings} | -| DX Review | \`/plan-devex-review\` | Developer experience gaps | {runs} | {status} | {findings} | \`\`\` Below the table, add these lines (omit any that are empty/not applicable): @@ -868,17 +865,17 @@ After displaying the Review Readiness Dashboard, recommend the next review(s) ba **If both are needed, recommend eng review first** (required gate). -**Recommend design exploration skills when appropriate** — /design-shotgun and /design-html +**Recommend design exploration capabilities when appropriate** — use capabilities available in the current session produce design artifacts (mockups, HTML previews), not application code. They belong in plan mode alongside reviews. If this design review found visual issues that would benefit -from exploring new directions, recommend /design-shotgun. If approved mockups exist and -need to be turned into working HTML, recommend /design-html. +from exploring new directions, recommend the available design exploration capability. If approved mockups exist and +need to be turned into working HTML, recommend the available HTML prototyping capability. Use AskUserQuestion to present the next step. Include only applicable options: - **A)** Run /plan-eng-review next (required gate) - **B)** Run /plan-ceo-review (only if fundamental product gaps found) -- **C)** Run /design-shotgun — explore visual design variants for issues found -- **D)** Run /design-html — generate Pretext-native HTML from approved mockups +- **C)** Explore visual design variants for issues found using an available capability +- **D)** Generate HTML from approved mockups using an available capability - **E)** Skip — I'll handle next steps manually ## Formatting Rules diff --git a/src/crates/assembly/core/builtin_skills/gstack-plan-eng-review/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-plan-eng-review/SKILL.md index 89e1540da3..98667aaf83 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-plan-eng-review/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-plan-eng-review/SKILL.md @@ -767,8 +767,6 @@ Parse each JSONL entry. Each skill logs different fields: → Findings: "{issues_found} issues, {critical_gaps} critical gaps" - **plan-design-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`unresolved\`, \`decisions_made\`, \`commit\` → Findings: "score: {initial_score}/10 → {overall_score}/10, {decisions_made} decisions" -- **plan-devex-review**: \`status\`, \`initial_score\`, \`overall_score\`, \`product_type\`, \`tthw_current\`, \`tthw_target\`, \`mode\`, \`persona\`, \`competitive_tier\`, \`unresolved\`, \`commit\` - → Findings: "score: {initial_score}/10 → {overall_score}/10, TTHW: {tthw_current} → {tthw_target}" - **devex-review**: \`status\`, \`overall_score\`, \`product_type\`, \`tthw_measured\`, \`dimensions_tested\`, \`dimensions_inferred\`, \`boomerang\`, \`commit\` → Findings: "score: {overall_score}/10, TTHW: {tthw_measured}, {dimensions_tested} tested/{dimensions_inferred} inferred" - **outside-voice-review**: \`status\`, \`gate\`, \`findings\`, \`findings_fixed\` @@ -789,7 +787,6 @@ Produce this markdown table: | outside-voice sub-agent Review | \`BitFun Task outside-voice review\` | Independent 2nd opinion | {runs} | {status} | {findings} | | Eng Review | \`/plan-eng-review\` | Architecture & tests (required) | {runs} | {status} | {findings} | | Design Review | \`/plan-design-review\` | UI/UX gaps | {runs} | {status} | {findings} | -| DX Review | \`/plan-devex-review\` | Developer experience gaps | {runs} | {status} | {findings} | \`\`\` Below the table, add these lines (omit any that are empty/not applicable): diff --git a/src/crates/assembly/core/builtin_skills/gstack-qa-only/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-qa-only/SKILL.md index 07dfd92160..1fe4261425 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-qa-only/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-qa-only/SKILL.md @@ -18,7 +18,7 @@ You are a QA engineer. Test web applications like a real user — click everythi When this skill is invoked by BitFun Team Mode, this skill supplies the report-only QA methodology. Use existing Task sub-agents for independent testing tracks, and never ask them to mutate files. - Do not assume a QA Reporter sub-agent exists. Choose only from the Task tool's available agents. -- Prefer a matching custom QA/browser sub-agent if available; otherwise use `ComputerUse` for browser/desktop testing when available, and `Explore` for diff-aware test-scope mapping. +- Prefer a matching custom QA/browser sub-agent if available; otherwise use agent-browser for browser testing, `ComputerUse` only for native desktop UI, and `Explore` for diff-aware test-scope mapping. - Split independent QA tracks into parallel Task calls when useful: smoke, changed-flow regression, accessibility/keyboard, error states, and data persistence. - Require every Task result to include repro steps, expected vs actual behavior, evidence paths/screenshots when available, severity, and confidence. - The main Team orchestrator consolidates duplicates and decides what blocks Ship. @@ -37,7 +37,9 @@ When this skill is invoked by BitFun Team Mode, this skill supplies the report-o **If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works. -**Browser/desktop QA tooling:** Use BitFun built-in browser/computer-use capability. Do not install, build, or call any external browse binary. Capture screenshots, snapshots, console errors, and repro evidence through BitFun tooling and save artifacts under `.bitfun/team/qa-reports/`. +**Browser/desktop QA tooling:** Use agent-browser for browser QA and BitFun ComputerUse only for native desktop surfaces it cannot reach. Save QA artifacts under `.bitfun/team/qa-reports/`. + +**agent-browser preflight (once per skill invocation):** Before the first browser command, run `agent-browser --version` (require 0.32.3 or newer) and load `agent-browser skills get core`. Reuse that guidance for the rest of this invocation. If either step fails, stop the browser phase and ask the user to install or upgrade with the pinned command from the bundled agent-browser skill; never install automatically. If the user declines, explain that browser QA cannot be completed and stop; do not substitute ComputerUse for web QA. **Create output directories:** @@ -84,16 +86,16 @@ This is the **primary mode** for developers verifying their work. When the user - View/template/component files → which pages render them - Model/service files → which pages use those models (check controllers that reference them) - CSS/style files → which pages include those stylesheets - - API endpoints → test them directly with `BitFun browser/computer-use js "await fetch('/api/...')"` + - API endpoints → test them directly with `agent-browser eval "await fetch('/api/...')"` - Static pages (markdown, HTML) → navigate to them directly **If no obvious pages/routes are identified from the diff:** Do not skip browser testing. The user invoked /qa because they want browser-based verification. Fall back to Quick mode — navigate to the homepage, follow the top 5 navigation targets, check console for errors, and test any interactive elements found. Backend, config, and infrastructure changes affect app behavior — always verify the app still works. 3. **Detect the running app** — check common local dev ports: ```bash - BitFun browser/computer-use goto http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \ - BitFun browser/computer-use goto http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \ - BitFun browser/computer-use goto http://localhost:8080 2>/dev/null && echo "Found app on :8080" + agent-browser open http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \ + agent-browser open http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \ + agent-browser open http://localhost:8080 2>/dev/null && echo "Found app on :8080" ``` If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL. @@ -102,7 +104,7 @@ This is the **primary mode** for developers verifying their work. When the user - Take a screenshot - Check console for errors - If the change was interactive (forms, buttons, flows), test the interaction end-to-end - - Use `snapshot -D` before and after actions to verify the change had the expected effect + - Use `agent-browser diff snapshot` after actions to verify the change had the expected effect 5. **Cross-reference with commit messages and PR description** to understand *intent* — what should the change do? Verify it actually does that. @@ -130,29 +132,33 @@ Run full mode, then load `baseline.json` from a previous run. Diff: which issues ### Phase 1: Initialize -1. Find BitFun browser/computer-use tooling (see Setup above) +1. Find the agent-browser CLI (see Setup above) 2. Create output directories -3. Copy report template from `qa/templates/qa-report-template.md` to output dir +3. Create a new report file in the output directory 4. Start timer for duration tracking ### Phase 2: Authenticate (if needed) -**If the user specified auth credentials:** +**If authentication needs credentials:** Never put a password in command arguments. Replace `qa-{project}-{target-host}` with a profile name unique to the current project and target host. Ask the user to run this in their own interactive terminal and confirm when the profile is saved: + +```bash +agent-browser auth save "qa-{project}-{target-host}" --url --username user@example.com --password-stdin +``` + +After confirmation, run: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i # find the login form -BitFun browser/computer-use fill @e3 "user@example.com" -BitFun browser/computer-use fill @e4 "[REDACTED]" # NEVER include real passwords in report -BitFun browser/computer-use click @e5 # submit -BitFun browser/computer-use snapshot -D # verify login succeeded +agent-browser auth login "qa-{project}-{target-host}" +agent-browser get url +agent-browser snapshot -i # verify the expected signed-in page or account marker ``` -**If the user provided a cookie file:** +**If the user provided a cookie file or Copy-as-cURL export:** ```bash -BitFun browser/computer-use cookie-import cookies.json -BitFun browser/computer-use goto +agent-browser open +agent-browser cookies set --curl cookies.json +agent-browser open ``` **If 2FA/OTP is required:** Ask the user for the code and wait. @@ -164,10 +170,10 @@ BitFun browser/computer-use goto Get a map of the application: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/initial.png" -BitFun browser/computer-use links # map navigation structure -BitFun browser/computer-use console --errors # any errors on landing? +agent-browser open +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/initial.png" +agent-browser snapshot -i -u # map navigation structure +agent-browser errors # any errors on landing? ``` **Detect framework** (note in report metadata): @@ -183,12 +189,12 @@ BitFun browser/computer-use console --errors # any errors on landi Visit pages systematically. At each page: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/page-name.png" -BitFun browser/computer-use console --errors +agent-browser open +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/page-name.png" +agent-browser errors ``` -Then follow the **per-page exploration checklist** (see `qa/references/issue-taxonomy.md`): +Then follow the **per-page exploration checklist** below: 1. **Visual scan** — Look at the annotated screenshot for layout issues 2. **Interactive elements** — Click buttons, links, controls. Do they work? @@ -198,9 +204,9 @@ Then follow the **per-page exploration checklist** (see `qa/references/issue-tax 6. **Console** — Any new JS errors after interactions? 7. **Responsiveness** — Check mobile viewport if relevant: ```bash - BitFun browser/computer-use viewport 375x812 - BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/page-mobile.png" - BitFun browser/computer-use viewport 1280x720 + agent-browser set viewport 375 812 + agent-browser screenshot "$REPORT_DIR/screenshots/page-mobile.png" + agent-browser set viewport 1280 720 ``` **Depth judgment:** Spend more time on core features (homepage, dashboard, checkout, search) and less on secondary pages (about, terms, privacy). @@ -217,14 +223,14 @@ Document each issue **immediately when found** — don't batch them. 1. Take a screenshot before the action 2. Perform the action 3. Take a screenshot showing the result -4. Use `snapshot -D` to show what changed +4. Use `agent-browser diff snapshot` after the action to show what changed 5. Write repro steps referencing screenshots ```bash -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png" -BitFun browser/computer-use click @e5 -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/issue-001-result.png" -BitFun browser/computer-use snapshot -D +agent-browser screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png" +agent-browser click @e5 +agent-browser screenshot "$REPORT_DIR/screenshots/issue-001-result.png" +agent-browser diff snapshot ``` **Static bugs** (typos, layout issues, missing images): @@ -232,10 +238,10 @@ BitFun browser/computer-use snapshot -D 2. Describe what's wrong ```bash -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/issue-002.png" ``` -**Write each issue to the report immediately** using the template format from `qa/templates/qa-report-template.md`. +**Write each issue to the report immediately** using the issue format defined below. ### Phase 6: Wrap Up @@ -341,8 +347,8 @@ Minimum 0 per category. 7. **Test like a user.** Use realistic data. Walk through complete workflows end-to-end. 8. **Depth over breadth.** 5-10 well-documented issues with evidence > 20 vague descriptions. 9. **Never delete output files.** Screenshots and reports accumulate — that's intentional. -10. **Use `snapshot -C` for tricky UIs.** Finds clickable divs that the accessibility tree misses. -11. **Show screenshots to the user.** After every `BitFun browser/computer-use screenshot`, `BitFun browser/computer-use snapshot -a -o`, or `BitFun browser/computer-use responsive` command, use the Read tool on the output file(s) so the user can see them inline. For `responsive` (3 files), Read all three. This is critical — without it, screenshots are invisible to the user. +10. **Use `screenshot --annotate` for tricky UIs.** It labels interactive targets that need visual inspection. +11. **Show screenshots to the user.** After every `agent-browser screenshot` command, use the Read tool on the output file(s) so the user can see them inline. Read every viewport capture. This is critical — without it, screenshots are invisible to the user. 12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing. Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the browser and test. --- diff --git a/src/crates/assembly/core/builtin_skills/gstack-qa/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-qa/SKILL.md index 54eeccd0b7..1618645403 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-qa/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-qa/SKILL.md @@ -21,7 +21,7 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real When this skill is invoked by BitFun Team Mode, this skill supplies the QA methodology. Use existing Task sub-agents for independent testing tracks, then keep triage and fix ownership explicit in the main Team session. - Do not assume a QA Lead sub-agent exists. Choose only from the Task tool's available agents. -- Prefer a matching custom QA/browser sub-agent if available; otherwise use `ComputerUse` for browser/desktop testing when available, and `Explore` for diff-aware test-scope mapping. +- Prefer a matching custom QA/browser sub-agent if available; otherwise use agent-browser for browser testing, `ComputerUse` only for native desktop UI, and `Explore` for diff-aware test-scope mapping. - Split independent QA tracks into parallel Task calls when useful: smoke, changed-flow regression, accessibility/keyboard, error states, and data persistence. - Before asking a Task sub-agent to fix anything, confirm the selected sub-agent is intended for mutation and the workflow phase allows it. Otherwise request report-only output. - The main Team orchestrator owns bug prioritization, regression-test decisions, fixes, and re-review triggers. @@ -46,8 +46,9 @@ When this skill is invoked by BitFun Team Mode, this skill supplies the QA metho **If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works. -**Browser session detection:** Use BitFun browser/computer-use state to detect whether an existing user browser session is available. -If `CDP_MODE=true`: skip cookie import prompts (the real browser already has cookies), skip user-agent overrides (real browser has real user-agent), and skip headless detection workarounds. The user's real auth sessions are already available. +**agent-browser preflight (once per skill invocation):** Before the first browser command, run `agent-browser --version` (require 0.32.3 or newer) and load `agent-browser skills get core`. Reuse that guidance for the rest of this invocation. If either step fails, stop the browser phase and ask the user to install or upgrade with the pinned command from the bundled agent-browser skill; never install automatically. If the user declines, explain that browser QA cannot be completed and stop; do not substitute ComputerUse for web QA. + +**Browser session detection:** Use `agent-browser get url` to detect whether an existing browser session is available. Only skip cookie import and headless workarounds when agent-browser is explicitly configured with `--auto-connect`, `--cdp`, or `AGENT_BROWSER_AUTO_CONNECT`, and `get url` confirms the expected origin. **Check for clean working tree:** @@ -67,7 +68,7 @@ RECOMMENDATION: Choose A because uncommitted work should be preserved as a commi After the user chooses, execute their choice (commit or stash), then continue with setup. -**Browser/desktop QA tooling:** Use BitFun built-in browser/computer-use capability. Do not install, build, or call any external browse binary. Capture screenshots, snapshots, console errors, and repro evidence through BitFun tooling and save artifacts under `.bitfun/team/qa-reports/`. +**Browser/desktop QA tooling:** Use agent-browser for browser QA and BitFun ComputerUse only for native desktop surfaces it cannot reach. Save QA artifacts under `.bitfun/team/qa-reports/`. **Check test framework (bootstrap if needed):** @@ -271,16 +272,16 @@ This is the **primary mode** for developers verifying their work. When the user - View/template/component files → which pages render them - Model/service files → which pages use those models (check controllers that reference them) - CSS/style files → which pages include those stylesheets - - API endpoints → test them directly with `BitFun browser/computer-use js "await fetch('/api/...')"` + - API endpoints → test them directly with `agent-browser eval "await fetch('/api/...')"` - Static pages (markdown, HTML) → navigate to them directly **If no obvious pages/routes are identified from the diff:** Do not skip browser testing. The user invoked /qa because they want browser-based verification. Fall back to Quick mode — navigate to the homepage, follow the top 5 navigation targets, check console for errors, and test any interactive elements found. Backend, config, and infrastructure changes affect app behavior — always verify the app still works. 3. **Detect the running app** — check common local dev ports: ```bash - BitFun browser/computer-use goto http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \ - BitFun browser/computer-use goto http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \ - BitFun browser/computer-use goto http://localhost:8080 2>/dev/null && echo "Found app on :8080" + agent-browser open http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \ + agent-browser open http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \ + agent-browser open http://localhost:8080 2>/dev/null && echo "Found app on :8080" ``` If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL. @@ -289,7 +290,7 @@ This is the **primary mode** for developers verifying their work. When the user - Take a screenshot - Check console for errors - If the change was interactive (forms, buttons, flows), test the interaction end-to-end - - Use `snapshot -D` before and after actions to verify the change had the expected effect + - Use `agent-browser diff snapshot` after actions to verify the change had the expected effect 5. **Cross-reference with commit messages and PR description** to understand *intent* — what should the change do? Verify it actually does that. @@ -317,29 +318,33 @@ Run full mode, then load `baseline.json` from a previous run. Diff: which issues ### Phase 1: Initialize -1. Find BitFun browser/computer-use tooling (see Setup above) +1. Find the agent-browser CLI (see Setup above) 2. Create output directories -3. Copy report template from `qa/templates/qa-report-template.md` to output dir +3. Create a new report file in the output directory 4. Start timer for duration tracking ### Phase 2: Authenticate (if needed) -**If the user specified auth credentials:** +**If authentication needs credentials:** Never put a password in command arguments. Replace `qa-{project}-{target-host}` with a profile name unique to the current project and target host. Ask the user to run this in their own interactive terminal and confirm when the profile is saved: + +```bash +agent-browser auth save "qa-{project}-{target-host}" --url --username user@example.com --password-stdin +``` + +After confirmation, run: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i # find the login form -BitFun browser/computer-use fill @e3 "user@example.com" -BitFun browser/computer-use fill @e4 "[REDACTED]" # NEVER include real passwords in report -BitFun browser/computer-use click @e5 # submit -BitFun browser/computer-use snapshot -D # verify login succeeded +agent-browser auth login "qa-{project}-{target-host}" +agent-browser get url +agent-browser snapshot -i # verify the expected signed-in page or account marker ``` -**If the user provided a cookie file:** +**If the user provided a cookie file or Copy-as-cURL export:** ```bash -BitFun browser/computer-use cookie-import cookies.json -BitFun browser/computer-use goto +agent-browser open +agent-browser cookies set --curl cookies.json +agent-browser open ``` **If 2FA/OTP is required:** Ask the user for the code and wait. @@ -351,10 +356,10 @@ BitFun browser/computer-use goto Get a map of the application: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/initial.png" -BitFun browser/computer-use links # map navigation structure -BitFun browser/computer-use console --errors # any errors on landing? +agent-browser open +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/initial.png" +agent-browser snapshot -i -u # map navigation structure +agent-browser errors # any errors on landing? ``` **Detect framework** (note in report metadata): @@ -370,12 +375,12 @@ BitFun browser/computer-use console --errors # any errors on landi Visit pages systematically. At each page: ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/page-name.png" -BitFun browser/computer-use console --errors +agent-browser open +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/page-name.png" +agent-browser errors ``` -Then follow the **per-page exploration checklist** (see `qa/references/issue-taxonomy.md`): +Then follow the **per-page exploration checklist** below: 1. **Visual scan** — Look at the annotated screenshot for layout issues 2. **Interactive elements** — Click buttons, links, controls. Do they work? @@ -385,9 +390,9 @@ Then follow the **per-page exploration checklist** (see `qa/references/issue-tax 6. **Console** — Any new JS errors after interactions? 7. **Responsiveness** — Check mobile viewport if relevant: ```bash - BitFun browser/computer-use viewport 375x812 - BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/page-mobile.png" - BitFun browser/computer-use viewport 1280x720 + agent-browser set viewport 375 812 + agent-browser screenshot "$REPORT_DIR/screenshots/page-mobile.png" + agent-browser set viewport 1280 720 ``` **Depth judgment:** Spend more time on core features (homepage, dashboard, checkout, search) and less on secondary pages (about, terms, privacy). @@ -404,14 +409,14 @@ Document each issue **immediately when found** — don't batch them. 1. Take a screenshot before the action 2. Perform the action 3. Take a screenshot showing the result -4. Use `snapshot -D` to show what changed +4. Use `agent-browser diff snapshot` after the action to show what changed 5. Write repro steps referencing screenshots ```bash -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png" -BitFun browser/computer-use click @e5 -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/issue-001-result.png" -BitFun browser/computer-use snapshot -D +agent-browser screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png" +agent-browser click @e5 +agent-browser screenshot "$REPORT_DIR/screenshots/issue-001-result.png" +agent-browser diff snapshot ``` **Static bugs** (typos, layout issues, missing images): @@ -419,10 +424,10 @@ BitFun browser/computer-use snapshot -D 2. Describe what's wrong ```bash -BitFun browser/computer-use snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" +agent-browser screenshot --annotate "$REPORT_DIR/screenshots/issue-002.png" ``` -**Write each issue to the report immediately** using the template format from `qa/templates/qa-report-template.md`. +**Write each issue to the report immediately** using the issue format defined below. ### Phase 6: Wrap Up @@ -528,8 +533,8 @@ Minimum 0 per category. 7. **Test like a user.** Use realistic data. Walk through complete workflows end-to-end. 8. **Depth over breadth.** 5-10 well-documented issues with evidence > 20 vague descriptions. 9. **Never delete output files.** Screenshots and reports accumulate — that's intentional. -10. **Use `snapshot -C` for tricky UIs.** Finds clickable divs that the accessibility tree misses. -11. **Show screenshots to the user.** After every `BitFun browser/computer-use screenshot`, `BitFun browser/computer-use snapshot -a -o`, or `BitFun browser/computer-use responsive` command, use the Read tool on the output file(s) so the user can see them inline. For `responsive` (3 files), Read all three. This is critical — without it, screenshots are invisible to the user. +10. **Use `screenshot --annotate` for tricky UIs.** It labels interactive targets that need visual inspection. +11. **Show screenshots to the user.** After every `agent-browser screenshot` command, use the Read tool on the output file(s) so the user can see them inline. Read every viewport capture. This is critical — without it, screenshots are invisible to the user. 12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing. Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the browser and test. Record baseline health score at end of Phase 6. @@ -602,13 +607,13 @@ git commit -m "fix(qa): ISSUE-NNN — short description" - Navigate back to the affected page - Take **before/after screenshot pair** - Check console for errors -- Use `snapshot -D` to verify the change had the expected effect +- Use `agent-browser diff snapshot` after the action to verify the change had the expected effect ```bash -BitFun browser/computer-use goto -BitFun browser/computer-use screenshot "$REPORT_DIR/screenshots/issue-NNN-after.png" -BitFun browser/computer-use console --errors -BitFun browser/computer-use snapshot -D +agent-browser open +agent-browser screenshot "$REPORT_DIR/screenshots/issue-NNN-after.png" +agent-browser errors +agent-browser diff snapshot ``` ### 8e. Classify diff --git a/src/crates/assembly/core/builtin_skills/gstack-ship/SKILL.md b/src/crates/assembly/core/builtin_skills/gstack-ship/SKILL.md index 14142c6340..cefc61f00f 100644 --- a/src/crates/assembly/core/builtin_skills/gstack-ship/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/gstack-ship/SKILL.md @@ -1869,11 +1869,10 @@ Print the branch name, remote URL, and instruct the user to create the PR/MR man ## Step 8.5: Auto-invoke /document-release -After the PR is created, automatically sync project documentation. Read the -`document-release/SKILL.md` skill file (adjacent to this skill's directory) and -execute its full workflow: +After the PR is created, automatically sync project documentation. Load +`user::bitfun-system::gstack-document-release` with the Skill tool and execute its full workflow: -1. Read the `/document-release` skill: `cat the bundled document-release skill via the Skill tool` +1. Load `user::bitfun-system::gstack-document-release` with the Skill tool. 2. Follow its instructions — it reads all .md files in the project, cross-references the diff, and updates anything that drifted (README, ARCHITECTURE, CONTRIBUTING, AGENTS.md, TODOS, etc.) diff --git a/src/crates/assembly/core/builtin_skills/pptx/SKILL.md b/src/crates/assembly/core/builtin_skills/pptx/SKILL.md index df5000e17e..4a72b9bc18 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/pptx/SKILL.md @@ -1,52 +1,81 @@ --- name: pptx -description: "Use this skill any time a .pptx file is involved in any way — as input, output, or both. This includes: creating slide decks, pitch decks, or presentations; reading, parsing, or extracting text from any .pptx file (even if the extracted content will be used elsewhere, like in an email or summary); editing, modifying, or updating existing presentations; combining or splitting slide files; working with templates, layouts, speaker notes, or comments. Trigger whenever the user mentions \"deck,\" \"slides,\" \"presentation,\" or references a .pptx filename, regardless of what they plan to do with the content afterward. If a .pptx file needs to be opened, created, or touched, use this skill." +description: "Use this skill any time a .pptx or .potx file is involved in any way — as input, output, or both. This includes: creating slide decks, pitch decks, or presentations; reading, parsing, or extracting text from any .pptx or .potx file (even if the extracted content will be used elsewhere, like in an email or summary); editing, modifying, or updating existing presentations; combining or splitting slide files; working with templates (.potx), layouts, speaker notes, or comments. Trigger whenever the user mentions \"deck,\" \"slides,\" \"presentation,\" or references a .pptx or .potx filename, regardless of what they plan to do with the content afterward. If a .pptx or .potx file needs to be opened, created, or touched, use this skill." license: Proprietary. LICENSE.txt has complete terms --- -# PPTX Skill +# PPTX creation, editing, and analysis -## Quick Reference +A `.pptx` is a ZIP archive of XML files. Choose your approach by task: -| Task | Guide | -|------|-------| -| Read/analyze content | `python -m markitdown presentation.pptx` | -| Edit or create from template | Read [editing.md](editing.md) | -| Create from scratch | Read [pptxgenjs.md](pptxgenjs.md) | +| Task | Approach | +|---|---| +| **Create** a new deck | Write a `pptxgenjs` script — see gotchas below | +| **Edit** an existing deck, or build from a template | `safe_extract` → edit `ppt/slides/slideN.xml` → `rezip` | +| **Read** content | `markitdown deck.pptx` (one block per slide under `` markers); visual grid: `python scripts/thumbnail.py deck.pptx` | ---- +## Scripts -## Reading Content +Paths are relative to this skill's directory. Everything else is plain Python, `node`, or shell. -```bash -# Text extraction -python -m markitdown presentation.pptx +| Script | What it does | +|---|---| +| `scripts/thumbnail.py deck.pptx [prefix]` | Labeled grid of every slide, for picking template layouts. `.pptx` only. Pass `prefix` — it defaults to `thumbnails`, which overwrites the grids of any other deck done in the same directory | +| `scripts/add_slide.py unpacked/ slide2.xml [--after slideN.xml]` | Duplicate a slide (or a `slideLayoutN.xml`) with all the package bookkeeping. Also takes a `.pptx` directly with `-o out.pptx` | +| `scripts/clean.py unpacked/` | Delete slides, media, and rels no longer referenced. Run **after** `` is final | +| `scripts/office/validate.py deck.pptx [--original src.pptx]` | Schema, relationship, content-type, chart and slide checks; each failure names its fix. Pass `--original` for any template-derived deck — it baselines the schema checks against the template, so the template's own XSD errors don't read as yours | +| `scripts/office/soffice.py --headless --convert-to pdf deck.pptx` | LibreOffice wrapper — bare `soffice` hangs in this sandbox | -# Visual overview -python scripts/thumbnail.py presentation.pptx +## Creating with pptxgenjs — gotchas -# Raw XML -python scripts/office/unpack.py presentation.pptx unpacked/ -``` - ---- +`pptxgenjs` is preinstalled — do not run `npm install` first; write the script and `require('pptxgenjs')` directly. Only if that require fails: `npm install pptxgenjs`. The model knows the API; these are the footguns: -## Editing Workflow +- **Set `pres.layout` before adding slides.** The default canvas is `LAYOUT_16x9` = **10" × 5.625"**, not 13.3" wide. Coordinates past the edge are written, not clamped — the shape just isn't on the slide. (`LAYOUT_WIDE` is 13.3" × 7.5".) +- **Hex colors: never `#`, never 8 digits.** `color: "FF0000"`. Both `"#FF0000"` and alpha baked into the hex (`"00000020"`) **corrupt the file**. For translucency: `transparency: 0-100` on fills and images, `opacity: 0.0-1.0` on shadows — each is silently ignored on the other. +- **pptxgenjs mutates option objects in place** (converts values to EMU on first use). Never share one `shadow`/options object across two `add*` calls — build a fresh object each time. +- **Shadow `offset` must be ≥ 0** — a negative offset corrupts the file. To cast a shadow upward, use `angle: 270` with a positive offset. +- **`letterSpacing` is silently ignored** — the real option is `charSpacing`. +- **Lists:** `bullet: true` on each item, never a literal `•` (renders double bullets). Set `breakLine: true` on every array item except the last. Space bulleted paragraphs with `paraSpaceAfter`, not `lineSpacing` (huge gaps). +- **One `new pptxgen()` per output file** — never reuse an instance. +- **`rectRadius` only works on `ROUNDED_RECTANGLE`**, not `RECTANGLE`. +- **Gradient fills aren't supported** — use a gradient image as the background instead. +- **Text boxes have built-in internal padding** — set `margin: 0` whenever text must align with a shape, line, or icon at the same x. +- **Speaker notes go in `slide.addNotes("...")`** (plain text, once per slide), never in a text box on the slide. +- **Keep charts native.** Use `addChart()` for everything PowerPoint can chart (pass an array of `{type, data, options}` for combos). For PowerPoint-native features the library doesn't expose (trendlines, error bars), compute the extra series yourself or post-process the generated OOXML — do not fall back to a rendered image. Only chart types PowerPoint has no native form for (Sankey, network, chord) go in as images. +- **Default charts render bare** — no title, no data labels, dated palette. Set `showTitle` + `title`, `showValue: true` + `dataLabelPosition`, `chartColors: [...]` from your palette, and quiet the frame (`catAxisLabelColor`/`valAxisLabelColor`, `valGridLine: { color, size }`, `catGridLine: { style: "none" }`, `showLegend: false` for a single series). +- **On a stacked bar or column chart, `dataLabelPosition` must be `ctr`, `inEnd`, or `inBase`.** `outEnd` **corrupts the file**. +- **A combo series using `secondaryValAxis`/`secondaryCatAxis` needs both `valAxes` and `catAxes` on the chart options, two entries each.** Without them pptxgenjs writes axis *ids* it never declares, and PowerPoint **discards that chart** and reports the file as corrupt. Supplying only `valAxes` is not enough. +- **After `writeFile()`, run `python scripts/office/validate.py deck.pptx`.** It reports the two chart faults above and the slide-XML defects PowerPoint refuses, and names the fix for each. Fix them in your generator, not by hand-editing the packed XML. +- **Never reorder the children of ``.** pptxgenjs writes `` right after `` and points both masters at one theme part. PowerPoint reads that happily — move the element and the same deck becomes unopenable. +- **Icons:** render `react-icons` to SVG (`ReactDOMServer.renderToStaticMarkup`), rasterize with `sharp` at ≥256px, and insert via `addImage({ data: "image/png;base64," + buf.toString("base64") })` — the `image/png;base64,` prefix is required (`react-icons`, `react`, `react-dom`, and `sharp` are preinstalled — `npm install react-icons react react-dom sharp` only if a require fails). -**Read [editing.md](editing.md) for full details.** +## Editing existing decks and templates -1. Analyze template with `thumbnail.py` -2. Unpack → manipulate slides → edit content → clean → pack +Pick layouts first: `python scripts/thumbnail.py template.pptx template-thumbs` writes a labeled grid of every slide and prints the file(s) it created — `template-thumbs.jpg`, split into `template-thumbs-N.jpg` past 12 slides. **Always pass that second argument, named after the deck.** It defaults to `thumbnails`, so two decks thumbnailed in one directory silently overwrite each other's grids — the first deck's are simply gone (template analysis only — visual QA needs the full-resolution renders from [Converting to Images](#converting-to-images); it only accepts `.pptx`, so copy a `.potx` to a `.pptx` name first). Use it with `markitdown` to map each content section onto a template slide, and vary the layouts — don't put every section on the same title-and-bullets slide. ---- +```bash +python -c "import sys,zipfile; from pathlib import Path; from scripts.office.helpers import safe_extract; zf=zipfile.ZipFile(sys.argv[1]); safe_extract(zf, Path('unpacked')); zf.close()" deck.pptx +python scripts/add_slide.py unpacked/ slide2.xml --after slide2.xml # duplicate a slide (or slideLayoutN.xml); prints the new slide's path +# reorder / delete slides = edit in ppt/presentation.xml +python scripts/clean.py unpacked/ # after deletions: removes orphaned slides, media, rels +# edit slide content in ppt/slides/slideN.xml +python -c "from pathlib import Path; from scripts.office.helpers import rezip; rezip(Path('unpacked'), Path('out.pptx'))" +python scripts/office/validate.py out.pptx --original deck.pptx +``` -## Creating from Scratch +- **Do all structural work — add, delete, reorder — before editing any slide's content.** `add_slide.py` copies a slide file verbatim, so duplicating after you edit clones the edited content; and `clean.py` deletes any slide missing from ``, including one you just wrote. +- **Never copy a slide file by hand** — `add_slide.py` does every registration a new slide needs and reports what it made (`Created ppt/slides/slide17.xml from slide2.xml`). It also works directly on a file: `add_slide.py deck.pptx slide2.xml -o out.pptx` — **pass `-o`, or it rewrites the input deck in place.** A duplicated slide still *references* its source's chart/SmartArt/embedded-object parts rather than cloning them, so editing one slide's chart changes the other's. +- **If you use `python-pptx`**, three things it won't do: duplicate a slide (its only entry point is `add_slide(layout)`), preserve formatting through `text_frame.text = "..."` (that collapses the paragraph to a single unstyled run — assign `run.text` instead), or read the SVG/EMF most template art uses (`add_picture` raises `UnidentifiedImageError`). +- Legacy `.ppt` must be converted first: `python scripts/office/soffice.py --headless --convert-to pptx file.ppt`. `.potx` templates unpack and pack identically — keep the `.potx` extension on the output. +- To reuse a template icon or image, duplicate a slide or layout that already contains it. -**Read [pptxgenjs.md](pptxgenjs.md) for full details.** +When filling in a template: -Use when no template or reference presentation is available. - ---- +- If you script an XML transform, parse with `defusedxml.minidom` — round-tripping OOXML through `xml.etree.ElementTree` rewrites namespace prefixes and corrupts the deck. +- **Template slots ≠ source items.** If the template shows 4 team members and you have 3, delete the 4th member's entire group (image + text boxes), not just its text — then check for orphaned visuals in QA. +- One `` per list item — never concatenate items into a single paragraph. Copy the sibling `` to preserve spacing, and put `b="1"` on the `` of titles, section headers, and inline labels (`Status:`, `Owner:`). +- Let bullets inherit from the layout; only add ``, `` (numbered), or `` to override — never a literal `•` in the text. +- Text with leading or trailing spaces needs `xml:space="preserve"` on its ``. ## Design Ideas @@ -57,7 +86,7 @@ Use when no template or reference presentation is available. - **Pick a bold, content-informed color palette**: The palette should feel designed for THIS topic. If swapping your colors into a completely different presentation would still "work," you haven't made specific enough choices. - **Dominance over equality**: One color should dominate (60-70% visual weight), with 1-2 supporting tones and one sharp accent. Never give all colors equal weight. - **Dark/light contrast**: Dark backgrounds for title + conclusion slides, light for content ("sandwich" structure). Or commit to dark throughout for a premium feel. -- **Commit to a visual motif**: Pick ONE distinctive element and repeat it — rounded image frames, icons in colored circles, thick single-side borders. Carry it across every slide. +- **Commit to a visual motif**: Pick ONE distinctive element and repeat it — rounded image frames, icons in colored circles. Carry it across every slide. **Do not use a color bar or accent stripe as your motif** (see Avoid list). ### Color Palettes @@ -97,18 +126,13 @@ Choose colors that match your topic — don't default to generic blue. Use these ### Typography -**Choose an interesting font pairing** — don't default to Arial. Pick a header font with personality and pair it with a clean body font. +**Font names you write into the .pptx are rendered by the user's PowerPoint, not by this environment.** Your visual QA renders via LibreOffice, which substitutes fonts it doesn't have — and for some fonts the substitute has different widths, so your QA preview can show text overflow (or fit) that the real deck won't have. To keep your QA trustworthy: -| Header Font | Body Font | -|-------------|-----------| -| Georgia | Calibri | -| Arial Black | Arial | -| Calibri | Calibri Light | -| Cambria | Calibri | -| Trebuchet MS | Calibri | -| Impact | Arial | -| Palatino | Garamond | -| Consolas | Calibri | +- **Safe fonts** (render true-to-width in QA *and* ship with Office): **Arial, Calibri, Cambria, Times New Roman, Courier New, Bookman Old Style, Century Schoolbook**. Use these for body text and anything where fit matters. +- **Headers with personality at zero QA risk**: pair a safe-list serif header (Cambria, Bookman Old Style, Century Schoolbook) with a safe-list sans body (Calibri or Arial). You get visual contrast without giving up reliable overflow checks. +- **If the user asks for a font outside the safe list** (e.g. Georgia or Trebuchet MS): use it where the user asked, but size those containers with extra slack (~10%) and don't trust QA text-fit on those elements — the preview of that font is approximate. If the user hasn't specified, prefer safe-list fonts for body text. +- **QA-unreliable fonts** (substitute has different widths — overflow checks can be wrong): Georgia, Trebuchet MS, Impact, Arial Black, Garamond, Consolas, Palatino Linotype. Calibri Light substitution varies by environment; treat as QA-unreliable. Fine for titles/accents with slack; don't trust QA text-fit on these. +- **Never default to Aptos** — Office's post-2023 default has no metric-compatible substitute here *and* is missing from older Office installs, so it's unreliable on both ends. | Element | Size | |---------|------| @@ -135,19 +159,18 @@ Choose colors that match your topic — don't default to generic blue. Use these - **Don't forget text box padding** — when aligning lines or shapes with text edges, set `margin: 0` on the text box or offset the shape to account for padding - **Don't use low-contrast elements** — icons AND text need strong contrast against the background; avoid light text on light backgrounds or dark text on dark backgrounds - **NEVER use accent lines under titles** — these are a hallmark of AI-generated slides; use whitespace or background color instead - ---- +- **NEVER add decorative color bars or accent stripes** — this includes: header/footer bars spanning the slide width, vertical sidebar stripes down one edge of the slide, thin accent stripes along one edge of a card or content block, and "single-side borders" on rectangles. These read as AI-generated filler. If you want to set a card apart, use a subtle background tint, a drop shadow, or an icon — not an edge stripe. +- **Don't default to cream/beige backgrounds** — when no background is specified, use white (`FFFFFF`) or the user's brand palette; avoid warm-neutral defaults like `F5F5DC`, `FAF0E6`, `FAEBD7`, `FFF8E1` +- **Don't ship text that overflows its shape** — if text doesn't fit, reduce font size, split across slides, or enlarge the container; never leave content cut off or spilling past bounds ## QA (Required) -**Assume there are problems. Your job is to find them.** - -Your first render is almost never correct. Approach QA as a bug hunt, not a confirmation step. If you found zero issues on first inspection, you weren't looking hard enough. +Your first render usually has a few real issues — overlaps, overflow, misalignment. Find and fix those, re-render only the slides you changed, and stop. ### Content QA ```bash -python -m markitdown output.pptx +markitdown output.pptx ``` Check for missing content, typos, wrong order. @@ -155,78 +178,61 @@ Check for missing content, typos, wrong order. **When using templates, check for leftover placeholder text:** ```bash -python -m markitdown output.pptx | grep -iE "xxxx|lorem|ipsum|this.*(page|slide).*layout" +markitdown output.pptx | grep -iE "\bx{3,}\b|lorem|ipsum|\bTODO|\[insert|this.*(page|slide).*layout" ``` If grep returns results, fix them before declaring success. -### Visual QA +### File QA (required) -**⚠️ USE SUBAGENTS** — even for 2-3 slides. You've been staring at the code and will see what you expect, not what's there. Subagents have fresh eyes. +```bash +python scripts/office/validate.py output.pptx # built from scratch +python scripts/office/validate.py output.pptx --original src.pptx # built from a template +``` -Convert slides to images (see [Converting to Images](#converting-to-images)), then use this prompt: +**If the deck came from a template, always pass `--original`.** A template may itself +contain parts the XSD rejects, so a bare run can report failures you never caused — and +a genuine regression can hide among them. `--original` baselines +the schema and slide checks against the template, suppressing errors it already had. +The structural checks — relationships, content types, charts — ignore `--original` and +report template-inherited problems either way, so read those on their own merits. -``` -Visually inspect these slides. Assume there are issues — find them. +pptxgenjs emits chart XML PowerPoint refuses to open, and every other tool +accepts: python-pptx opens those decks, LibreOffice renders them, the XSD +passes them. Every failure names its fix. Fix it in the generator and rebuild. -Look for: +### Visual QA + +Convert the slides to images (see [Converting to Images](#converting-to-images)) and inspect every one. After staring at the generating code you tend to see what you expect rather than what rendered, so look at the images fresh (a subagent works well for this if you have one). User-visible defects to look for: + +- **Text overflow or text cut off at a box or slide boundary — check this first.** It is the most common defect and always user-visible. (For a font the previewer renders unreliably per Typography, the preview is approximate: trust the ~10% slack you left, not its apparent fit.) - Overlapping elements (text through shapes, lines through words, stacked elements) -- Text overflow or cut off at edges/box boundaries -- Decorative lines positioned for single-line text but title wrapped to two lines - Source citations or footers colliding with content above - Elements too close (< 0.3" gaps) or cards/sections nearly touching - Uneven gaps (large empty area in one place, cramped in another) - Insufficient margin from slide edges (< 0.5") - Columns or similar elements not aligned consistently - Low-contrast text (e.g., light gray text on cream-colored background) +- Template decoration mispositioned after text replacement — e.g., a title underline positioned for one line, but the replaced title wrapped to two - Low-contrast icons (e.g., dark icons on dark backgrounds without a contrasting circle) - Text boxes too narrow causing excessive wrapping - Leftover placeholder content -For each slide, list issues or areas of concern, even if minor. - -Read and analyze these images: -1. /path/to/slide-01.jpg (Expected: [brief description]) -2. /path/to/slide-02.jpg (Expected: [brief description]) - -Report ALL issues found, including minor ones. -``` - -### Verification Loop - -1. Generate slides → Convert to images → Inspect -2. **List issues found** (if none found, look again more critically) -3. Fix issues -4. **Re-verify affected slides** — one fix often creates another problem -5. Repeat until a full pass reveals no new issues - -**Do not declare success until you've completed at least one fix-and-verify cycle.** - ---- - ## Converting to Images Convert presentations to individual slide images for visual inspection: ```bash python scripts/office/soffice.py --headless --convert-to pdf output.pptx +rm -f slide-*.jpg pdftoppm -jpeg -r 150 output.pdf slide +ls -1 "$PWD"/slide-*.jpg ``` -This creates `slide-01.jpg`, `slide-02.jpg`, etc. - -To re-render specific slides after fixes: +**Pass the absolute paths printed above directly to the view tool.** The `rm` clears stale images from prior runs. `pdftoppm` zero-pads based on page count: `slide-1.jpg` for decks under 10 pages, `slide-01.jpg` for 10-99, `slide-001.jpg` for 100+. -```bash -pdftoppm -jpeg -r 150 -f N -l N output.pdf slide-fixed -``` - ---- +**After fixes, rerun all four commands above** — the PDF must be regenerated from the edited `.pptx` before `pdftoppm` can reflect your changes. ## Dependencies -- `pip install "markitdown[pptx]"` - text extraction -- `pip install Pillow` - thumbnail grids -- `npm install -g pptxgenjs` - creating from scratch -- LibreOffice (`soffice`) - PDF conversion (auto-configured for sandboxed environments via `scripts/office/soffice.py`) -- Poppler (`pdftoppm`) - PDF to images +`pptxgenjs` (npm, preinstalled — install only if `require('pptxgenjs')` fails) · `markitdown[pptx]`, `Pillow`, `defusedxml`, `lxml` (pip — text dump, thumbnail, clean, validate) · LibreOffice (`soffice`, auto-configured for sandboxed environments via `scripts/office/soffice.py`) · `pdftoppm` (Poppler) diff --git a/src/crates/assembly/core/builtin_skills/pptx/editing.md b/src/crates/assembly/core/builtin_skills/pptx/editing.md deleted file mode 100644 index f873e8a04a..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/editing.md +++ /dev/null @@ -1,205 +0,0 @@ -# Editing Presentations - -## Template-Based Workflow - -When using an existing presentation as a template: - -1. **Analyze existing slides**: - ```bash - python scripts/thumbnail.py template.pptx - python -m markitdown template.pptx - ``` - Review `thumbnails.jpg` to see layouts, and markitdown output to see placeholder text. - -2. **Plan slide mapping**: For each content section, choose a template slide. - - ⚠️ **USE VARIED LAYOUTS** — monotonous presentations are a common failure mode. Don't default to basic title + bullet slides. Actively seek out: - - Multi-column layouts (2-column, 3-column) - - Image + text combinations - - Full-bleed images with text overlay - - Quote or callout slides - - Section dividers - - Stat/number callouts - - Icon grids or icon + text rows - - **Avoid:** Repeating the same text-heavy layout for every slide. - - Match content type to layout style (e.g., key points → bullet slide, team info → multi-column, testimonials → quote slide). - -3. **Unpack**: `python scripts/office/unpack.py template.pptx unpacked/` - -4. **Build presentation** (do this yourself, not with subagents): - - Delete unwanted slides (remove from ``) - - Duplicate slides you want to reuse (`add_slide.py`) - - Reorder slides in `` - - **Complete all structural changes before step 5** - -5. **Edit content**: Update text in each `slide{N}.xml`. - **Use subagents here if available** — slides are separate XML files, so subagents can edit in parallel. - -6. **Clean**: `python scripts/clean.py unpacked/` - -7. **Pack**: `python scripts/office/pack.py unpacked/ output.pptx --original template.pptx` - ---- - -## Scripts - -| Script | Purpose | -|--------|---------| -| `unpack.py` | Extract and pretty-print PPTX | -| `add_slide.py` | Duplicate slide or create from layout | -| `clean.py` | Remove orphaned files | -| `pack.py` | Repack with validation | -| `thumbnail.py` | Create visual grid of slides | - -### unpack.py - -```bash -python scripts/office/unpack.py input.pptx unpacked/ -``` - -Extracts PPTX, pretty-prints XML, escapes smart quotes. - -### add_slide.py - -```bash -python scripts/add_slide.py unpacked/ slide2.xml # Duplicate slide -python scripts/add_slide.py unpacked/ slideLayout2.xml # From layout -``` - -Prints `` to add to `` at desired position. - -### clean.py - -```bash -python scripts/clean.py unpacked/ -``` - -Removes slides not in ``, unreferenced media, orphaned rels. - -### pack.py - -```bash -python scripts/office/pack.py unpacked/ output.pptx --original input.pptx -``` - -Validates, repairs, condenses XML, re-encodes smart quotes. - -### thumbnail.py - -```bash -python scripts/thumbnail.py input.pptx [output_prefix] [--cols N] -``` - -Creates `thumbnails.jpg` with slide filenames as labels. Default 3 columns, max 12 per grid. - -**Use for template analysis only** (choosing layouts). For visual QA, use `soffice` + `pdftoppm` to create full-resolution individual slide images—see SKILL.md. - ---- - -## Slide Operations - -Slide order is in `ppt/presentation.xml` → ``. - -**Reorder**: Rearrange `` elements. - -**Delete**: Remove ``, then run `clean.py`. - -**Add**: Use `add_slide.py`. Never manually copy slide files—the script handles notes references, Content_Types.xml, and relationship IDs that manual copying misses. - ---- - -## Editing Content - -**Subagents:** If available, use them here (after completing step 4). Each slide is a separate XML file, so subagents can edit in parallel. In your prompt to subagents, include: -- The slide file path(s) to edit -- **"Use the Edit tool for all changes"** -- The formatting rules and common pitfalls below - -For each slide: -1. Read the slide's XML -2. Identify ALL placeholder content—text, images, charts, icons, captions -3. Replace each placeholder with final content - -**Use the Edit tool, not sed or Python scripts.** The Edit tool forces specificity about what to replace and where, yielding better reliability. - -### Formatting Rules - -- **Bold all headers, subheadings, and inline labels**: Use `b="1"` on ``. This includes: - - Slide titles - - Section headers within a slide - - Inline labels like (e.g.: "Status:", "Description:") at the start of a line -- **Never use unicode bullets (•)**: Use proper list formatting with `` or `` -- **Bullet consistency**: Let bullets inherit from the layout. Only specify `` or ``. - ---- - -## Common Pitfalls - -### Template Adaptation - -When source content has fewer items than the template: -- **Remove excess elements entirely** (images, shapes, text boxes), don't just clear text -- Check for orphaned visuals after clearing text content -- Run visual QA to catch mismatched counts - -When replacing text with different length content: -- **Shorter replacements**: Usually safe -- **Longer replacements**: May overflow or wrap unexpectedly -- Test with visual QA after text changes -- Consider truncating or splitting content to fit the template's design constraints - -**Template slots ≠ Source items**: If template has 4 team members but source has 3 users, delete the 4th member's entire group (image + text boxes), not just the text. - -### Multi-Item Content - -If source has multiple items (numbered lists, multiple sections), create separate `` elements for each — **never concatenate into one string**. - -**❌ WRONG** — all items in one paragraph: -```xml - - Step 1: Do the first thing. Step 2: Do the second thing. - -``` - -**✅ CORRECT** — separate paragraphs with bold headers: -```xml - - - Step 1 - - - - Do the first thing. - - - - Step 2 - - -``` - -Copy `` from the original paragraph to preserve line spacing. Use `b="1"` on headers. - -### Smart Quotes - -Handled automatically by unpack/pack. But the Edit tool converts smart quotes to ASCII. - -**When adding new text with quotes, use XML entities:** - -```xml -the “Agreement” -``` - -| Character | Name | Unicode | XML Entity | -|-----------|------|---------|------------| -| `“` | Left double quote | U+201C | `“` | -| `”` | Right double quote | U+201D | `”` | -| `‘` | Left single quote | U+2018 | `‘` | -| `’` | Right single quote | U+2019 | `’` | - -### Other - -- **Whitespace**: Use `xml:space="preserve"` on `` with leading/trailing spaces -- **XML parsing**: Use `defusedxml.minidom`, not `xml.etree.ElementTree` (corrupts namespaces) diff --git a/src/crates/assembly/core/builtin_skills/pptx/pptxgenjs.md b/src/crates/assembly/core/builtin_skills/pptx/pptxgenjs.md deleted file mode 100644 index 6bfed908c9..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/pptxgenjs.md +++ /dev/null @@ -1,420 +0,0 @@ -# PptxGenJS Tutorial - -## Setup & Basic Structure - -```javascript -const pptxgen = require("pptxgenjs"); - -let pres = new pptxgen(); -pres.layout = 'LAYOUT_16x9'; // or 'LAYOUT_16x10', 'LAYOUT_4x3', 'LAYOUT_WIDE' -pres.author = 'Your Name'; -pres.title = 'Presentation Title'; - -let slide = pres.addSlide(); -slide.addText("Hello World!", { x: 0.5, y: 0.5, fontSize: 36, color: "363636" }); - -pres.writeFile({ fileName: "Presentation.pptx" }); -``` - -## Layout Dimensions - -Slide dimensions (coordinates in inches): -- `LAYOUT_16x9`: 10" × 5.625" (default) -- `LAYOUT_16x10`: 10" × 6.25" -- `LAYOUT_4x3`: 10" × 7.5" -- `LAYOUT_WIDE`: 13.3" × 7.5" - ---- - -## Text & Formatting - -```javascript -// Basic text -slide.addText("Simple Text", { - x: 1, y: 1, w: 8, h: 2, fontSize: 24, fontFace: "Arial", - color: "363636", bold: true, align: "center", valign: "middle" -}); - -// Character spacing (use charSpacing, not letterSpacing which is silently ignored) -slide.addText("SPACED TEXT", { x: 1, y: 1, w: 8, h: 1, charSpacing: 6 }); - -// Rich text arrays -slide.addText([ - { text: "Bold ", options: { bold: true } }, - { text: "Italic ", options: { italic: true } } -], { x: 1, y: 3, w: 8, h: 1 }); - -// Multi-line text (requires breakLine: true) -slide.addText([ - { text: "Line 1", options: { breakLine: true } }, - { text: "Line 2", options: { breakLine: true } }, - { text: "Line 3" } // Last item doesn't need breakLine -], { x: 0.5, y: 0.5, w: 8, h: 2 }); - -// Text box margin (internal padding) -slide.addText("Title", { - x: 0.5, y: 0.3, w: 9, h: 0.6, - margin: 0 // Use 0 when aligning text with other elements like shapes or icons -}); -``` - -**Tip:** Text boxes have internal margin by default. Set `margin: 0` when you need text to align precisely with shapes, lines, or icons at the same x-position. - ---- - -## Lists & Bullets - -```javascript -// ✅ CORRECT: Multiple bullets -slide.addText([ - { text: "First item", options: { bullet: true, breakLine: true } }, - { text: "Second item", options: { bullet: true, breakLine: true } }, - { text: "Third item", options: { bullet: true } } -], { x: 0.5, y: 0.5, w: 8, h: 3 }); - -// ❌ WRONG: Never use unicode bullets -slide.addText("• First item", { ... }); // Creates double bullets - -// Sub-items and numbered lists -{ text: "Sub-item", options: { bullet: true, indentLevel: 1 } } -{ text: "First", options: { bullet: { type: "number" }, breakLine: true } } -``` - ---- - -## Shapes - -```javascript -slide.addShape(pres.shapes.RECTANGLE, { - x: 0.5, y: 0.8, w: 1.5, h: 3.0, - fill: { color: "FF0000" }, line: { color: "000000", width: 2 } -}); - -slide.addShape(pres.shapes.OVAL, { x: 4, y: 1, w: 2, h: 2, fill: { color: "0000FF" } }); - -slide.addShape(pres.shapes.LINE, { - x: 1, y: 3, w: 5, h: 0, line: { color: "FF0000", width: 3, dashType: "dash" } -}); - -// With transparency -slide.addShape(pres.shapes.RECTANGLE, { - x: 1, y: 1, w: 3, h: 2, - fill: { color: "0088CC", transparency: 50 } -}); - -// Rounded rectangle (rectRadius only works with ROUNDED_RECTANGLE, not RECTANGLE) -// ⚠️ Don't pair with rectangular accent overlays — they won't cover rounded corners. Use RECTANGLE instead. -slide.addShape(pres.shapes.ROUNDED_RECTANGLE, { - x: 1, y: 1, w: 3, h: 2, - fill: { color: "FFFFFF" }, rectRadius: 0.1 -}); - -// With shadow -slide.addShape(pres.shapes.RECTANGLE, { - x: 1, y: 1, w: 3, h: 2, - fill: { color: "FFFFFF" }, - shadow: { type: "outer", color: "000000", blur: 6, offset: 2, angle: 135, opacity: 0.15 } -}); -``` - -Shadow options: - -| Property | Type | Range | Notes | -|----------|------|-------|-------| -| `type` | string | `"outer"`, `"inner"` | | -| `color` | string | 6-char hex (e.g. `"000000"`) | No `#` prefix, no 8-char hex — see Common Pitfalls | -| `blur` | number | 0-100 pt | | -| `offset` | number | 0-200 pt | **Must be non-negative** — negative values corrupt the file | -| `angle` | number | 0-359 degrees | Direction the shadow falls (135 = bottom-right, 270 = upward) | -| `opacity` | number | 0.0-1.0 | Use this for transparency, never encode in color string | - -To cast a shadow upward (e.g. on a footer bar), use `angle: 270` with a positive offset — do **not** use a negative offset. - -**Note**: Gradient fills are not natively supported. Use a gradient image as a background instead. - ---- - -## Images - -### Image Sources - -```javascript -// From file path -slide.addImage({ path: "images/chart.png", x: 1, y: 1, w: 5, h: 3 }); - -// From URL -slide.addImage({ path: "https://example.com/image.jpg", x: 1, y: 1, w: 5, h: 3 }); - -// From base64 (faster, no file I/O) -slide.addImage({ data: "image/png;base64,iVBORw0KGgo...", x: 1, y: 1, w: 5, h: 3 }); -``` - -### Image Options - -```javascript -slide.addImage({ - path: "image.png", - x: 1, y: 1, w: 5, h: 3, - rotate: 45, // 0-359 degrees - rounding: true, // Circular crop - transparency: 50, // 0-100 - flipH: true, // Horizontal flip - flipV: false, // Vertical flip - altText: "Description", // Accessibility - hyperlink: { url: "https://example.com" } -}); -``` - -### Image Sizing Modes - -```javascript -// Contain - fit inside, preserve ratio -{ sizing: { type: 'contain', w: 4, h: 3 } } - -// Cover - fill area, preserve ratio (may crop) -{ sizing: { type: 'cover', w: 4, h: 3 } } - -// Crop - cut specific portion -{ sizing: { type: 'crop', x: 0.5, y: 0.5, w: 2, h: 2 } } -``` - -### Calculate Dimensions (preserve aspect ratio) - -```javascript -const origWidth = 1978, origHeight = 923, maxHeight = 3.0; -const calcWidth = maxHeight * (origWidth / origHeight); -const centerX = (10 - calcWidth) / 2; - -slide.addImage({ path: "image.png", x: centerX, y: 1.2, w: calcWidth, h: maxHeight }); -``` - -### Supported Formats - -- **Standard**: PNG, JPG, GIF (animated GIFs work in Microsoft 365) -- **SVG**: Works in modern PowerPoint/Microsoft 365 - ---- - -## Icons - -Use react-icons to generate SVG icons, then rasterize to PNG for universal compatibility. - -### Setup - -```javascript -const React = require("react"); -const ReactDOMServer = require("react-dom/server"); -const sharp = require("sharp"); -const { FaCheckCircle, FaChartLine } = require("react-icons/fa"); - -function renderIconSvg(IconComponent, color = "#000000", size = 256) { - return ReactDOMServer.renderToStaticMarkup( - React.createElement(IconComponent, { color, size: String(size) }) - ); -} - -async function iconToBase64Png(IconComponent, color, size = 256) { - const svg = renderIconSvg(IconComponent, color, size); - const pngBuffer = await sharp(Buffer.from(svg)).png().toBuffer(); - return "image/png;base64," + pngBuffer.toString("base64"); -} -``` - -### Add Icon to Slide - -```javascript -const iconData = await iconToBase64Png(FaCheckCircle, "#4472C4", 256); - -slide.addImage({ - data: iconData, - x: 1, y: 1, w: 0.5, h: 0.5 // Size in inches -}); -``` - -**Note**: Use size 256 or higher for crisp icons. The size parameter controls the rasterization resolution, not the display size on the slide (which is set by `w` and `h` in inches). - -### Icon Libraries - -Install: `npm install -g react-icons react react-dom sharp` - -Popular icon sets in react-icons: -- `react-icons/fa` - Font Awesome -- `react-icons/md` - Material Design -- `react-icons/hi` - Heroicons -- `react-icons/bi` - Bootstrap Icons - ---- - -## Slide Backgrounds - -```javascript -// Solid color -slide.background = { color: "F1F1F1" }; - -// Color with transparency -slide.background = { color: "FF3399", transparency: 50 }; - -// Image from URL -slide.background = { path: "https://example.com/bg.jpg" }; - -// Image from base64 -slide.background = { data: "image/png;base64,iVBORw0KGgo..." }; -``` - ---- - -## Tables - -```javascript -slide.addTable([ - ["Header 1", "Header 2"], - ["Cell 1", "Cell 2"] -], { - x: 1, y: 1, w: 8, h: 2, - border: { pt: 1, color: "999999" }, fill: { color: "F1F1F1" } -}); - -// Advanced with merged cells -let tableData = [ - [{ text: "Header", options: { fill: { color: "6699CC" }, color: "FFFFFF", bold: true } }, "Cell"], - [{ text: "Merged", options: { colspan: 2 } }] -]; -slide.addTable(tableData, { x: 1, y: 3.5, w: 8, colW: [4, 4] }); -``` - ---- - -## Charts - -```javascript -// Bar chart -slide.addChart(pres.charts.BAR, [{ - name: "Sales", labels: ["Q1", "Q2", "Q3", "Q4"], values: [4500, 5500, 6200, 7100] -}], { - x: 0.5, y: 0.6, w: 6, h: 3, barDir: 'col', - showTitle: true, title: 'Quarterly Sales' -}); - -// Line chart -slide.addChart(pres.charts.LINE, [{ - name: "Temp", labels: ["Jan", "Feb", "Mar"], values: [32, 35, 42] -}], { x: 0.5, y: 4, w: 6, h: 3, lineSize: 3, lineSmooth: true }); - -// Pie chart -slide.addChart(pres.charts.PIE, [{ - name: "Share", labels: ["A", "B", "Other"], values: [35, 45, 20] -}], { x: 7, y: 1, w: 5, h: 4, showPercent: true }); -``` - -### Better-Looking Charts - -Default charts look dated. Apply these options for a modern, clean appearance: - -```javascript -slide.addChart(pres.charts.BAR, chartData, { - x: 0.5, y: 1, w: 9, h: 4, barDir: "col", - - // Custom colors (match your presentation palette) - chartColors: ["0D9488", "14B8A6", "5EEAD4"], - - // Clean background - chartArea: { fill: { color: "FFFFFF" }, roundedCorners: true }, - - // Muted axis labels - catAxisLabelColor: "64748B", - valAxisLabelColor: "64748B", - - // Subtle grid (value axis only) - valGridLine: { color: "E2E8F0", size: 0.5 }, - catGridLine: { style: "none" }, - - // Data labels on bars - showValue: true, - dataLabelPosition: "outEnd", - dataLabelColor: "1E293B", - - // Hide legend for single series - showLegend: false, -}); -``` - -**Key styling options:** -- `chartColors: [...]` - hex colors for series/segments -- `chartArea: { fill, border, roundedCorners }` - chart background -- `catGridLine/valGridLine: { color, style, size }` - grid lines (`style: "none"` to hide) -- `lineSmooth: true` - curved lines (line charts) -- `legendPos: "r"` - legend position: "b", "t", "l", "r", "tr" - ---- - -## Slide Masters - -```javascript -pres.defineSlideMaster({ - title: 'TITLE_SLIDE', background: { color: '283A5E' }, - objects: [{ - placeholder: { options: { name: 'title', type: 'title', x: 1, y: 2, w: 8, h: 2 } } - }] -}); - -let titleSlide = pres.addSlide({ masterName: "TITLE_SLIDE" }); -titleSlide.addText("My Title", { placeholder: "title" }); -``` - ---- - -## Common Pitfalls - -⚠️ These issues cause file corruption, visual bugs, or broken output. Avoid them. - -1. **NEVER use "#" with hex colors** - causes file corruption - ```javascript - color: "FF0000" // ✅ CORRECT - color: "#FF0000" // ❌ WRONG - ``` - -2. **NEVER encode opacity in hex color strings** - 8-char colors (e.g., `"00000020"`) corrupt the file. Use the `opacity` property instead. - ```javascript - shadow: { type: "outer", blur: 6, offset: 2, color: "00000020" } // ❌ CORRUPTS FILE - shadow: { type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.12 } // ✅ CORRECT - ``` - -3. **Use `bullet: true`** - NEVER unicode symbols like "•" (creates double bullets) - -4. **Use `breakLine: true`** between array items or text runs together - -5. **Avoid `lineSpacing` with bullets** - causes excessive gaps; use `paraSpaceAfter` instead - -6. **Each presentation needs fresh instance** - don't reuse `pptxgen()` objects - -7. **NEVER reuse option objects across calls** - PptxGenJS mutates objects in-place (e.g. converting shadow values to EMU). Sharing one object between multiple calls corrupts the second shape. - ```javascript - const shadow = { type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.15 }; - slide.addShape(pres.shapes.RECTANGLE, { shadow, ... }); // ❌ second call gets already-converted values - slide.addShape(pres.shapes.RECTANGLE, { shadow, ... }); - - const makeShadow = () => ({ type: "outer", blur: 6, offset: 2, color: "000000", opacity: 0.15 }); - slide.addShape(pres.shapes.RECTANGLE, { shadow: makeShadow(), ... }); // ✅ fresh object each time - slide.addShape(pres.shapes.RECTANGLE, { shadow: makeShadow(), ... }); - ``` - -8. **Don't use `ROUNDED_RECTANGLE` with accent borders** - rectangular overlay bars won't cover rounded corners. Use `RECTANGLE` instead. - ```javascript - // ❌ WRONG: Accent bar doesn't cover rounded corners - slide.addShape(pres.shapes.ROUNDED_RECTANGLE, { x: 1, y: 1, w: 3, h: 1.5, fill: { color: "FFFFFF" } }); - slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 0.08, h: 1.5, fill: { color: "0891B2" } }); - - // ✅ CORRECT: Use RECTANGLE for clean alignment - slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 3, h: 1.5, fill: { color: "FFFFFF" } }); - slide.addShape(pres.shapes.RECTANGLE, { x: 1, y: 1, w: 0.08, h: 1.5, fill: { color: "0891B2" } }); - ``` - ---- - -## Quick Reference - -- **Shapes**: RECTANGLE, OVAL, LINE, ROUNDED_RECTANGLE -- **Charts**: BAR, LINE, PIE, DOUGHNUT, SCATTER, BUBBLE, RADAR -- **Layouts**: LAYOUT_16x9 (10"×5.625"), LAYOUT_16x10, LAYOUT_4x3, LAYOUT_WIDE -- **Alignment**: "left", "center", "right" -- **Chart data labels**: "outEnd", "inEnd", "center" diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/add_slide.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/add_slide.py index 13700df012..f013ea94d1 100755 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/add_slide.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/add_slide.py @@ -1,51 +1,41 @@ -"""Add a new slide to an unpacked PPTX directory. - -Usage: python add_slide.py - -The source can be: - - A slide file (e.g., slide2.xml) - duplicates the slide - - A layout file (e.g., slideLayout2.xml) - creates from layout - -Examples: - python add_slide.py unpacked/ slide2.xml - # Duplicates slide2, creates slide5.xml - - python add_slide.py unpacked/ slideLayout2.xml - # Creates slide5.xml from slideLayout2.xml - -To see available layouts: ls unpacked/ppt/slideLayouts/ - -Prints the element to add to presentation.xml. +"""Add a slide to a PPTX: duplicate an existing slide or instantiate a layout. + +Does all of the package bookkeeping, so the deck stays valid: + - writes the new ppt/slides/slideN.xml (and its .rels, minus any + notesSlide reference, so the source's speaker notes aren't shared) + - registers it in [Content_Types].xml + - adds a slide relationship with a fresh rId to presentation.xml.rels + - inserts with a fresh id into + — at the end, or after --after SLIDE + +Works on an unpacked directory (during an editing session) or directly on a +.pptx/.potx file (extracted to a temp dir, then rezipped atomically; the +temp dir is discarded, so unpack the output if you still need to edit the +new slide's content). + +Usage: + python add_slide.py unpacked/ slide2.xml # duplicate slide2 + python add_slide.py unpacked/ slideLayout3.xml # new slide from a layout + python add_slide.py unpacked/ slide2.xml --after slide2.xml + python add_slide.py deck.pptx slide2.xml # rewrite deck.pptx in place + python add_slide.py deck.pptx slide2.xml -o out.pptx + +A duplicated slide still holds the source's content: edit ppt/slides/slideN.xml +(printed on success) to change it. To list layouts: ls

/ppt/slideLayouts/ """ +import argparse import re import shutil import sys +from typing import NoReturn +import tempfile +import zipfile from pathlib import Path +from office.helpers import rezip, safe_extract -def get_next_slide_number(slides_dir: Path) -> int: - existing = [int(m.group(1)) for f in slides_dir.glob("slide*.xml") - if (m := re.match(r"slide(\d+)\.xml", f.name))] - return max(existing) + 1 if existing else 1 - - -def create_slide_from_layout(unpacked_dir: Path, layout_file: str) -> None: - slides_dir = unpacked_dir / "ppt" / "slides" - rels_dir = slides_dir / "_rels" - layouts_dir = unpacked_dir / "ppt" / "slideLayouts" - - layout_path = layouts_dir / layout_file - if not layout_path.exists(): - print(f"Error: {layout_path} not found", file=sys.stderr) - sys.exit(1) - - next_num = get_next_slide_number(slides_dir) - dest = f"slide{next_num}.xml" - dest_slide = slides_dir / dest - dest_rels = rels_dir / f"{dest}.rels" - - slide_xml = ''' +MINIMAL_SLIDE_XML = ''' @@ -68,63 +58,147 @@ def create_slide_from_layout(unpacked_dir: Path, layout_file: str) -> None: ''' - dest_slide.write_text(slide_xml, encoding="utf-8") + +SHARED_PART_TYPES = ("chart", "diagramData", "oleObject", "package") + +NOTES_SLIDE_TYPE_RE = re.compile(r"""Type=["'][^"']*/relationships/notesSlide["']""") +RELATIONSHIP_RE = re.compile(r"]*?(?:/>|>.*?)", re.DOTALL) + +SLIDE_ID_MIN = 256 +SLIDE_ID_MAX = 2147483647 + + +def _die(msg: str) -> NoReturn: + print(f"Error: {msg}", file=sys.stderr) + sys.exit(1) + + +def get_next_slide_number(slides_dir: Path) -> int: + existing = [int(m.group(1)) for f in slides_dir.glob("slide*.xml") + if (m := re.match(r"slide(\d+)\.xml", f.name))] + return max(existing) + 1 if existing else 1 + + +def parse_source(source: str) -> tuple[str, str | None]: + if source.startswith("slideLayout") and source.endswith(".xml"): + return ("layout", source) + + return ("slide", None) + + +def create_slide_from_layout(unpacked_dir: Path, layout_file: str, after: str | None = None) -> str: + slides_dir = unpacked_dir / "ppt" / "slides" + rels_dir = slides_dir / "_rels" + layout_path = unpacked_dir / "ppt" / "slideLayouts" / layout_file + + if not layout_path.exists(): + _die(f"{layout_path} not found") + + next_num = get_next_slide_number(slides_dir) + dest = f"slide{next_num}.xml" + after_rid = _precheck_registration(unpacked_dir, after, dest) + slides_dir.mkdir(parents=True, exist_ok=True) + + (slides_dir / dest).write_text(MINIMAL_SLIDE_XML, encoding="utf-8") rels_dir.mkdir(exist_ok=True) rels_xml = f''' ''' - dest_rels.write_text(rels_xml, encoding="utf-8") + (rels_dir / f"{dest}.rels").write_text(rels_xml, encoding="utf-8") - _add_to_content_types(unpacked_dir, dest) + _register_slide(unpacked_dir, dest, layout_file, after_rid) + return dest - rid = _add_to_presentation_rels(unpacked_dir, dest) - next_slide_id = _get_next_slide_id(unpacked_dir) - - print(f"Created {dest} from {layout_file}") - print(f'Add to presentation.xml : ') - - -def duplicate_slide(unpacked_dir: Path, source: str) -> None: +def duplicate_slide(unpacked_dir: Path, source: str, after: str | None = None) -> str: slides_dir = unpacked_dir / "ppt" / "slides" rels_dir = slides_dir / "_rels" - source_slide = slides_dir / source if not source_slide.exists(): - print(f"Error: {source_slide} not found", file=sys.stderr) - sys.exit(1) + _die(f"{source_slide} not found") next_num = get_next_slide_number(slides_dir) dest = f"slide{next_num}.xml" - dest_slide = slides_dir / dest - - source_rels = rels_dir / f"{source}.rels" - dest_rels = rels_dir / f"{dest}.rels" + after_rid = _precheck_registration(unpacked_dir, after, dest) - shutil.copy2(source_slide, dest_slide) + shutil.copy2(source_slide, slides_dir / dest) + source_rels = rels_dir / f"{source}.rels" + shared_parts: list[str] = [] if source_rels.exists(): + dest_rels = rels_dir / f"{dest}.rels" shutil.copy2(source_rels, dest_rels) - rels_content = dest_rels.read_text(encoding="utf-8") - rels_content = re.sub( - r'\s*]*Type="[^"]*notesSlide"[^>]*/>\s*', - "\n", + rels_content = RELATIONSHIP_RE.sub( + lambda m: "" if NOTES_SLIDE_TYPE_RE.search(m.group(0)) else m.group(0), rels_content, ) dest_rels.write_text(rels_content, encoding="utf-8") + shared_parts = sorted({ + t for t in re.findall(r'Type="[^"]*/relationships/(\w+)"', rels_content) + if t in SHARED_PART_TYPES + }) + + _register_slide(unpacked_dir, dest, source, after_rid) + if shared_parts: + print( + f"Note: {dest} shares its {', '.join(shared_parts)} part(s) with {source} " + f"(they are referenced, not copied) — editing those parts changes both slides" + ) + return dest - _add_to_content_types(unpacked_dir, dest) - rid = _add_to_presentation_rels(unpacked_dir, dest) +def _precheck_registration(unpacked_dir: Path, after: str | None, dest: str) -> str | None: + pres_path = unpacked_dir / "ppt" / "presentation.xml" + if not pres_path.exists(): + _die(f"{pres_path} not found — is this an unpacked PPTX?") + xml = pres_path.read_text(encoding="utf-8") + + has_slot = ( + "" in xml + or re.search(r"", xml) + or "" in xml + ) + if not has_slot: + _die("presentation.xml has no (or to anchor a new one)") + + stale = [] + content_types = unpacked_dir / "[Content_Types].xml" + if content_types.exists() and f'PartName="/ppt/slides/{dest}"' in content_types.read_text(encoding="utf-8"): + stale.append("[Content_Types].xml") + pres_rels = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels" + if pres_rels.exists() and _find_slide_relationship( + pres_rels.read_text(encoding="utf-8"), dest + ): + stale.append("presentation.xml.rels") + if stale: + _die( + f"{dest} is still registered in {' and '.join(stale)} but absent from ppt/slides/ — " + f"run clean.py first" + ) + + if not after: + return None + after_rid = _rid_for_slide(unpacked_dir, after) + if not re.search(rf']*r:id="{re.escape(after_rid)}"[^>]*>', xml): + _die(f"{after} ({after_rid}) is not listed in ") + return after_rid - next_slide_id = _get_next_slide_id(unpacked_dir) - print(f"Created {dest} from {source}") - print(f'Add to presentation.xml : ') +def _register_slide(unpacked_dir: Path, dest: str, source_desc: str, after_rid: str | None) -> None: + _add_to_content_types(unpacked_dir, dest) + rid = _add_to_presentation_rels(unpacked_dir, dest) + slide_id = _get_next_slide_id(unpacked_dir) + pos, total = _insert_into_sld_id_lst(unpacked_dir, slide_id, rid, after_rid) + + print(f"Created ppt/slides/{dest} from {source_desc}") + print( + f'Inserted into ' + f"at position {pos} of {total}" + ) def _add_to_content_types(unpacked_dir: Path, dest: str) -> None: @@ -133,7 +207,7 @@ def _add_to_content_types(unpacked_dir: Path, dest: str) -> None: new_override = f'' - if f"/ppt/slides/{dest}" not in content_types: + if f'PartName="/ppt/slides/{dest}"' not in content_types: content_types = content_types.replace("", f" {new_override}\n") content_types_path.write_text(content_types, encoding="utf-8") @@ -142,54 +216,152 @@ def _add_to_presentation_rels(unpacked_dir: Path, dest: str) -> str: pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels" pres_rels = pres_rels_path.read_text(encoding="utf-8") - rids = [int(m) for m in re.findall(r'Id="rId(\d+)"', pres_rels)] - next_rid = max(rids) + 1 if rids else 1 - rid = f"rId{next_rid}" + existing = _find_slide_relationship(pres_rels, dest) + if existing: + return existing - new_rel = f'' + pres_xml = (unpacked_dir / "ppt" / "presentation.xml").read_text(encoding="utf-8") + used = {int(n) for n in re.findall(r'\bId="rId(\d+)"', pres_rels)} + used |= {int(n) for n in re.findall(r'\br:id="rId(\d+)"', pres_xml)} + rid = f"rId{max(used) + 1 if used else 1}" - if f"slides/{dest}" not in pres_rels: - pres_rels = pres_rels.replace("", f" {new_rel}\n") - pres_rels_path.write_text(pres_rels, encoding="utf-8") + new_rel = f'' + pres_rels = pres_rels.replace("", f" {new_rel}\n") + pres_rels_path.write_text(pres_rels, encoding="utf-8") return rid +def _find_slide_relationship(pres_rels: str, slide_name: str) -> str | None: + for m in re.finditer(r"]*>", pres_rels): + element = m.group(0) + if re.search(rf'Target="(?:/ppt/)?slides/{re.escape(slide_name)}"', element): + id_match = re.search(r'\bId="([^"]+)"', element) + if id_match: + return id_match.group(1) + return None + + def _get_next_slide_id(unpacked_dir: Path) -> int: + pres_content = (unpacked_dir / "ppt" / "presentation.xml").read_text(encoding="utf-8") + used = {int(m) for m in re.findall(r']*\bid="(\d+)"', pres_content)} + + candidate = max((i for i in used if i >= SLIDE_ID_MIN), default=SLIDE_ID_MIN - 1) + 1 + if candidate <= SLIDE_ID_MAX and candidate not in used: + return candidate + for i in range(SLIDE_ID_MIN, SLIDE_ID_MAX + 1): + if i not in used: + return i + _die("no slide id available in [256, 2147483647] — the deck is full") + + +def _insert_into_sld_id_lst( + unpacked_dir: Path, slide_id: int, rid: str, after_rid: str | None = None +) -> tuple[int, int]: pres_path = unpacked_dir / "ppt" / "presentation.xml" - pres_content = pres_path.read_text(encoding="utf-8") - slide_ids = [int(m) for m in re.findall(r']*id="(\d+)"', pres_content)] - return max(slide_ids) + 1 if slide_ids else 256 + xml = pres_path.read_text(encoding="utf-8") + entry = f'' + + if f'r:id="{rid}"' in xml: + _die(f"presentation.xml already references {rid}; refusing to add a duplicate") + + if after_rid: + open_tag = re.search(rf']*r:id="{re.escape(after_rid)}"[^>]*>', xml) + if not open_tag: + _die(f"{after_rid} is not listed in ") + end = open_tag.end() + if not open_tag.group(0).endswith("/>"): + close = xml.find("", end) + if close == -1: + _die(f"unclosed for {after_rid} in presentation.xml") + end = close + len("") + xml = xml[:end] + entry + xml[end:] + elif "" in xml: + xml = xml.replace("", f"{entry}", 1) + elif re.search(r"", xml): + xml = re.sub(r"", f"{entry}", xml, count=1) + elif "" in xml: + xml = xml.replace( + "", f"{entry}", 1 + ) + else: + _die("presentation.xml has no (or to anchor a new one)") + pres_path.write_text(xml, encoding="utf-8") -def parse_source(source: str) -> tuple[str, str | None]: - if source.startswith("slideLayout") and source.endswith(".xml"): - return ("layout", source) + lst = re.search(r"(.*)", xml, re.DOTALL) + entries = re.findall(r"]*>", lst.group(1)) if lst else [] + position = next( + (i for i, e in enumerate(entries, 1) if f'r:id="{rid}"' in e), len(entries) + ) + return position, len(entries) - return ("slide", None) +def _rid_for_slide(unpacked_dir: Path, slide_name: str) -> str: + pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels" + rid = _find_slide_relationship(pres_rels_path.read_text(encoding="utf-8"), slide_name) + if not rid: + _die(f"{slide_name} has no relationship in presentation.xml.rels") + return rid -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: python add_slide.py ", file=sys.stderr) - print("", file=sys.stderr) - print("Source can be:", file=sys.stderr) - print(" slide2.xml - duplicate an existing slide", file=sys.stderr) - print(" slideLayout2.xml - create from a layout template", file=sys.stderr) - print("", file=sys.stderr) - print("To see available layouts: ls /ppt/slideLayouts/", file=sys.stderr) - sys.exit(1) - - unpacked_dir = Path(sys.argv[1]) - source = sys.argv[2] - - if not unpacked_dir.exists(): - print(f"Error: {unpacked_dir} not found", file=sys.stderr) - sys.exit(1) +def add_slide(unpacked_dir: Path, source: str, after: str | None = None) -> str: source_type, layout_file = parse_source(source) - if source_type == "layout" and layout_file is not None: - create_slide_from_layout(unpacked_dir, layout_file) + return create_slide_from_layout(unpacked_dir, layout_file, after) + return duplicate_slide(unpacked_dir, source, after) + + +def add_slide_to_package( + package: Path, source: str, after: str | None = None, output: Path | None = None +) -> str: + out = output or package + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + with zipfile.ZipFile(package) as zf: + safe_extract(zf, tmp_path) + dest = add_slide(tmp_path, source, after) + rezip(tmp_path, out) + print(f"Wrote {out} — the new slide is ppt/slides/{dest} inside it (unpack to edit its content)") + return dest + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Add a slide to a PPTX: duplicate a slide or instantiate a layout. " + "Registers content types, relationships, and ." + ) + parser.add_argument("target", help="Unpacked PPTX directory OR a .pptx/.potx file") + parser.add_argument( + "source", + help="slideN.xml to duplicate, or slideLayoutN.xml to create from a layout " + "(list layouts with: ls /ppt/slideLayouts/)", + ) + parser.add_argument( + "--after", + metavar="SLIDE", + help="insert after this slide, e.g. slide2.xml (default: append at the end)", + ) + parser.add_argument( + "-o", + "--output", + help="output file (only with a .pptx/.potx target; default: rewrite the input in place)", + ) + args = parser.parse_args() + + target = Path(args.target) + if target.is_dir(): + if args.output: + parser.error("--output is only valid for .pptx/.potx input; a directory is modified in place") + add_slide(target, args.source, args.after) + elif target.is_file() and target.suffix.lower() in (".pptx", ".potx"): + try: + add_slide_to_package(target, args.source, args.after, Path(args.output) if args.output else None) + except (OSError, ValueError, zipfile.BadZipFile) as e: + _die(str(e)) else: - duplicate_slide(unpacked_dir, source) + _die(f"{target} is neither a directory nor a .pptx/.potx file") + + +if __name__ == "__main__": + main() diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/clean.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/clean.py index 3d13994cfe..551dd23192 100755 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/clean.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/clean.py @@ -15,13 +15,30 @@ - Content-Type overrides for deleted files """ +import posixpath +import re import sys from pathlib import Path import defusedxml.minidom +from office.helpers import SLIDE_REL_TYPE, opc_target, rels_source_part -import re + +def _slide_rids(pres_rels_path: Path, unpacked_dir: Path) -> dict[str, str]: + source_part = rels_source_part(pres_rels_path, unpacked_dir) + rels_dom = defusedxml.minidom.parse(str(pres_rels_path)) + + rids: dict[str, str] = {} + for rel in rels_dom.getElementsByTagName("Relationship"): + if rel.getAttribute("Type") != SLIDE_REL_TYPE: + continue + part = opc_target( + rel.getAttribute("Target"), source_part, rel.getAttribute("TargetMode") + ) + if part is not None: + rids[rel.getAttribute("Id")] = part + return rids def get_slides_in_sldidlst(unpacked_dir: Path) -> set[str]: @@ -31,19 +48,20 @@ def get_slides_in_sldidlst(unpacked_dir: Path) -> set[str]: if not pres_path.exists() or not pres_rels_path.exists(): return set() - rels_dom = defusedxml.minidom.parse(str(pres_rels_path)) - rid_to_slide = {} - for rel in rels_dom.getElementsByTagName("Relationship"): - rid = rel.getAttribute("Id") - target = rel.getAttribute("Target") - rel_type = rel.getAttribute("Type") - if "slide" in rel_type and target.startswith("slides/"): - rid_to_slide[rid] = target.replace("slides/", "") + rid_to_slide = _slide_rids(pres_rels_path, unpacked_dir) pres_content = pres_path.read_text(encoding="utf-8") referenced_rids = set(re.findall(r']*r:id="([^"]+)"', pres_content)) - return {rid_to_slide[rid] for rid in referenced_rids if rid in rid_to_slide} + return { + posixpath.basename(rid_to_slide[rid]) + for rid in referenced_rids + if rid in rid_to_slide + } + + +class RefusedToClean(Exception): + """The package does not look the way a readable package should.""" def remove_orphaned_slides(unpacked_dir: Path) -> list[str]: @@ -55,9 +73,25 @@ def remove_orphaned_slides(unpacked_dir: Path) -> list[str]: return [] referenced_slides = get_slides_in_sldidlst(unpacked_dir) + on_disk = sorted(slides_dir.glob("slide*.xml")) + + if on_disk and not any(s.name in referenced_slides for s in on_disk): + listed = re.findall( + r']*r:id="([^"]+)"', + (unpacked_dir / "ppt" / "presentation.xml").read_text(encoding="utf-8") + if (unpacked_dir / "ppt" / "presentation.xml").exists() + else "", + ) + if listed: + raise RefusedToClean( + f" lists {len(listed)} slide(s) and none of the " + f"{len(on_disk)} slide(s) on disk match any of them. Refusing to " + f"delete them all — this is a parse failure, not an empty deck." + ) + removed = [] - for slide_file in slides_dir.glob("slide*.xml"): + for slide_file in on_disk: if slide_file.name not in referenced_slides: rel_path = slide_file.relative_to(unpacked_dir) slide_file.unlink() @@ -70,16 +104,21 @@ def remove_orphaned_slides(unpacked_dir: Path) -> list[str]: if removed and pres_rels_path.exists(): rels_dom = defusedxml.minidom.parse(str(pres_rels_path)) + source_part = rels_source_part(pres_rels_path, unpacked_dir) changed = False for rel in list(rels_dom.getElementsByTagName("Relationship")): - target = rel.getAttribute("Target") - if target.startswith("slides/"): - slide_name = target.replace("slides/", "") - if slide_name not in referenced_slides: - if rel.parentNode: - rel.parentNode.removeChild(rel) - changed = True + if rel.getAttribute("Type") != SLIDE_REL_TYPE: + continue + part = opc_target( + rel.getAttribute("Target"), source_part, rel.getAttribute("TargetMode") + ) + if part is None: + continue + if posixpath.basename(part) not in referenced_slides: + if rel.parentNode: + rel.parentNode.removeChild(rel) + changed = True if changed: with open(pres_rels_path, "wb") as f: @@ -103,24 +142,18 @@ def remove_trash_directory(unpacked_dir: Path) -> list[str]: return removed -def get_slide_referenced_files(unpacked_dir: Path) -> set: +def _referenced_by(rels_files, unpacked_dir: Path) -> set: referenced = set() - slides_rels_dir = unpacked_dir / "ppt" / "slides" / "_rels" - - if not slides_rels_dir.exists(): - return referenced - for rels_file in slides_rels_dir.glob("*.rels"): + for rels_file in rels_files: + source_part = rels_source_part(rels_file, unpacked_dir) dom = defusedxml.minidom.parse(str(rels_file)) for rel in dom.getElementsByTagName("Relationship"): - target = rel.getAttribute("Target") - if not target: - continue - target_path = (rels_file.parent.parent / target).resolve() - try: - referenced.add(target_path.relative_to(unpacked_dir.resolve())) - except ValueError: - pass + part = opc_target( + rel.getAttribute("Target"), source_part, rel.getAttribute("TargetMode") + ) + if part is not None: + referenced.add(Path(part)) return referenced @@ -128,7 +161,6 @@ def get_slide_referenced_files(unpacked_dir: Path) -> set: def remove_orphaned_rels_files(unpacked_dir: Path) -> list[str]: resource_dirs = ["charts", "diagrams", "drawings"] removed = [] - slide_referenced = get_slide_referenced_files(unpacked_dir) for dir_name in resource_dirs: rels_dir = unpacked_dir / "ppt" / dir_name / "_rels" @@ -137,35 +169,15 @@ def remove_orphaned_rels_files(unpacked_dir: Path) -> list[str]: for rels_file in rels_dir.glob("*.rels"): resource_file = rels_dir.parent / rels_file.name.replace(".rels", "") - try: - resource_rel_path = resource_file.resolve().relative_to(unpacked_dir.resolve()) - except ValueError: - continue - - if not resource_file.exists() or resource_rel_path not in slide_referenced: + if not resource_file.exists(): rels_file.unlink() - rel_path = rels_file.relative_to(unpacked_dir) - removed.append(str(rel_path)) + removed.append(str(rels_file.relative_to(unpacked_dir))) return removed def get_referenced_files(unpacked_dir: Path) -> set: - referenced = set() - - for rels_file in unpacked_dir.rglob("*.rels"): - dom = defusedxml.minidom.parse(str(rels_file)) - for rel in dom.getElementsByTagName("Relationship"): - target = rel.getAttribute("Target") - if not target: - continue - target_path = (rels_file.parent.parent / target).resolve() - try: - referenced.add(target_path.relative_to(unpacked_dir.resolve())) - except ValueError: - pass - - return referenced + return _referenced_by(sorted(unpacked_dir.rglob("*.rels")), unpacked_dir) def remove_orphaned_files(unpacked_dir: Path, referenced: set) -> list[str]: @@ -241,6 +253,12 @@ def update_content_types(unpacked_dir: Path, removed_files: list[str]) -> None: def clean_unused_files(unpacked_dir: Path) -> list[str]: all_removed = [] + if list(unpacked_dir.rglob("*.rels")) and not get_referenced_files(unpacked_dir): + raise RefusedToClean( + "no relationship in this package names a part we can resolve. " + "Refusing to treat every file as unreferenced." + ) + slides_removed = remove_orphaned_slides(unpacked_dir) all_removed.extend(slides_removed) @@ -276,7 +294,12 @@ def clean_unused_files(unpacked_dir: Path) -> list[str]: print(f"Error: {unpacked_dir} not found", file=sys.stderr) sys.exit(1) - removed = clean_unused_files(unpacked_dir) + try: + removed = clean_unused_files(unpacked_dir) + except (RefusedToClean, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + print("Nothing was deleted.", file=sys.stderr) + sys.exit(1) if removed: print(f"Removed {len(removed)} unreferenced files:") diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/__init__.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/__init__.py index e69de29bb2..188b00aff4 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/__init__.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/__init__.py @@ -0,0 +1,150 @@ +import os +import posixpath +import re +import stat +import tempfile +import urllib.parse +import zipfile +from pathlib import Path + +OOXML_FAMILY = { + ".docx": "docx", + ".dotx": "docx", + ".pptx": "pptx", + ".potx": "pptx", + ".xlsx": "xlsx", + ".xltx": "xlsx", +} + +_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:") + +SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" + +MAX_ARCHIVE_MEMBERS = 10_000 +MAX_ARCHIVE_MEMBER_SIZE = 1 * 1024 * 1024 * 1024 +MAX_ARCHIVE_TOTAL_SIZE = 4 * 1024 * 1024 * 1024 +MAX_ARCHIVE_COMPRESSION_RATIO = 1_000 + + +def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None: + if not target: + return None + if target_mode.lower() == "external": + return None + if _SCHEME_RE.match(target): + return None + + target = urllib.parse.unquote(target) + + if "\\" in target: + raise ValueError(f"relationship target is not a POSIX part name: {target!r}") + + if target.startswith("/"): + joined = target.lstrip("/") + else: + joined = posixpath.join(posixpath.dirname(source_part), target) + + parts: list[str] = [] + for segment in posixpath.normpath(joined).split("/"): + if segment in ("", "."): + continue + if segment == "..": + if not parts: + raise ValueError(f"relationship target escapes the package: {target!r}") + parts.pop() + else: + parts.append(segment) + + if not parts: + raise ValueError(f"relationship target resolves to nothing: {target!r}") + return "/".join(parts) + + +def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str: + owner_dir = rels_file.parent.parent.relative_to(unpacked_dir) + return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./") + + +def part_text(data: bytes) -> str: + return data.decode("utf-8", "surrogateescape") + + +XML_SPACE = " \t\r\n" + + +def rendered_text(text: str, preserve: bool) -> str: + return text if preserve else text.strip(XML_SPACE) + + +def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None: + dest = dest.resolve() + members = zf.infolist() + if len(members) > MAX_ARCHIVE_MEMBERS: + raise ValueError(f"archive has too many entries: {len(members)}") + + total_size = 0 + targets: set[str] = set() + file_targets: set[str] = set() + validated: list[tuple[zipfile.ZipInfo, Path]] = [] + for m in members: + if stat.S_ISLNK(m.external_attr >> 16): + raise ValueError(f"symlink archive entry not allowed: {m.filename!r}") + target = (dest / m.filename).resolve() + if target == dest or not target.is_relative_to(dest): + raise ValueError(f"unsafe archive entry: {m.filename!r}") + target_key = os.path.normcase(str(target)) + if target_key in targets: + raise ValueError(f"duplicate archive entry: {m.filename!r}") + targets.add(target_key) + if not m.is_dir(): + file_targets.add(target_key) + validated.append((m, target)) + if m.file_size > MAX_ARCHIVE_MEMBER_SIZE: + raise ValueError(f"archive entry is too large: {m.filename!r}") + total_size += m.file_size + if total_size > MAX_ARCHIVE_TOTAL_SIZE: + raise ValueError("archive expands beyond the allowed total size") + if m.file_size and ( + m.compress_size == 0 + or m.file_size > m.compress_size * MAX_ARCHIVE_COMPRESSION_RATIO + ): + raise ValueError(f"archive entry has an unsafe compression ratio: {m.filename!r}") + + for m, target in validated: + for parent in target.parents: + if parent == dest: + break + if os.path.normcase(str(parent)) in file_targets: + raise ValueError(f"archive file entry conflicts with child path: {m.filename!r}") + + for m, _ in validated: + zf.extract(m, dest) + + +def rezip(src_dir: Path, out_path: Path) -> None: + files = sorted(p for p in src_dir.rglob("*") if p.is_file()) + ct = src_dir / "[Content_Types].xml" + fd, tmp_name = tempfile.mkstemp( + prefix=out_path.name + ".", suffix=".tmp", dir=out_path.parent + ) + tmp_out = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as fh: + with zipfile.ZipFile(fh, "w", zipfile.ZIP_DEFLATED) as zf: + if ct.exists(): + zf.write(ct, ct.relative_to(src_dir), compress_type=zipfile.ZIP_STORED) + for f in files: + if f == ct: + continue + zf.write(f, f.relative_to(src_dir)) + if out_path.exists(): + mode = out_path.stat().st_mode & 0o777 + else: + umask = os.umask(0) + os.umask(umask) + mode = 0o666 & ~umask + os.chmod(tmp_out, mode) + os.replace(tmp_out, out_path) + finally: + if tmp_out.exists(): + tmp_out.unlink() diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/merge_runs.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/merge_runs.py deleted file mode 100644 index ad7c25eec0..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/merge_runs.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Merge adjacent runs with identical formatting in DOCX. - -Merges adjacent elements that have identical properties. -Works on runs in paragraphs and inside tracked changes (, ). - -Also: -- Removes rsid attributes from runs (revision metadata that doesn't affect rendering) -- Removes proofErr elements (spell/grammar markers that block merging) -""" - -from pathlib import Path - -import defusedxml.minidom - - -def merge_runs(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - _remove_elements(root, "proofErr") - _strip_run_rsid_attrs(root) - - containers = {run.parentNode for run in _find_elements(root, "r")} - - merge_count = 0 - for container in containers: - merge_count += _merge_runs_in(container) - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Merged {merge_count} runs" - - except Exception as e: - return 0, f"Error: {e}" - - - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def _get_child(parent, tag: str): - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - return child - return None - - -def _get_children(parent, tag: str) -> list: - results = [] - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(child) - return results - - -def _is_adjacent(elem1, elem2) -> bool: - node = elem1.nextSibling - while node: - if node == elem2: - return True - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - return False - - - - -def _remove_elements(root, tag: str): - for elem in _find_elements(root, tag): - if elem.parentNode: - elem.parentNode.removeChild(elem) - - -def _strip_run_rsid_attrs(root): - for run in _find_elements(root, "r"): - for attr in list(run.attributes.values()): - if "rsid" in attr.name.lower(): - run.removeAttribute(attr.name) - - - - -def _merge_runs_in(container) -> int: - merge_count = 0 - run = _first_child_run(container) - - while run: - while True: - next_elem = _next_element_sibling(run) - if next_elem and _is_run(next_elem) and _can_merge(run, next_elem): - _merge_run_content(run, next_elem) - container.removeChild(next_elem) - merge_count += 1 - else: - break - - _consolidate_text(run) - run = _next_sibling_run(run) - - return merge_count - - -def _first_child_run(container): - for child in container.childNodes: - if child.nodeType == child.ELEMENT_NODE and _is_run(child): - return child - return None - - -def _next_element_sibling(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - return sibling - sibling = sibling.nextSibling - return None - - -def _next_sibling_run(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - if _is_run(sibling): - return sibling - sibling = sibling.nextSibling - return None - - -def _is_run(node) -> bool: - name = node.localName or node.tagName - return name == "r" or name.endswith(":r") - - -def _can_merge(run1, run2) -> bool: - rpr1 = _get_child(run1, "rPr") - rpr2 = _get_child(run2, "rPr") - - if (rpr1 is None) != (rpr2 is None): - return False - if rpr1 is None: - return True - return rpr1.toxml() == rpr2.toxml() - - -def _merge_run_content(target, source): - for child in list(source.childNodes): - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name != "rPr" and not name.endswith(":rPr"): - target.appendChild(child) - - -def _consolidate_text(run): - t_elements = _get_children(run, "t") - - for i in range(len(t_elements) - 1, 0, -1): - curr, prev = t_elements[i], t_elements[i - 1] - - if _is_adjacent(prev, curr): - prev_text = prev.firstChild.data if prev.firstChild else "" - curr_text = curr.firstChild.data if curr.firstChild else "" - merged = prev_text + curr_text - - if prev.firstChild: - prev.firstChild.data = merged - else: - prev.appendChild(run.ownerDocument.createTextNode(merged)) - - if merged.startswith(" ") or merged.endswith(" "): - prev.setAttribute("xml:space", "preserve") - elif prev.hasAttribute("xml:space"): - prev.removeAttribute("xml:space") - - run.removeChild(curr) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_chart.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_chart.py new file mode 100644 index 0000000000..209cb7c58b --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_chart.py @@ -0,0 +1,170 @@ +"""Find chart XML that PowerPoint refuses but the schema accepts. + +Detection only: for either fault more than one repair is valid, and only the +author knows which was meant. +""" + + +from __future__ import annotations + +import re +from typing import Mapping + +from . import part_text + + +_CHART_PART_RE = re.compile(r"ppt/charts/chart\d+\.xml") + +_GROUPING_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") +_DLBL_POS_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") + +def _strip_ext_lst(text: str) -> str: + out, cursor = [], 0 + for lo, hi in _ext_lst_spans(text): + out.append(text[cursor:lo]) + cursor = hi + out.append(text[cursor:]) + return "".join(out) + +_BAR_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +STACKED_GROUPINGS = frozenset({"stacked", "percentStacked"}) +ILLEGAL_ON_STACKED = frozenset({"outEnd"}) +LEGAL_ON_STACKED = ("ctr", "inEnd", "inBase") + + +def _check_stacked_label_positions(part: str, xml: str) -> list[str]: + problems: list[str] = [] + for match in _BAR_GROUP_RE.finditer(xml): + block = _strip_ext_lst(match.group(0)) + group = match.group(1) + + grouping = _GROUPING_RE.search(block) + if grouping is None or grouping.group(1) not in STACKED_GROUPINGS: + continue + + bad = [p for p in _DLBL_POS_RE.findall(block) if p in ILLEGAL_ON_STACKED] + for pos in sorted(set(bad)): + problems.append( + f'{part}: {bad.count(pos)} data label(s) use dLblPos="{pos}" on a ' + f"{grouping.group(1)} {group}; PowerPoint allows only " + f"{', '.join(LEGAL_ON_STACKED)} there" + ) + return problems + + + +_ANY_CHART_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +_AXID_RE = re.compile( + r"""\s*]*?\bval=["'](-?\d+)["']\s*(?:/>|>\s*)""" +) + +_AXIS_DECL_RE = re.compile( + r"""]*(?\s*]*?\bval=["'](-?\d+)["']""" +) + +AXID_LIMIT = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 3, "line3DChart": 3, "area3DChart": 3, + "surfaceChart": 3, "surface3DChart": 3, +} + +AXID_MINIMUM = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 2, "area3DChart": 2, "surfaceChart": 2, + "line3DChart": 3, "surface3DChart": 3, +} + + +def _declared_axes(xml: str) -> dict[str, list[str]]: + axes: dict[str, list[str]] = {} + for kind, axid in _AXIS_DECL_RE.findall(xml): + axes.setdefault(kind, []).append(axid) + return axes + + +def _canonical_ids(axes: dict[str, list[str]], limit: int) -> list[str] | None: + category = axes.get("catAx", []) + axes.get("dateAx", []) + value = axes.get("valAx", []) + series = axes.get("serAx", []) + if len(category) != 1 or len(value) != 1 or len(series) > 1: + return None + ids = [category[0], value[0]] + if limit >= 3 and series: + ids.append(series[0]) + return ids + + +def _undeclared_axes(kind: str, block: str, axes: dict[str, list[str]]) -> list[str] | None: + if kind not in AXID_LIMIT: + return None + ids = _AXID_RE.findall(block) + declared = {i for group in axes.values() for i in group} + if len([i for i in ids if i in declared]) >= 2: + return None + return ids + + +def _check_chart_axis_references(part: str, xml: str) -> list[str]: + axes = _declared_axes(xml) + problems: list[str] = [] + declared = {i for group in axes.values() for i in group} + for match in _ANY_CHART_GROUP_RE.finditer(xml): + kind, block = match.group(1), match.group(0) + ids = _undeclared_axes(kind, block, axes) + if ids is None: + continue + if not ids: + problems.append( + f"{part}: declares no this part can resolve; a chart " + f"group needs {AXID_MINIMUM[kind]}, and PowerPoint discards one with fewer" + ) + continue + dead = [i for i in ids if i not in declared] + canonical = _canonical_ids(axes, AXID_LIMIT[kind]) + if canonical is not None and len(canonical) >= AXID_MINIMUM[kind]: + hint = f"Fix: point them at the axes this part declares ({', '.join(canonical)})" + else: + hint = ("Fix: the part declares several axes of a kind -- declare the " + "secondary axes the series expects, or drop them") + detail = (f"of which {', '.join(dead)} name no declared axis" + if dead else f"only {len(ids)} of which this part declares") + problems.append( + f"{part}: references axId {', '.join(ids)}, {detail}, " + f"leaving fewer than two live axes; PowerPoint discards the chart. {hint}" + ) + return problems + + +def _ext_lst_spans(text: str) -> list[tuple[int, int]]: + spans: list[tuple[int, int]] = [] + depth = 0 + start = 0 + for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text): + closing, self_closing = match.group(1), match.group(2) + if self_closing: + continue + if closing: + depth -= 1 + if depth == 0: + spans.append((start, match.end())) + else: + if depth == 0: + start = match.start() + depth += 1 + return spans + + +CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references) + + +def find_chart_problems(files: Mapping[str, bytes]) -> list[str]: + problems: list[str] = [] + for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)): + xml = part_text(files[part]) + for check in CHART_CHECKS: + problems.extend(check(part, xml)) + return problems diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_slide.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_slide.py new file mode 100644 index 0000000000..22f9aee0ff --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_slide.py @@ -0,0 +1,60 @@ +"""Pick the slide-XML schema errors PowerPoint refuses the file over. + +A denylist over lxml's messages, so an unrecognised error class is a miss rather +than a false alarm. +""" + + +from __future__ import annotations + +import re + +SLIDE_PART_RE = re.compile( + r"ppt/(slides|slideLayouts|slideMasters|notesSlides|notesMasters|handoutMasters)" + r"/[^/]+\.xml" +) + +FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"\}tableStyleId': This element is not expected"), + "two in one (the schema allows one)", + ), + ( + re.compile(r"\}srgbClr', attribute 'val'"), + "a colour that is not six hex digits", + ), + ( + re.compile(r"\}txBody': Missing child element"), + "a with no children", + ), + ( + re.compile(r"\}miter', attribute 'lim'"), + 'a line join with lim="NaN"', + ), + ( + re.compile(r"\}uLnTx': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}overrideClrMapping': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}nvGrpSpPr': Missing child element"), + "a with no children", + ), +) + + +def is_schema_verdict(error: str) -> bool: + return error.startswith("Element ") + + +def fatal_slide_errors(errors: set[str]) -> list[str]: + out = [] + for error in sorted(errors): + for pattern, meaning in FATAL_SLIDE_ERRORS: + if pattern.search(error): + out.append(f"{meaning}: {error}") + break + return out diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_theme.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_theme.py new file mode 100644 index 0000000000..5ef4c3e835 --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/pptx_theme.py @@ -0,0 +1,114 @@ +"""Find masters sharing a theme part in the way PowerPoint refuses to open. + +Reports only; the fix is to move back to directly after + in ppt/presentation.xml. +""" + + +from __future__ import annotations + +import posixpath +import re +from typing import Mapping + +from . import part_text + +THEME_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" + +_MASTER_RE = re.compile( + r"^ppt/(?PslideMasters|notesMasters|handoutMasters)/" + r"(?:slide|notes|handout)Master(?P\d+)\.xml$" +) +_GROUP_ORDER = {"slideMasters": 0, "notesMasters": 1, "handoutMasters": 2} + +_RELATIONSHIP_RE = re.compile( + r"]*?(?:/>|>.*?)", re.DOTALL +) + + +def _sort_key(name: str) -> tuple[int, int]: + m = _MASTER_RE.match(name) + assert m is not None + return (_GROUP_ORDER[m.group("group")], int(m.group("num"))) + + +def _rels_path(part: str) -> str: + directory, base = posixpath.split(part) + return f"{directory}/_rels/{base}.rels" + + +def _resolve(rels_path: str, target: str) -> str: + if target.startswith("/"): + return target.lstrip("/") + part_dir = posixpath.dirname(posixpath.dirname(rels_path)) + return posixpath.normpath(posixpath.join(part_dir, target)) + + +def _theme_rel(files: Mapping[str, bytes], master: str): + rels_path = _rels_path(master) + rels = files.get(rels_path) + if rels is None: + return None + for element in _RELATIONSHIP_RE.findall(part_text(rels)): + if f'Type="{THEME_REL_TYPE}"' not in element: + continue + target = re.search(r'\bTarget="([^"]+)"', element) + if target is None: + continue + return rels_path, element, _resolve(rels_path, target.group(1)) + return None + + +def _masters(files: Mapping[str, bytes]) -> list[str]: + return sorted((n for n in files if _MASTER_RE.match(n)), key=_sort_key) + + +_PRESENTATION = "ppt/presentation.xml" +_NOTES_MASTERS = "ppt/notesMasters/" +_IGNORABLE_RE = re.compile(r"|<\?.*?\?>", re.DOTALL) +_AFTER_SLDIDLST_RE = re.compile( + r"]*/>|[^>]*>.*?)\s*(<[^>\s/]+)", re.DOTALL +) + + +def _notes_master_share_is_inert(files: Mapping[str, bytes]) -> bool: + data = files.get(_PRESENTATION) + if data is None: + return False + match = _AFTER_SLDIDLST_RE.search(_IGNORABLE_RE.sub("", part_text(data))) + return match is not None and match.group(1) == " bool: + return inert_notes and master.startswith(_NOTES_MASTERS) + + +def find_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + ] + + +def live_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + inert_notes = _notes_master_share_is_inert(files) + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + if not _is_inert(master, inert_notes) + ] diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/simplify_redlines.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/simplify_redlines.py deleted file mode 100644 index db963bb998..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/helpers/simplify_redlines.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Simplify tracked changes by merging adjacent w:ins or w:del elements. - -Merges adjacent elements from the same author into a single element. -Same for elements. This makes heavily-redlined documents easier to -work with by reducing the number of tracked change wrappers. - -Rules: -- Only merges w:ins with w:ins, w:del with w:del (same element type) -- Only merges if same author (ignores timestamp differences) -- Only merges if truly adjacent (only whitespace between them) -""" - -import xml.etree.ElementTree as ET -import zipfile -from pathlib import Path - -import defusedxml.minidom - -WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" - - -def simplify_redlines(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - merge_count = 0 - - containers = _find_elements(root, "p") + _find_elements(root, "tc") - - for container in containers: - merge_count += _merge_tracked_changes_in(container, "ins") - merge_count += _merge_tracked_changes_in(container, "del") - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Simplified {merge_count} tracked changes" - - except Exception as e: - return 0, f"Error: {e}" - - -def _merge_tracked_changes_in(container, tag: str) -> int: - merge_count = 0 - - tracked = [ - child - for child in container.childNodes - if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag) - ] - - if len(tracked) < 2: - return 0 - - i = 0 - while i < len(tracked) - 1: - curr = tracked[i] - next_elem = tracked[i + 1] - - if _can_merge_tracked(curr, next_elem): - _merge_tracked_content(curr, next_elem) - container.removeChild(next_elem) - tracked.pop(i + 1) - merge_count += 1 - else: - i += 1 - - return merge_count - - -def _is_element(node, tag: str) -> bool: - name = node.localName or node.tagName - return name == tag or name.endswith(f":{tag}") - - -def _get_author(elem) -> str: - author = elem.getAttribute("w:author") - if not author: - for attr in elem.attributes.values(): - if attr.localName == "author" or attr.name.endswith(":author"): - return attr.value - return author - - -def _can_merge_tracked(elem1, elem2) -> bool: - if _get_author(elem1) != _get_author(elem2): - return False - - node = elem1.nextSibling - while node and node != elem2: - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - - return True - - -def _merge_tracked_content(target, source): - while source.firstChild: - child = source.firstChild - source.removeChild(child) - target.appendChild(child) - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def get_tracked_change_authors(doc_xml_path: Path) -> dict[str, int]: - if not doc_xml_path.exists(): - return {} - - try: - tree = ET.parse(doc_xml_path) - root = tree.getroot() - except ET.ParseError: - return {} - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - - return authors - - -def _get_authors_from_docx(docx_path: Path) -> dict[str, int]: - try: - with zipfile.ZipFile(docx_path, "r") as zf: - if "word/document.xml" not in zf.namelist(): - return {} - with zf.open("word/document.xml") as f: - tree = ET.parse(f) - root = tree.getroot() - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - return authors - except (zipfile.BadZipFile, ET.ParseError): - return {} - - -def infer_author(modified_dir: Path, original_docx: Path, default: str = "Claude") -> str: - modified_xml = modified_dir / "word" / "document.xml" - modified_authors = get_tracked_change_authors(modified_xml) - - if not modified_authors: - return default - - original_authors = _get_authors_from_docx(original_docx) - - new_changes: dict[str, int] = {} - for author, count in modified_authors.items(): - original_count = original_authors.get(author, 0) - diff = count - original_count - if diff > 0: - new_changes[author] = diff - - if not new_changes: - return default - - if len(new_changes) == 1: - return next(iter(new_changes)) - - raise ValueError( - f"Multiple authors added new changes: {new_changes}. " - "Cannot infer which author to validate." - ) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/pack.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/pack.py deleted file mode 100755 index db29ed8b1c..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/pack.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Pack a directory into a DOCX, PPTX, or XLSX file. - -Validates with auto-repair, condenses XML formatting, and creates the Office file. - -Usage: - python pack.py [--original ] [--validate true|false] - -Examples: - python pack.py unpacked/ output.docx --original input.docx - python pack.py unpacked/ output.pptx --validate false -""" - -import argparse -import sys -import shutil -import tempfile -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator - -def pack( - input_directory: str, - output_file: str, - original_file: str | None = None, - validate: bool = True, - infer_author_func=None, -) -> tuple[None, str]: - input_dir = Path(input_directory) - output_path = Path(output_file) - suffix = output_path.suffix.lower() - - if not input_dir.is_dir(): - return None, f"Error: {input_dir} is not a directory" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {output_file} must be a .docx, .pptx, or .xlsx file" - - if validate and original_file: - original_path = Path(original_file) - if original_path.exists(): - success, output = _run_validation( - input_dir, original_path, suffix, infer_author_func - ) - if output: - print(output) - if not success: - return None, f"Error: Validation failed for {input_dir}" - - with tempfile.TemporaryDirectory() as temp_dir: - temp_content_dir = Path(temp_dir) / "content" - shutil.copytree(input_dir, temp_content_dir) - - for pattern in ["*.xml", "*.rels"]: - for xml_file in temp_content_dir.rglob(pattern): - _condense_xml(xml_file) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf: - for f in temp_content_dir.rglob("*"): - if f.is_file(): - zf.write(f, f.relative_to(temp_content_dir)) - - return None, f"Successfully packed {input_dir} to {output_file}" - - -def _run_validation( - unpacked_dir: Path, - original_file: Path, - suffix: str, - infer_author_func=None, -) -> tuple[bool, str | None]: - output_lines = [] - validators = [] - - if suffix == ".docx": - author = "Claude" - if infer_author_func: - try: - author = infer_author_func(unpacked_dir, original_file) - except ValueError as e: - print(f"Warning: {e} Using default author 'Claude'.", file=sys.stderr) - - validators = [ - DOCXSchemaValidator(unpacked_dir, original_file), - RedliningValidator(unpacked_dir, original_file, author=author), - ] - elif suffix == ".pptx": - validators = [PPTXSchemaValidator(unpacked_dir, original_file)] - - if not validators: - return True, None - - total_repairs = sum(v.repair() for v in validators) - if total_repairs: - output_lines.append(f"Auto-repaired {total_repairs} issue(s)") - - success = all(v.validate() for v in validators) - - if success: - output_lines.append("All validations PASSED!") - - return success, "\n".join(output_lines) if output_lines else None - - -def _condense_xml(xml_file: Path) -> None: - try: - with open(xml_file, encoding="utf-8") as f: - dom = defusedxml.minidom.parse(f) - - for element in dom.getElementsByTagName("*"): - if element.tagName.endswith(":t"): - continue - - for child in list(element.childNodes): - if ( - child.nodeType == child.TEXT_NODE - and child.nodeValue - and child.nodeValue.strip() == "" - ) or child.nodeType == child.COMMENT_NODE: - element.removeChild(child) - - xml_file.write_bytes(dom.toxml(encoding="UTF-8")) - except Exception as e: - print(f"ERROR: Failed to parse {xml_file.name}: {e}", file=sys.stderr) - raise - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Pack a directory into a DOCX, PPTX, or XLSX file" - ) - parser.add_argument("input_directory", help="Unpacked Office document directory") - parser.add_argument("output_file", help="Output Office file (.docx/.pptx/.xlsx)") - parser.add_argument( - "--original", - help="Original file for validation comparison", - ) - parser.add_argument( - "--validate", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Run validation with auto-repair (default: true)", - ) - args = parser.parse_args() - - _, message = pack( - args.input_directory, - args.output_file, - original_file=args.original, - validate=args.validate, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/soffice.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/soffice.py index c7f7e3289f..0b4c99deca 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/soffice.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/soffice.py @@ -4,20 +4,23 @@ at runtime and applies an LD_PRELOAD shim if needed. Usage: - from office.soffice import run_soffice, get_soffice_env + from office.soffice import run_soffice - # Option 1 – run soffice directly result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"]) - # Option 2 – get env dict for your own subprocess calls - env = get_soffice_env() - subprocess.run(["soffice", ...], env=env) +Call soffice through run_soffice, not through subprocess with get_soffice_env(): +the env dict carries the shim but names no user profile, and a non-root sandbox +cannot bootstrap the default one -- soffice aborts with "User installation could +not be completed" and converts nothing. get_soffice_env() stays public for the +callers that build their own argv (they must pass -env:UserInstallation too). """ +import contextlib import os import socket import subprocess import tempfile +from collections.abc import Iterable from pathlib import Path @@ -32,9 +35,15 @@ def get_soffice_env() -> dict: return env -def run_soffice(args: list[str], **kwargs) -> subprocess.CompletedProcess: - env = get_soffice_env() - return subprocess.run(["soffice"] + args, env=env, **kwargs) +def run_soffice(args: Iterable[str], **kwargs) -> subprocess.CompletedProcess: + args = list(args) + with contextlib.ExitStack() as stack: + if not any(str(a).startswith("-env:UserInstallation") for a in args): + profile = stack.enter_context( + tempfile.TemporaryDirectory(prefix="lo_profile_", ignore_cleanup_errors=True) + ) + args = [f"-env:UserInstallation={Path(profile).as_uri()}"] + args + return subprocess.run(["soffice"] + args, env=get_soffice_env(), **kwargs) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/unpack.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/unpack.py deleted file mode 100755 index 00152533ac..0000000000 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/unpack.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Unpack Office files (DOCX, PPTX, XLSX) for editing. - -Extracts the ZIP archive, pretty-prints XML files, and optionally: -- Merges adjacent runs with identical formatting (DOCX only) -- Simplifies adjacent tracked changes from same author (DOCX only) - -Usage: - python unpack.py [options] - -Examples: - python unpack.py document.docx unpacked/ - python unpack.py presentation.pptx unpacked/ - python unpack.py document.docx unpacked/ --merge-runs false -""" - -import argparse -import sys -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from helpers.merge_runs import merge_runs as do_merge_runs -from helpers.simplify_redlines import simplify_redlines as do_simplify_redlines - -SMART_QUOTE_REPLACEMENTS = { - "\u201c": "“", - "\u201d": "”", - "\u2018": "‘", - "\u2019": "’", -} - - -def unpack( - input_file: str, - output_directory: str, - merge_runs: bool = True, - simplify_redlines: bool = True, -) -> tuple[None, str]: - input_path = Path(input_file) - output_path = Path(output_directory) - suffix = input_path.suffix.lower() - - if not input_path.exists(): - return None, f"Error: {input_file} does not exist" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {input_file} must be a .docx, .pptx, or .xlsx file" - - try: - output_path.mkdir(parents=True, exist_ok=True) - - with zipfile.ZipFile(input_path, "r") as zf: - zf.extractall(output_path) - - xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels")) - for xml_file in xml_files: - _pretty_print_xml(xml_file) - - message = f"Unpacked {input_file} ({len(xml_files)} XML files)" - - if suffix == ".docx": - if simplify_redlines: - simplify_count, _ = do_simplify_redlines(str(output_path)) - message += f", simplified {simplify_count} tracked changes" - - if merge_runs: - merge_count, _ = do_merge_runs(str(output_path)) - message += f", merged {merge_count} runs" - - for xml_file in xml_files: - _escape_smart_quotes(xml_file) - - return None, message - - except zipfile.BadZipFile: - return None, f"Error: {input_file} is not a valid Office file" - except Exception as e: - return None, f"Error unpacking: {e}" - - -def _pretty_print_xml(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - dom = defusedxml.minidom.parseString(content) - xml_file.write_bytes(dom.toprettyxml(indent=" ", encoding="utf-8")) - except Exception: - pass - - -def _escape_smart_quotes(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - for char, entity in SMART_QUOTE_REPLACEMENTS.items(): - content = content.replace(char, entity) - xml_file.write_text(content, encoding="utf-8") - except Exception: - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Unpack an Office file (DOCX, PPTX, XLSX) for editing" - ) - parser.add_argument("input_file", help="Office file to unpack") - parser.add_argument("output_directory", help="Output directory") - parser.add_argument( - "--merge-runs", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent runs with identical formatting (DOCX only, default: true)", - ) - parser.add_argument( - "--simplify-redlines", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent tracked changes from same author (DOCX only, default: true)", - ) - args = parser.parse_args() - - _, message = unpack( - args.input_file, - args.output_directory, - merge_runs=args.merge_runs, - simplify_redlines=args.simplify_redlines, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validate.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validate.py index 03b01f6e3b..8fbd2f71ca 100755 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validate.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validate.py @@ -6,7 +6,7 @@ The first argument can be either: - An unpacked directory containing the Office document XML files -- A packed Office file (.docx/.pptx/.xlsx) which will be unpacked to a temp directory +- A packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx template) which will be unpacked to a temp directory Auto-repair fixes: - paraId/durableId values that exceed OOXML limits @@ -19,20 +19,43 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import OOXML_FAMILY, rezip, safe_extract from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator +WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + +def _fail(message: str): + print(f"Error: {message}", file=sys.stderr) + sys.exit(2) + + +def _has_tracked_changes(unpacked_dir: Path) -> bool: + document = unpacked_dir / "word" / "document.xml" + if not document.is_file(): + return False + try: + root = ET.parse(document).getroot() + except (ET.ParseError, DefusedXmlException): + return False + tracked = {f"{{{WORD_NS}}}ins", f"{{{WORD_NS}}}del"} + return any(elem.tag in tracked for elem in root.iter()) + def main(): parser = argparse.ArgumentParser(description="Validate Office document XML files") parser.add_argument( "path", - help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx)", + help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx)", ) parser.add_argument( "--original", required=False, default=None, - help="Path to original file (.docx/.pptx/.xlsx). If omitted, all XSD errors are reported and redlining validation is skipped.", + help="Path to original file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx). If omitted, all XSD errors are reported and redlining validation is skipped.", ) parser.add_argument( "-v", @@ -43,63 +66,102 @@ def main(): parser.add_argument( "--auto-repair", action="store_true", - help="Automatically repair common issues (hex IDs, whitespace preservation)", + help="Automatically repair common issues (hex IDs, whitespace preservation). " + "Modifies the input in place: repairs to a packed file are written back to it.", ) parser.add_argument( "--author", - default="Claude", - help="Author name for redlining validation (default: Claude)", + default=None, + help="The name you are redlining under. Passing it turns on the " + "tracked-change check: any text differing from --original without a " + "/ recording it is reported. Untracked edits carry no " + "author, so the check covers them whoever made them — the name marks " + "the run as redlining work and is not used to filter. Requires " + "--original; docx only.", ) args = parser.parse_args() + if args.author is not None and not args.original: + _fail("--author requires --original") + path = Path(args.path) - assert path.exists(), f"Error: {path} does not exist" + if not path.exists(): + _fail(f"{path} does not exist") original_file = None if args.original: original_file = Path(args.original) - assert original_file.is_file(), f"Error: {original_file} is not a file" - assert original_file.suffix.lower() in [".docx", ".pptx", ".xlsx"], ( - f"Error: {original_file} must be a .docx, .pptx, or .xlsx file" + if not original_file.is_file(): + _fail(f"{original_file} is not a file") + if original_file.suffix.lower() not in OOXML_FAMILY: + _fail(f"{original_file} must be one of: {', '.join(sorted(OOXML_FAMILY))}") + + family = OOXML_FAMILY.get((original_file or path).suffix.lower()) + if family is None: + _fail( + f"Cannot determine file type from {path}. Use --original or provide one of: {', '.join(sorted(OOXML_FAMILY))}." ) - file_extension = (original_file or path).suffix.lower() - assert file_extension in [".docx", ".pptx", ".xlsx"], ( - f"Error: Cannot determine file type from {path}. Use --original or provide a .docx/.pptx/.xlsx file." - ) - - if path.is_file() and path.suffix.lower() in [".docx", ".pptx", ".xlsx"]: - temp_dir = tempfile.mkdtemp() - with zipfile.ZipFile(path, "r") as zf: - zf.extractall(temp_dir) - unpacked_dir = Path(temp_dir) + if args.author is not None and family != "docx": + _fail(f"--author only applies to docx files, not {family}") + + packed_file = None + temp_dir_ctx = None + if path.is_file() and path.suffix.lower() in OOXML_FAMILY: + packed_file = path + temp_dir_ctx = tempfile.TemporaryDirectory() + unpacked_dir = Path(temp_dir_ctx.name) + try: + with zipfile.ZipFile(path, "r") as zf: + safe_extract(zf, unpacked_dir) + except (zipfile.BadZipFile, ValueError, OSError) as e: + _fail(f"cannot unpack {path}: {e}") else: - assert path.is_dir(), f"Error: {path} is not a directory or Office file" + if not path.is_dir(): + _fail(f"{path} is not a directory or Office file") unpacked_dir = path - match file_extension: - case ".docx": + match family: + case "docx": validators = [ DOCXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] - if original_file: + if args.author is not None: validators.append( - RedliningValidator(unpacked_dir, original_file, verbose=args.verbose, author=args.author) + RedliningValidator(unpacked_dir, original_file, verbose=args.verbose) + ) + elif original_file and _has_tracked_changes(unpacked_dir): + print( + "Note: this document has tracked changes; they were not " + "checked against the original (pass --author to check)." ) - case ".pptx": + case "pptx": validators = [ PPTXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] + case "xlsx": + exts = ", ".join(k for k, v in sorted(OOXML_FAMILY.items()) if v == "xlsx") + print( + f"No XSD schema validation is performed for xlsx-family files ({exts}). " + "For formula-error checking, use scripts/recalc.py instead." + ) + sys.exit(0) case _: - print(f"Error: Validation not supported for file type {file_extension}") + print(f"Error: Validation not supported for file type {family}") sys.exit(1) if args.auto_repair: total_repairs = sum(v.repair() for v in validators) if total_repairs: print(f"Auto-repaired {total_repairs} issue(s)") + if packed_file is not None: + rezip(unpacked_dir, packed_file) + print(f"Wrote repaired file to {packed_file}") + + success = all([v.validate() for v in validators]) - success = all(v.validate() for v in validators) + if temp_dir_ctx is not None: + temp_dir_ctx.cleanup() if success: print("All validations PASSED!") diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/base.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/base.py index db4a06a229..19d52a7fe0 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/base.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/base.py @@ -6,8 +6,20 @@ from pathlib import Path import defusedxml.minidom +from functools import lru_cache + import lxml.etree +from helpers import safe_extract + + +@lru_cache(maxsize=None) +def _load_schema(schema_path: str): + with open(schema_path, "rb") as xsd_file: + xsd_doc = lxml.etree.parse( + xsd_file, parser=lxml.etree.XMLParser(), base_url=schema_path + ) + return lxml.etree.XMLSchema(xsd_doc) class BaseSchemaValidator: @@ -119,21 +131,28 @@ def repair_whitespace_preservation(self) -> int: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) - modified = False + pending = [] for elem in dom.getElementsByTagName("*"): - if elem.tagName.endswith(":t") and elem.firstChild: - text = elem.firstChild.nodeValue - if text and (text.startswith((' ', '\t')) or text.endswith((' ', '\t'))): + local_name = elem.tagName.rsplit(":", 1)[-1] + if local_name in ("t", "delText", "instrText", "delInstrText"): + text = "".join( + child.data + for child in elem.childNodes + if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE) + ) + ws = (" ", "\t", "\n", "\r") + if text and (text.startswith(ws) or text.endswith(ws)): if elem.getAttribute("xml:space") != "preserve": elem.setAttribute("xml:space", "preserve") text_preview = repr(text[:30]) + "..." if len(text) > 30 else repr(text) - print(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - repairs += 1 - modified = True + pending.append(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - if modified: + if pending: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass @@ -212,6 +231,8 @@ def validate_unique_ids(self): elem.getparent().remove(elem) for elem in root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue tag = ( elem.tag.split("}")[-1].lower() if "}" in elem.tag @@ -326,6 +347,8 @@ def validate_file_references(self): namespaces={"ns": self.PACKAGE_RELATIONSHIPS_NAMESPACE}, ): target = rel.get("Target") + if rel.get("TargetMode") == "External": + continue if target and not target.startswith( ("http", "mailto:") ): @@ -423,6 +446,8 @@ def validate_all_relationship_ids(self): r_ns = self.OFFICE_RELATIONSHIPS_NAMESPACE rid_attrs_to_check = ["id", "embed", "link"] for elem in xml_root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue for attr_name in rid_attrs_to_check: rid_attr = elem.get(f"{{{r_ns}}}{attr_name}") if not rid_attr: @@ -747,18 +772,16 @@ def _preprocess_for_mc_ignorable(self, xml_doc): return xml_doc - def _validate_single_file_xsd(self, xml_file, base_path): - schema_path = self._get_schema_path(xml_file) + def _preprocess_for_schema(self, xml_doc, relative_path): + return xml_doc + + def _validate_single_file_xsd(self, xml_file, base_path, schema_path=None): + schema_path = schema_path or self._get_schema_path(xml_file) if not schema_path: return None, None try: - with open(schema_path, "rb") as xsd_file: - parser = lxml.etree.XMLParser() - xsd_doc = lxml.etree.parse( - xsd_file, parser=parser, base_url=str(schema_path) - ) - schema = lxml.etree.XMLSchema(xsd_doc) + schema = _load_schema(str(schema_path)) with open(xml_file, "r") as f: xml_doc = lxml.etree.parse(f) @@ -773,6 +796,8 @@ def _validate_single_file_xsd(self, xml_file, base_path): ): xml_doc = self._clean_ignorable_namespaces(xml_doc) + xml_doc = self._preprocess_for_schema(xml_doc, relative_path) + if schema.validate(xml_doc): return True, set() else: @@ -784,7 +809,7 @@ def _validate_single_file_xsd(self, xml_file, base_path): except Exception as e: return False, {str(e)} - def _get_original_file_errors(self, xml_file): + def _get_original_file_errors(self, xml_file, schema_path=None): if self.original_file is None: return set() @@ -798,8 +823,11 @@ def _get_original_file_errors(self, xml_file): with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - with zipfile.ZipFile(self.original_file, "r") as zip_ref: - zip_ref.extractall(temp_path) + try: + with zipfile.ZipFile(self.original_file, "r") as zip_ref: + safe_extract(zip_ref, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() original_xml_file = temp_path / relative_path @@ -807,7 +835,7 @@ def _get_original_file_errors(self, xml_file): return set() is_valid, errors = self._validate_single_file_xsd( - original_xml_file, temp_path + original_xml_file, temp_path, schema_path=schema_path ) return errors if errors else set() diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/docx.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/docx.py index fec405e694..0d18b6979a 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/docx.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/docx.py @@ -6,10 +6,13 @@ import re import tempfile import zipfile +from pathlib import Path import defusedxml.minidom import lxml.etree +from helpers import safe_extract + from .base import BaseSchemaValidator @@ -186,7 +189,7 @@ def count_paragraphs_in_original(self): try: with tempfile.TemporaryDirectory() as temp_dir: with zipfile.ZipFile(original, "r") as zip_ref: - zip_ref.extractall(temp_dir) + safe_extract(zip_ref, Path(temp_dir)) doc_xml_path = temp_dir + "/word/document.xml" root = lxml.etree.parse(doc_xml_path).getroot() @@ -241,9 +244,12 @@ def validate_insertions(self): return True def compare_paragraph_counts(self): - original_count = self.count_paragraphs_in_original() new_count = self.count_paragraphs_in_unpacked() + if self.original_file is None: + print(f"\nParagraphs: {new_count}") + return + original_count = self.count_paragraphs_in_original() diff = new_count - original_count diff_str = f"+{diff}" if diff > 0 else str(diff) print(f"\nParagraphs: {original_count} → {new_count} ({diff_str})") @@ -260,9 +266,15 @@ def validate_id_constraints(self): try: for elem in lxml.etree.parse(str(xml_file)).iter(): if val := elem.get(para_id_attr): - if self._parse_id_value(val, base=16) >= 0x80000000: + try: + if self._parse_id_value(val, base=16) >= 0x80000000: + errors.append( + f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + ) + except ValueError: errors.append( - f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + f" {xml_file.name}:{elem.sourceline}: " + f"paraId={val} is not valid hex" ) if val := elem.get(durable_id_attr): @@ -279,13 +291,19 @@ def validate_id_constraints(self): f"durableId={val} must be decimal in numbering.xml" ) else: - if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + try: + if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + errors.append( + f" {xml_file.name}:{elem.sourceline}: " + f"durableId={val} >= 0x7FFFFFFF" + ) + except ValueError: errors.append( f" {xml_file.name}:{elem.sourceline}: " - f"durableId={val} >= 0x7FFFFFFF" + f"durableId={val} is not valid hex" ) - except Exception: - pass + except lxml.etree.XMLSyntaxError: + continue if errors: print(f"FAILED - {len(errors)} ID constraint violations:") @@ -389,52 +407,54 @@ def repair(self) -> int: return repairs def repair_durableId(self) -> int: + DURABLE_ID_ATTRS = ("w16cid:durableId", "w16cex:durableId") repairs = 0 + renames: dict = {} for xml_file in self.xml_files: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) + is_numbering = xml_file.name == "numbering.xml" + base = 10 if is_numbering else 16 + pending = [] + seen_in_file = set() modified = False for elem in dom.getElementsByTagName("*"): - if not elem.hasAttribute("w16cid:durableId"): - continue + for attr_name in DURABLE_ID_ATTRS: + if not elem.hasAttribute(attr_name): + continue - durable_id = elem.getAttribute("w16cid:durableId") - needs_repair = False - - if xml_file.name == "numbering.xml": + durable_id = elem.getAttribute(attr_name) try: - needs_repair = ( - self._parse_id_value(durable_id, base=10) >= 0x7FFFFFFF - ) + key = self._parse_id_value(durable_id, base=base) + needs_repair = key >= 0x7FFFFFFF except ValueError: + key = durable_id needs_repair = True - else: - try: - needs_repair = ( - self._parse_id_value(durable_id, base=16) >= 0x7FFFFFFF - ) - except ValueError: - needs_repair = True - - if needs_repair: - value = random.randint(1, 0x7FFFFFFE) - if xml_file.name == "numbering.xml": - new_id = str(value) - else: - new_id = f"{value:08X}" - elem.setAttribute("w16cid:durableId", new_id) - print( - f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" - ) - repairs += 1 - modified = True + if needs_repair: + if key in seen_in_file: + value = random.randint(1, 0x7FFFFFFE) + else: + seen_in_file.add(key) + if key not in renames: + renames[key] = random.randint(1, 0x7FFFFFFE) + value = renames[key] + new_id = str(value) if is_numbering else f"{value:08X}" + + elem.setAttribute(attr_name, new_id) + pending.append( + f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" + ) + modified = True if modified: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/pptx.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/pptx.py index 09842aa998..7b53d0d3e4 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/pptx.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/pptx.py @@ -3,6 +3,9 @@ """ import re +from pathlib import Path + +from helpers import opc_target, rels_source_part, safe_extract from .base import BaseSchemaValidator @@ -57,8 +60,171 @@ def validate(self): if not self.validate_no_duplicate_slide_layouts(): all_valid = False + if not self.validate_master_theme_uniqueness(): + all_valid = False + + if not self.validate_charts(): + all_valid = False + + if not self.validate_slides(): + all_valid = False + return all_valid + def _package_map(self) -> dict: + wanted = [] + wanted += list(self.unpacked_dir.glob("[[]Content_Types[]].xml")) + wanted += list(self.unpacked_dir.glob("ppt/presentation.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/*.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/_rels/*.rels")) + wanted += list(self.unpacked_dir.glob("ppt/charts/chart*.xml")) + for group in ("slideMasters", "notesMasters", "handoutMasters"): + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/*.xml")) + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/_rels/*.rels")) + return { + p.relative_to(self.unpacked_dir).as_posix(): p.read_bytes() + for p in wanted + if p.is_file() + } + + def validate_master_theme_uniqueness(self): + from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes + + shared = live_shared_master_themes(self._package_map()) + if shared: + print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:") + for message in shared: + print(f" {message}") + if any(m.startswith(_NOTES_MASTERS) for m in shared): + print(" Fix: in ppt/presentation.xml, move back to " + "directly after . PowerPoint reads that happily.") + else: + print(" Fix: give each master its own theme part.") + return False + + if self.verbose: + print("PASSED - No master shares a theme part in a way PowerPoint refuses") + return True + + def validate_charts(self): + from helpers.pptx_chart import find_chart_problems + + problems = find_chart_problems(self._package_map()) + if problems: + print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:") + for message in problems: + print(f" {message}") + return False + + if self.verbose: + print("PASSED - Charts satisfy the constraints PowerPoint enforces") + return True + + def _original_slide_defects(self, schema) -> set[str]: + import tempfile + import zipfile + + from helpers.pptx_slide import SLIDE_PART_RE, fatal_slide_errors + + if self.original_file is None: + return set() + + found: set[str] = set() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + try: + with zipfile.ZipFile(self.original_file, "r") as zf: + safe_extract(zf, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() + + for part in sorted(temp_path.rglob("*.xml")): + relative = part.relative_to(temp_path).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + part.resolve(), temp_path.resolve(), schema_path=schema + ) + if ok is None or ok or not errors: + continue + found |= set(fatal_slide_errors(set(errors))) + return found + + def validate_slides(self): + from helpers.pptx_slide import ( + SLIDE_PART_RE, + fatal_slide_errors, + is_schema_verdict, + ) + + schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"] + inherited = self._original_slide_defects(schema) + problems: list[str] = [] + broken: list[str] = [] + + for xml_file in self.xml_files: + relative = xml_file.relative_to(self.unpacked_dir).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema + ) + if ok is None or not errors: + continue + + unreadable = [f"{relative}: {e}" for e in errors if not is_schema_verdict(e)] + if unreadable: + broken.extend(unreadable) + continue + if ok: + continue + + for message in fatal_slide_errors(set(errors)): + if message in inherited: + continue + problems.append(f"{relative}: {message}") + + if broken: + print(f"FAILED - Could not check {len(broken)} slide part(s):") + for message in sorted(broken): + print(f" {message[:240]}") + + if problems: + print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:") + for message in sorted(problems): + print(f" {message[:240]}") + + if broken or problems: + return False + + if self.verbose: + print("PASSED - Slide XML has none of the defects PowerPoint refuses") + return True + + def _get_schema_path(self, xml_file): + if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"): + return None + return super()._get_schema_path(xml_file) + + def _preprocess_for_schema(self, xml_doc, relative_path): + if relative_path.as_posix() != "ppt/presentation.xml": + return xml_doc + + root = xml_doc.getroot() + ns = f"{{{self.PRESENTATIONML_NAMESPACE}}}" + notes = root.find(f"{ns}notesMasterIdLst") + slides = root.find(f"{ns}sldIdLst") + if notes is None or slides is None: + return xml_doc + + children = list(root) + if children.index(notes) < children.index(slides): + return xml_doc + + root.remove(notes) + root.insert(list(root).index(slides), notes) + return xml_doc + def validate_uuid_ids(self): import lxml.etree @@ -229,17 +395,17 @@ def validate_notes_slide_references(self): ): rel_type = rel.get("Type", "") if "notesSlide" in rel_type: - target = rel.get("Target", "") - if target: - normalized_target = target.replace("../", "") - + part = opc_target( + rel.get("Target", ""), + rels_source_part(rels_file, self.unpacked_dir), + rel.get("TargetMode", ""), + ) + if part: slide_name = rels_file.stem.replace( ".xml", "" ) - if normalized_target not in notes_slide_references: - notes_slide_references[normalized_target] = [] - notes_slide_references[normalized_target].append( + notes_slide_references.setdefault(part, []).append( (slide_name, rels_file) ) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/redlining.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/redlining.py index 71c81b6bf4..18d0c68be9 100644 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/redlining.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/office/validators/redlining.py @@ -1,5 +1,14 @@ """ Validator for tracked changes in Word documents. + +Detects untracked edits in word/document.xml: text that differs from the +original without a / wrapper recording it. The tracked changes +that are new relative to the original are undone, and the result is compared +against the original; whatever text still differs was edited without being +tracked. + +Only the document body is compared. Headers, footers, footnotes and endnotes +are separate parts and are not checked. """ import subprocess @@ -7,14 +16,18 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import rendered_text, safe_extract + class RedliningValidator: - def __init__(self, unpacked_dir, original_docx, verbose=False, author="Claude"): + def __init__(self, unpacked_dir, original_docx, verbose=False): self.unpacked_dir = Path(unpacked_dir) self.original_docx = Path(original_docx) self.verbose = verbose - self.author = author self.namespaces = { "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main" } @@ -28,40 +41,12 @@ def validate(self): print(f"FAILED - Modified document.xml not found at {modified_file}") return False - try: - import xml.etree.ElementTree as ET - - tree = ET.parse(modified_file) - root = tree.getroot() - - del_elements = root.findall(".//w:del", self.namespaces) - ins_elements = root.findall(".//w:ins", self.namespaces) - - author_del_elements = [ - elem - for elem in del_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - author_ins_elements = [ - elem - for elem in ins_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - - if not author_del_elements and not author_ins_elements: - if self.verbose: - print(f"PASSED - No tracked changes by {self.author} found.") - return True - - except Exception: - pass - with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) try: with zipfile.ZipFile(self.original_docx, "r") as zip_ref: - zip_ref.extractall(temp_path) + safe_extract(zip_ref, temp_path) except Exception as e: print(f"FAILED - Error unpacking original docx: {e}") return False @@ -74,18 +59,16 @@ def validate(self): return False try: - import xml.etree.ElementTree as ET - modified_tree = ET.parse(modified_file) modified_root = modified_tree.getroot() original_tree = ET.parse(original_file) original_root = original_tree.getroot() - except ET.ParseError as e: + except (ET.ParseError, DefusedXmlException) as e: print(f"FAILED - Error parsing XML files: {e}") return False - self._remove_author_tracked_changes(original_root) - self._remove_author_tracked_changes(modified_root) + new_changes = self._new_tracked_changes(original_root, modified_root) + self._remove_tracked_changes(modified_root, new_changes) modified_text = self._extract_text_content(modified_root) original_text = self._extract_text_content(original_root) @@ -98,20 +81,91 @@ def validate(self): return False if self.verbose: - print(f"PASSED - All changes by {self.author} are properly tracked") + print( + f"PASSED - All {len(new_changes)} change(s) against the original " + "are properly tracked" + ) return True + def _tracked_change_elements(self, root): + ins_tag = f"{{{self.namespaces['w']}}}ins" + del_tag = f"{{{self.namespaces['w']}}}del" + return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)] + + def _rendered_text(self, elem): + preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" + return rendered_text(elem.text or "", preserve) + + def _text_elements(self, elem): + w = self.namespaces["w"] + return [ + node + for node in elem.iter() + if node.tag in (f"{{{w}}}t", f"{{{w}}}delText") + ] + + def _tracked_change_key(self, elem): + w = self.namespaces["w"] + text = "".join(self._rendered_text(node) for node in self._text_elements(elem)) + return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text) + + def _new_tracked_changes(self, original_root, modified_root): + original = self._tracked_change_elements(original_root) + modified = self._tracked_change_elements(modified_root) + + pool = {} + for elem in original: + pool.setdefault(self._tracked_change_key(elem), []).append(elem) + + matched, leftover = set(), [] + for elem in modified: + bucket = pool.get(self._tracked_change_key(elem)) + if bucket: + matched.add(bucket.pop()) + else: + leftover.append(elem) + + def group(elem): + return self._tracked_change_key(elem)[:3] + + def text_of(elems): + return "".join(self._tracked_change_key(e)[3] for e in elems) + + unmatched_original = {} + for elem in original: + if elem not in matched: + unmatched_original.setdefault(group(elem), []).append(elem) + + by_group = {} + for elem in leftover: + by_group.setdefault(group(elem), []).append(elem) + + new = set() + for key, elems in by_group.items(): + rebuilt = text_of(elems) + if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])): + continue + new.update(elems) + return new + def _generate_detailed_diff(self, original_text, modified_text): error_parts = [ - f"FAILED - Document text doesn't match after removing {self.author}'s tracked changes", + "FAILED - Document text doesn't match after removing the tracked changes", "", "Likely causes:", " 1. Modified text inside another author's or tags", " 2. Made edits without proper tracked changes", " 3. Didn't nest inside when deleting another's insertion", + " 4. Rewrote another author's / and changed its text on", + " the way. A tracked change from the original is recognised by its", + " author, date and text; anything that doesn't reproduce one exactly", + " reads as new, and the text it carried is reported missing.", "", "For pre-redlined documents, use correct patterns:", " - To reject another's INSERTION: Nest inside their ", + " - To reject PART of one: nest around only the runs you reject.", + " Their may be split around it, so long as the pieces keep", + " their author and date and still spell out the same text.", " - To restore another's DELETION: Add new AFTER their ", "", ] @@ -195,15 +249,14 @@ def _get_git_word_diff(self, original_text, modified_text): return None - def _remove_author_tracked_changes(self, root): + def _remove_tracked_changes(self, root, targets): ins_tag = f"{{{self.namespaces['w']}}}ins" del_tag = f"{{{self.namespaces['w']}}}del" - author_attr = f"{{{self.namespaces['w']}}}author" for parent in root.iter(): to_remove = [] for child in parent: - if child.tag == ins_tag and child.get(author_attr) == self.author: + if child.tag == ins_tag and child in targets: to_remove.append(child) for elem in to_remove: parent.remove(elem) @@ -214,7 +267,7 @@ def _remove_author_tracked_changes(self, root): for parent in root.iter(): to_process = [] for child in parent: - if child.tag == del_tag and child.get(author_attr) == self.author: + if child.tag == del_tag and child in targets: to_process.append((child, list(parent).index(child))) for del_elem, del_index in reversed(to_process): @@ -234,8 +287,7 @@ def _extract_text_content(self, root): for p_elem in root.findall(f".//{p_tag}"): text_parts = [] for t_elem in p_elem.findall(f".//{t_tag}"): - if t_elem.text: - text_parts.append(t_elem.text) + text_parts.append(self._rendered_text(t_elem)) paragraph_text = "".join(text_parts) if paragraph_text: paragraphs.append(paragraph_text) diff --git a/src/crates/assembly/core/builtin_skills/pptx/scripts/thumbnail.py b/src/crates/assembly/core/builtin_skills/pptx/scripts/thumbnail.py index edcbdc0f81..ae79b0e2fd 100755 --- a/src/crates/assembly/core/builtin_skills/pptx/scripts/thumbnail.py +++ b/src/crates/assembly/core/builtin_skills/pptx/scripts/thumbnail.py @@ -16,6 +16,7 @@ """ import argparse +import posixpath import subprocess import sys import tempfile @@ -23,9 +24,12 @@ from pathlib import Path import defusedxml.minidom -from office.soffice import get_soffice_env +from defusedxml import ElementTree +from office.helpers import SLIDE_REL_TYPE, opc_target +from office.soffice import run_soffice from PIL import Image, ImageDraw, ImageFont + THUMBNAIL_WIDTH = 300 CONVERSION_DPI = 100 MAX_COLS = 6 @@ -92,28 +96,45 @@ def main(): sys.exit(1) +def _is_hidden(zf: zipfile.ZipFile, part: str) -> bool: + try: + with zf.open(part) as f: + for _, root in ElementTree.iterparse(f, events=("start",)): + return root.get("show") in ("0", "false") + except (KeyError, ElementTree.ParseError): + return False + return False + + def get_slide_info(pptx_path: Path) -> list[dict]: with zipfile.ZipFile(pptx_path, "r") as zf: rels_content = zf.read("ppt/_rels/presentation.xml.rels").decode("utf-8") rels_dom = defusedxml.minidom.parseString(rels_content) - rid_to_slide = {} + rid_to_part = {} for rel in rels_dom.getElementsByTagName("Relationship"): - rid = rel.getAttribute("Id") - target = rel.getAttribute("Target") - rel_type = rel.getAttribute("Type") - if "slide" in rel_type and target.startswith("slides/"): - rid_to_slide[rid] = target.replace("slides/", "") + if rel.getAttribute("Type") != SLIDE_REL_TYPE: + continue + part = opc_target( + rel.getAttribute("Target"), + "ppt/presentation.xml", + rel.getAttribute("TargetMode"), + ) + if part is not None: + rid_to_part[rel.getAttribute("Id")] = part pres_content = zf.read("ppt/presentation.xml").decode("utf-8") pres_dom = defusedxml.minidom.parseString(pres_content) + present = set(zf.namelist()) + slides = [] for sld_id in pres_dom.getElementsByTagName("p:sldId"): - rid = sld_id.getAttribute("r:id") - if rid in rid_to_slide: - hidden = sld_id.getAttribute("show") == "0" - slides.append({"name": rid_to_slide[rid], "hidden": hidden}) + part = rid_to_part.get(sld_id.getAttribute("r:id")) + if part is not None and part in present: + slides.append( + {"name": posixpath.basename(part), "hidden": _is_hidden(zf, part)} + ) return slides @@ -123,6 +144,15 @@ def build_slide_list( visible_images: list[Path], temp_dir: Path, ) -> list[tuple[Path, str]]: + visible_count = sum(1 for info in slide_info if not info["hidden"]) + rendered_hidden = len(visible_images) == len(slide_info) != visible_count + + if not rendered_hidden and visible_count != len(visible_images): + raise ValueError( + f"LibreOffice rendered {len(visible_images)} page(s) for {visible_count} " + f"visible slide(s) of {len(slide_info)}; thumbnails would be mislabeled" + ) + if visible_images: with Image.open(visible_images[0]) as img: placeholder_size = img.size @@ -133,15 +163,15 @@ def build_slide_list( visible_idx = 0 for info in slide_info: - if info["hidden"]: + if info["hidden"] and not rendered_hidden: placeholder_path = temp_dir / f"hidden-{info['name']}.jpg" placeholder_img = create_hidden_placeholder(placeholder_size) placeholder_img.save(placeholder_path, "JPEG") slides.append((placeholder_path, f"{info['name']} (hidden)")) else: - if visible_idx < len(visible_images): - slides.append((visible_images[visible_idx], info["name"])) - visible_idx += 1 + label = f"{info['name']} (hidden)" if info["hidden"] else info["name"] + slides.append((visible_images[visible_idx], label)) + visible_idx += 1 return slides @@ -158,22 +188,14 @@ def create_hidden_placeholder(size: tuple[int, int]) -> Image.Image: def convert_to_images(pptx_path: Path, temp_dir: Path) -> list[Path]: pdf_path = temp_dir / f"{pptx_path.stem}.pdf" - result = subprocess.run( - [ - "soffice", - "--headless", - "--convert-to", - "pdf", - "--outdir", - str(temp_dir), - str(pptx_path), - ], + result = run_soffice( + ["--headless", "--convert-to", "pdf", "--outdir", str(temp_dir), str(pptx_path)], capture_output=True, text=True, - env=get_soffice_env(), ) if result.returncode != 0 or not pdf_path.exists(): - raise RuntimeError("PDF conversion failed") + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"PDF conversion failed: {detail}" if detail else "PDF conversion failed") result = subprocess.run( [ diff --git a/src/crates/assembly/core/builtin_skills/xlsx/SKILL.md b/src/crates/assembly/core/builtin_skills/xlsx/SKILL.md index 2daadd54a8..dab8cacf95 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/SKILL.md +++ b/src/crates/assembly/core/builtin_skills/xlsx/SKILL.md @@ -1,292 +1,99 @@ --- name: xlsx -description: "Use this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like \"the xlsx in my downloads\") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved." +description: "Use this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .xltx, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like \"the xlsx in my downloads\") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved." license: Proprietary. LICENSE.txt has complete terms --- -# Requirements for Outputs - -## All Excel files - -### Professional Font -- Use a consistent, professional font (e.g., Arial, Times New Roman) for all deliverables unless otherwise instructed by the user - -### Zero Formula Errors -- Every Excel model MUST be delivered with ZERO formula errors (#REF!, #DIV/0!, #VALUE!, #N/A, #NAME?) - -### Preserve Existing Templates (when updating templates) -- Study and EXACTLY match existing format, style, and conventions when modifying files -- Never impose standardized formatting on files with established patterns -- Existing template conventions ALWAYS override these guidelines - -## Financial models - -### Color Coding Standards -Unless otherwise stated by the user or existing template - -#### Industry-Standard Color Conventions -- **Blue text (RGB: 0,0,255)**: Hardcoded inputs, and numbers users will change for scenarios -- **Black text (RGB: 0,0,0)**: ALL formulas and calculations -- **Green text (RGB: 0,128,0)**: Links pulling from other worksheets within same workbook -- **Red text (RGB: 255,0,0)**: External links to other files -- **Yellow background (RGB: 255,255,0)**: Key assumptions needing attention or cells that need to be updated - -### Number Formatting Standards - -#### Required Format Rules -- **Years**: Format as text strings (e.g., "2026" not "2,026") -- **Currency**: Use $#,##0 format; ALWAYS specify units in headers ("Revenue ($mm)") -- **Zeros**: Use number formatting to make all zeros "-", including percentages (e.g., "$#,##0;($#,##0);-") -- **Percentages**: Default to 0.0% format (one decimal) -- **Multiples**: Format as 0.0x for valuation multiples (EV/EBITDA, P/E) -- **Negative numbers**: Use parentheses (123) not minus -123 - -### Formula Construction Rules - -#### Assumptions Placement -- Place ALL assumptions (growth rates, margins, multiples, etc.) in separate assumption cells -- Use cell references instead of hardcoded values in formulas -- Example: Use =B5*(1+$B$6) instead of =B5*1.05 - -#### Formula Error Prevention -- Verify all cell references are correct -- Check for off-by-one errors in ranges -- Ensure consistent formulas across all projection periods -- Test with edge cases (zero values, negative numbers) -- Verify no unintended circular references - -#### Documentation Requirements for Hardcodes -- Comment or in cells beside (if end of table). Format: "Source: [System/Document], [Date], [Specific Reference], [URL if applicable]" -- Examples: - - "Source: Company 10-K, FY2024, Page 45, Revenue Note, [SEC EDGAR URL]" - - "Source: Company 10-Q, Q2 2025, Exhibit 99.1, [SEC EDGAR URL]" - - "Source: Bloomberg Terminal, 8/15/2025, AAPL US Equity" - - "Source: FactSet, 8/20/2025, Consensus Estimates Screen" - # XLSX creation, editing, and analysis -## Overview - -A user may ask you to create, edit, or analyze the contents of an .xlsx file. You have different tools and workflows available for different tasks. +| Task | Approach | +|---|---| +| **Create** or **edit** with formulas/formatting | `openpyxl` — see gotchas below | +| **Bulk data** in or out | `pandas` (`read_excel`, `to_excel`) | +| **Quick look** at a sheet | `markitdown file.xlsx` — `## SheetName` per sheet; reads `.xlsm` too. No cell coordinates, so don't plan edits from it | +| **Read** a model (formulas *and* values) | two `load_workbook` passes — see gotchas | -## Important Requirements +> `openpyxl`, `pandas`, and `markitdown` are preinstalled — do not run `pip install` first; write the script and import directly. Only if an import fails (or the `markitdown` command is missing): `pip install` the missing package. -**LibreOffice Required for Formula Recalculation**: You can assume LibreOffice is installed for recalculating formula values using the `scripts/recalc.py` script. The script automatically configures LibreOffice on first run, including in sandboxed environments where Unix sockets are restricted (handled by `scripts/office/soffice.py`) +> Script paths below are relative to this skill's directory. -## Reading and analyzing data +## Requirements for every output -### Data analysis with pandas -For data analysis, visualization, and basic operations, use **pandas** which provides powerful data manipulation capabilities: +- **Professional font** (Arial, Times New Roman) throughout, unless the user says otherwise. +- **Zero formula errors.** Never ship while `recalc.py` reports `errors_found`. If you think an error predates you, prove it: load the *original* with `data_only=True` and look at that cell. An error you introduced looks exactly like one you inherited. +- **Use formulas, never hardcoded results.** Write `sheet['B10'] = '=SUM(B2:B9)'`, not the Python-computed total. The sheet must recalculate when its inputs change. +- **Follow the user's spec literally.** Exact tab names, exact column headers, and the formula they spelled out. A redesign that computes something else fails, however elegant. +- **Document every assumption and hardcoded number** where the reader will see it — a cell comment, or an adjacent cell at a table's end. Cite a real source when one exists (`Source: Company 10-K, FY2024, Page 45, Revenue Note, [SEC EDGAR URL]`); when the number came from the user, say so plainly. +- **A workbook *you create* for someone to fill in** needs a short legend naming which cells to edit, and one example row of realistic values showing the expected format. Never add such a row to a file you were asked to edit. +- **Editing an existing file: match its conventions exactly.** They override every guideline here. Find its designated input cells first — a distinct font color, fill, or shading marks them — write only there, and leave every existing formula untouched. -```python -import pandas as pd - -# Read Excel -df = pd.read_excel('file.xlsx') # Default: first sheet -all_sheets = pd.read_excel('file.xlsx', sheet_name=None) # All sheets as dict - -# Analyze -df.head() # Preview data -df.info() # Column info -df.describe() # Statistics - -# Write Excel -df.to_excel('output.xlsx', index=False) -``` - -## Excel File Workflows - -## CRITICAL: Use Formulas, Not Hardcoded Values - -**Always use Excel formulas instead of calculating values in Python and hardcoding them.** This ensures the spreadsheet remains dynamic and updateable. - -### ❌ WRONG - Hardcoding Calculated Values -```python -# Bad: Calculating in Python and hardcoding result -total = df['Sales'].sum() -sheet['B10'] = total # Hardcodes 5000 - -# Bad: Computing growth rate in Python -growth = (df.iloc[-1]['Revenue'] - df.iloc[0]['Revenue']) / df.iloc[0]['Revenue'] -sheet['C5'] = growth # Hardcodes 0.15 - -# Bad: Python calculation for average -avg = sum(values) / len(values) -sheet['D20'] = avg # Hardcodes 42.5 -``` - -### ✅ CORRECT - Using Excel Formulas -```python -# Good: Let Excel calculate the sum -sheet['B10'] = '=SUM(B2:B9)' - -# Good: Growth rate as Excel formula -sheet['C5'] = '=(C4-C2)/C2' - -# Good: Average using Excel function -sheet['D20'] = '=AVERAGE(D2:D19)' -``` +## Recalculate (mandatory whenever the file contains formulas) -This applies to ALL calculations - totals, percentages, ratios, differences, etc. The spreadsheet should be able to recalculate when source data changes. +openpyxl writes formulas as strings with **no cached values**. Until you recalculate, every +formula cell reads back as `None` to anything reading cached values — `pandas`, +`load_workbook(data_only=True)`, and most previewers. -## Common Workflow -1. **Choose tool**: pandas for data, openpyxl for formulas/formatting -2. **Create/Load**: Create new workbook or load existing file -3. **Modify**: Add/edit data, formulas, and formatting -4. **Save**: Write to file -5. **Recalculate formulas (MANDATORY IF USING FORMULAS)**: Use the scripts/recalc.py script - ```bash - python scripts/recalc.py output.xlsx - ``` -6. **Verify and fix any errors**: - - The script returns JSON with error details - - If `status` is `errors_found`, check `error_summary` for specific error types and locations - - Fix the identified errors and recalculate again - - Common errors to fix: - - `#REF!`: Invalid cell references - - `#DIV/0!`: Division by zero - - `#VALUE!`: Wrong data type in formula - - `#NAME?`: Unrecognized formula name - -### Creating new Excel files - -```python -# Using openpyxl for formulas and formatting -from openpyxl import Workbook -from openpyxl.styles import Font, PatternFill, Alignment - -wb = Workbook() -sheet = wb.active - -# Add data -sheet['A1'] = 'Hello' -sheet['B1'] = 'World' -sheet.append(['Row', 'of', 'data']) - -# Add formula -sheet['B2'] = '=SUM(A1:A10)' - -# Formatting -sheet['A1'].font = Font(bold=True, color='FF0000') -sheet['A1'].fill = PatternFill('solid', start_color='FFFF00') -sheet['A1'].alignment = Alignment(horizontal='center') - -# Column width -sheet.column_dimensions['A'].width = 20 - -wb.save('output.xlsx') -``` - -### Editing existing Excel files - -```python -# Using openpyxl to preserve formulas and formatting -from openpyxl import load_workbook - -# Load existing file -wb = load_workbook('existing.xlsx') -sheet = wb.active # or wb['SheetName'] for specific sheet - -# Working with multiple sheets -for sheet_name in wb.sheetnames: - sheet = wb[sheet_name] - print(f"Sheet: {sheet_name}") - -# Modify cells -sheet['A1'] = 'New Value' -sheet.insert_rows(2) # Insert row at position 2 -sheet.delete_cols(3) # Delete column 3 - -# Add new sheet -new_sheet = wb.create_sheet('NewSheet') -new_sheet['A1'] = 'Data' - -wb.save('modified.xlsx') -``` - -## Recalculating formulas - -Excel files created or modified by openpyxl contain formulas as strings but not calculated values. Use the provided `scripts/recalc.py` script to recalculate formulas: - -```bash -python scripts/recalc.py [timeout_seconds] -``` - -Example: ```bash -python scripts/recalc.py output.xlsx 30 +python scripts/recalc.py output.xlsx [timeout_seconds] # default 30 ``` -The script: -- Automatically sets up LibreOffice macro on first run -- Recalculates all formulas in all sheets -- Scans ALL cells for Excel errors (#REF!, #DIV/0!, etc.) -- Returns JSON with detailed error locations and counts -- Works on both Linux and macOS +LibreOffice computes every formula, the file is **rewritten in place**, and you get JSON: +`status` (`success` | `errors_found`), `total_formulas`, `total_errors`, and an +`error_summary` naming up to 100 cells per error type (`locations_truncated` says how many it +withheld — trust `total_errors`, not the length of the list). Fix what it names and run it +again. **JSON with an `error` key instead of a `status` means nothing was recalculated**, and +only that case exits non-zero — `errors_found` exits 0, so never treat a clean exit as a clean +workbook. + +**A green recalc proves your formulas *evaluate*, not that they are *right*.** An off-by-one +range or a reference to the wrong row yields a clean, error-free file with wrong numbers. +Write 2–3 formulas first and check they pull the values you expect, before building out a grid. + +**A workbook that links to another file loses those links** if you re-save it with openpyxl and +then recalculate. Such a formula reads `='[1]Returns Analysis'!$B$2` — the `[1]` is an index +into the workbook's external-reference list, naming a *separate file on disk*, not a sheet. +That file is rarely present here, so the cell's cached value is the only thing holding its +data. openpyxl strips that value on save; LibreOffice then has to resolve the reference for +real, fails, writes `#NAME?`, and deletes every link. `recalc.py` refuses to run in that state +— copy those cells' values out of the original before you save over them (`--force` overrides, +and accepts the loss). + +## Choosing formulas that survive verification + +LibreOffice implements fewer functions than Excel, and one it cannot evaluate becomes a +literal `#NAME?` baked into the file you deliver. + +- **Prefer Excel-2007-era functions** — `SUMIFS`, `INDEX`, `MATCH`, `IFERROR`, `SUMPRODUCT` — which need no prefix. +- **Six post-2007 functions work, but only with an `_xlfn.` prefix**, because openpyxl writes your formula into the XML verbatim and Excel stores post-2007 names prefixed (its UI hides the prefix): `_xlfn.TEXTJOIN`, `_xlfn.CONCAT`, `_xlfn.IFS`, `_xlfn.SWITCH`, `_xlfn.MAXIFS`, `_xlfn.MINIFS`. Written bare, each yields `#NAME?`. +- **Never use `XLOOKUP`, `XMATCH`, `SORT`, `FILTER`, `UNIQUE`, or `SEQUENCE`.** The runtime's LibreOffice cannot evaluate them under *any* prefix. Newer builds do evaluate them, but they are spilling array functions and an openpyxl-written file has no spill metadata, so only the top-left cell of the range gets a value — and `recalc.py` reports `total_errors: 0` on the truncated result. Use `INDEX`/`MATCH` for lookups, and sort, filter, and de-duplicate in Python before writing the cells. +- A formula LibreOffice could not parse is written back **lowercased** — a quick tell beside a `#NAME?`. + +## openpyxl gotchas + +- **Reading a model takes two loads.** `data_only=True` yields cached values with the formulas gone; the default yields formula strings with no values. One pass cannot give you both. +- **`data_only=True` is destructive if you save.** That workbook has no formulas left, so saving replaces every one with a literal — permanently. +- **`data_only=True` on a file openpyxl just wrote returns `None` everywhere** — run `recalc.py` first. (A formula whose result is `""` also reads back as `None`.) +- **Merged cells: write the top-left anchor only.** Every other cell in the range is a `MergedCell` whose `.value` is read-only. +- **`.xlsm` loses its macros unless you pass `keep_vba=True`** to `load_workbook`. +- **A sheet name containing a space must be quoted** in a cross-sheet reference: `='Assumptions Inputs'!$B$5`. Unquoted, it evaluates to `#VALUE!`. -## Formula Verification Checklist - -Quick checks to ensure formulas work correctly: - -### Essential Verification -- [ ] **Test 2-3 sample references**: Verify they pull correct values before building full model -- [ ] **Column mapping**: Confirm Excel columns match (e.g., column 64 = BL, not BK) -- [ ] **Row offset**: Remember Excel rows are 1-indexed (DataFrame row 5 = Excel row 6) - -### Common Pitfalls -- [ ] **NaN handling**: Check for null values with `pd.notna()` -- [ ] **Far-right columns**: FY data often in columns 50+ -- [ ] **Multiple matches**: Search all occurrences, not just first -- [ ] **Division by zero**: Check denominators before using `/` in formulas (#DIV/0!) -- [ ] **Wrong references**: Verify all cell references point to intended cells (#REF!) -- [ ] **Cross-sheet references**: Use correct format (Sheet1!A1) for linking sheets - -### Formula Testing Strategy -- [ ] **Start small**: Test formulas on 2-3 cells before applying broadly -- [ ] **Verify dependencies**: Check all cells referenced in formulas exist -- [ ] **Test edge cases**: Include zero, negative, and very large values - -### Interpreting scripts/recalc.py Output -The script returns JSON with error details: -```json -{ - "status": "success", // or "errors_found" - "total_errors": 0, // Total error count - "total_formulas": 42, // Number of formulas in file - "error_summary": { // Only present if errors found - "#REF!": { - "count": 2, - "locations": ["Sheet1!B5", "Sheet1!C10"] - } - } -} -``` +## Financial models -## Best Practices +Unless the user says otherwise, or the existing file already does something else. -### Library Selection -- **pandas**: Best for data analysis, bulk operations, and simple data export -- **openpyxl**: Best for complex formatting, formulas, and Excel-specific features +**Color:** blue text (`0,0,255`) for hardcoded inputs and scenario levers · black for formulas · +green (`0,128,0`) for links to another sheet · red (`255,0,0`) for links to another file · +yellow fill (`255,255,0`) for key assumptions and cells the user should fill in. -### Working with openpyxl -- Cell indices are 1-based (row=1, column=1 refers to cell A1) -- Use `data_only=True` to read calculated values: `load_workbook('file.xlsx', data_only=True)` -- **Warning**: If opened with `data_only=True` and saved, formulas are replaced with values and permanently lost -- For large files: Use `read_only=True` for reading or `write_only=True` for writing -- Formulas are preserved but not evaluated - use scripts/recalc.py to update values +**Numbers:** currency `$#,##0`, with the unit named in the header (`Revenue ($mm)`) · zeros +render as `-`, including in percentages (`$#,##0;($#,##0);-`) · negatives in parentheses · +percentages `0.0%`, **stored as fractions** (`0.15` renders `15.0%`; storing `15` renders +`1500.0%`) · valuation multiples `0.0x` · years as text (`"2026"`, never `2,026`). -### Working with pandas -- Specify data types to avoid inference issues: `pd.read_excel('file.xlsx', dtype={'id': str})` -- For large files, read specific columns: `pd.read_excel('file.xlsx', usecols=['A', 'C', 'E'])` -- Handle dates properly: `pd.read_excel('file.xlsx', parse_dates=['date_column'])` +**Structure:** every assumption in its own labeled cell, referenced by the formulas that use it +(`=B5*(1+$B$6)`, never `=B5*1.05`) · formulas consistent across every projection period, since a +lone edited cell mid-row is the commonest silent error · guard denominators that can be zero. -## Code Style Guidelines -**IMPORTANT**: When generating Python code for Excel operations: -- Write minimal, concise Python code without unnecessary comments -- Avoid verbose variable names and redundant operations -- Avoid unnecessary print statements +## Dependencies -**For Excel files themselves**: -- Add comments to cells with complex formulas or important assumptions -- Document data sources for hardcoded values -- Include notes for key calculations and model sections \ No newline at end of file +`openpyxl`, `pandas`, `markitdown` (pip, preinstalled — install only if an import fails or the command is missing) · LibreOffice (`soffice`, auto-configured for sandboxed environments via `scripts/office/soffice.py`) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/__init__.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/__init__.py index e69de29bb2..188b00aff4 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/__init__.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/__init__.py @@ -0,0 +1,150 @@ +import os +import posixpath +import re +import stat +import tempfile +import urllib.parse +import zipfile +from pathlib import Path + +OOXML_FAMILY = { + ".docx": "docx", + ".dotx": "docx", + ".pptx": "pptx", + ".potx": "pptx", + ".xlsx": "xlsx", + ".xltx": "xlsx", +} + +_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:") + +SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" + +MAX_ARCHIVE_MEMBERS = 10_000 +MAX_ARCHIVE_MEMBER_SIZE = 1 * 1024 * 1024 * 1024 +MAX_ARCHIVE_TOTAL_SIZE = 4 * 1024 * 1024 * 1024 +MAX_ARCHIVE_COMPRESSION_RATIO = 1_000 + + +def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None: + if not target: + return None + if target_mode.lower() == "external": + return None + if _SCHEME_RE.match(target): + return None + + target = urllib.parse.unquote(target) + + if "\\" in target: + raise ValueError(f"relationship target is not a POSIX part name: {target!r}") + + if target.startswith("/"): + joined = target.lstrip("/") + else: + joined = posixpath.join(posixpath.dirname(source_part), target) + + parts: list[str] = [] + for segment in posixpath.normpath(joined).split("/"): + if segment in ("", "."): + continue + if segment == "..": + if not parts: + raise ValueError(f"relationship target escapes the package: {target!r}") + parts.pop() + else: + parts.append(segment) + + if not parts: + raise ValueError(f"relationship target resolves to nothing: {target!r}") + return "/".join(parts) + + +def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str: + owner_dir = rels_file.parent.parent.relative_to(unpacked_dir) + return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./") + + +def part_text(data: bytes) -> str: + return data.decode("utf-8", "surrogateescape") + + +XML_SPACE = " \t\r\n" + + +def rendered_text(text: str, preserve: bool) -> str: + return text if preserve else text.strip(XML_SPACE) + + +def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None: + dest = dest.resolve() + members = zf.infolist() + if len(members) > MAX_ARCHIVE_MEMBERS: + raise ValueError(f"archive has too many entries: {len(members)}") + + total_size = 0 + targets: set[str] = set() + file_targets: set[str] = set() + validated: list[tuple[zipfile.ZipInfo, Path]] = [] + for m in members: + if stat.S_ISLNK(m.external_attr >> 16): + raise ValueError(f"symlink archive entry not allowed: {m.filename!r}") + target = (dest / m.filename).resolve() + if target == dest or not target.is_relative_to(dest): + raise ValueError(f"unsafe archive entry: {m.filename!r}") + target_key = os.path.normcase(str(target)) + if target_key in targets: + raise ValueError(f"duplicate archive entry: {m.filename!r}") + targets.add(target_key) + if not m.is_dir(): + file_targets.add(target_key) + validated.append((m, target)) + if m.file_size > MAX_ARCHIVE_MEMBER_SIZE: + raise ValueError(f"archive entry is too large: {m.filename!r}") + total_size += m.file_size + if total_size > MAX_ARCHIVE_TOTAL_SIZE: + raise ValueError("archive expands beyond the allowed total size") + if m.file_size and ( + m.compress_size == 0 + or m.file_size > m.compress_size * MAX_ARCHIVE_COMPRESSION_RATIO + ): + raise ValueError(f"archive entry has an unsafe compression ratio: {m.filename!r}") + + for m, target in validated: + for parent in target.parents: + if parent == dest: + break + if os.path.normcase(str(parent)) in file_targets: + raise ValueError(f"archive file entry conflicts with child path: {m.filename!r}") + + for m, _ in validated: + zf.extract(m, dest) + + +def rezip(src_dir: Path, out_path: Path) -> None: + files = sorted(p for p in src_dir.rglob("*") if p.is_file()) + ct = src_dir / "[Content_Types].xml" + fd, tmp_name = tempfile.mkstemp( + prefix=out_path.name + ".", suffix=".tmp", dir=out_path.parent + ) + tmp_out = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as fh: + with zipfile.ZipFile(fh, "w", zipfile.ZIP_DEFLATED) as zf: + if ct.exists(): + zf.write(ct, ct.relative_to(src_dir), compress_type=zipfile.ZIP_STORED) + for f in files: + if f == ct: + continue + zf.write(f, f.relative_to(src_dir)) + if out_path.exists(): + mode = out_path.stat().st_mode & 0o777 + else: + umask = os.umask(0) + os.umask(umask) + mode = 0o666 & ~umask + os.chmod(tmp_out, mode) + os.replace(tmp_out, out_path) + finally: + if tmp_out.exists(): + tmp_out.unlink() diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/merge_runs.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/merge_runs.py deleted file mode 100644 index ad7c25eec0..0000000000 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/merge_runs.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Merge adjacent runs with identical formatting in DOCX. - -Merges adjacent elements that have identical properties. -Works on runs in paragraphs and inside tracked changes (, ). - -Also: -- Removes rsid attributes from runs (revision metadata that doesn't affect rendering) -- Removes proofErr elements (spell/grammar markers that block merging) -""" - -from pathlib import Path - -import defusedxml.minidom - - -def merge_runs(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - _remove_elements(root, "proofErr") - _strip_run_rsid_attrs(root) - - containers = {run.parentNode for run in _find_elements(root, "r")} - - merge_count = 0 - for container in containers: - merge_count += _merge_runs_in(container) - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Merged {merge_count} runs" - - except Exception as e: - return 0, f"Error: {e}" - - - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def _get_child(parent, tag: str): - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - return child - return None - - -def _get_children(parent, tag: str) -> list: - results = [] - for child in parent.childNodes: - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(child) - return results - - -def _is_adjacent(elem1, elem2) -> bool: - node = elem1.nextSibling - while node: - if node == elem2: - return True - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - return False - - - - -def _remove_elements(root, tag: str): - for elem in _find_elements(root, tag): - if elem.parentNode: - elem.parentNode.removeChild(elem) - - -def _strip_run_rsid_attrs(root): - for run in _find_elements(root, "r"): - for attr in list(run.attributes.values()): - if "rsid" in attr.name.lower(): - run.removeAttribute(attr.name) - - - - -def _merge_runs_in(container) -> int: - merge_count = 0 - run = _first_child_run(container) - - while run: - while True: - next_elem = _next_element_sibling(run) - if next_elem and _is_run(next_elem) and _can_merge(run, next_elem): - _merge_run_content(run, next_elem) - container.removeChild(next_elem) - merge_count += 1 - else: - break - - _consolidate_text(run) - run = _next_sibling_run(run) - - return merge_count - - -def _first_child_run(container): - for child in container.childNodes: - if child.nodeType == child.ELEMENT_NODE and _is_run(child): - return child - return None - - -def _next_element_sibling(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - return sibling - sibling = sibling.nextSibling - return None - - -def _next_sibling_run(node): - sibling = node.nextSibling - while sibling: - if sibling.nodeType == sibling.ELEMENT_NODE: - if _is_run(sibling): - return sibling - sibling = sibling.nextSibling - return None - - -def _is_run(node) -> bool: - name = node.localName or node.tagName - return name == "r" or name.endswith(":r") - - -def _can_merge(run1, run2) -> bool: - rpr1 = _get_child(run1, "rPr") - rpr2 = _get_child(run2, "rPr") - - if (rpr1 is None) != (rpr2 is None): - return False - if rpr1 is None: - return True - return rpr1.toxml() == rpr2.toxml() - - -def _merge_run_content(target, source): - for child in list(source.childNodes): - if child.nodeType == child.ELEMENT_NODE: - name = child.localName or child.tagName - if name != "rPr" and not name.endswith(":rPr"): - target.appendChild(child) - - -def _consolidate_text(run): - t_elements = _get_children(run, "t") - - for i in range(len(t_elements) - 1, 0, -1): - curr, prev = t_elements[i], t_elements[i - 1] - - if _is_adjacent(prev, curr): - prev_text = prev.firstChild.data if prev.firstChild else "" - curr_text = curr.firstChild.data if curr.firstChild else "" - merged = prev_text + curr_text - - if prev.firstChild: - prev.firstChild.data = merged - else: - prev.appendChild(run.ownerDocument.createTextNode(merged)) - - if merged.startswith(" ") or merged.endswith(" "): - prev.setAttribute("xml:space", "preserve") - elif prev.hasAttribute("xml:space"): - prev.removeAttribute("xml:space") - - run.removeChild(curr) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_chart.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_chart.py new file mode 100644 index 0000000000..209cb7c58b --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_chart.py @@ -0,0 +1,170 @@ +"""Find chart XML that PowerPoint refuses but the schema accepts. + +Detection only: for either fault more than one repair is valid, and only the +author knows which was meant. +""" + + +from __future__ import annotations + +import re +from typing import Mapping + +from . import part_text + + +_CHART_PART_RE = re.compile(r"ppt/charts/chart\d+\.xml") + +_GROUPING_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") +_DLBL_POS_RE = re.compile(r"""]*?\bval=["'](\w+)["']""") + +def _strip_ext_lst(text: str) -> str: + out, cursor = [], 0 + for lo, hi in _ext_lst_spans(text): + out.append(text[cursor:lo]) + cursor = hi + out.append(text[cursor:]) + return "".join(out) + +_BAR_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +STACKED_GROUPINGS = frozenset({"stacked", "percentStacked"}) +ILLEGAL_ON_STACKED = frozenset({"outEnd"}) +LEGAL_ON_STACKED = ("ctr", "inEnd", "inBase") + + +def _check_stacked_label_positions(part: str, xml: str) -> list[str]: + problems: list[str] = [] + for match in _BAR_GROUP_RE.finditer(xml): + block = _strip_ext_lst(match.group(0)) + group = match.group(1) + + grouping = _GROUPING_RE.search(block) + if grouping is None or grouping.group(1) not in STACKED_GROUPINGS: + continue + + bad = [p for p in _DLBL_POS_RE.findall(block) if p in ILLEGAL_ON_STACKED] + for pos in sorted(set(bad)): + problems.append( + f'{part}: {bad.count(pos)} data label(s) use dLblPos="{pos}" on a ' + f"{grouping.group(1)} {group}; PowerPoint allows only " + f"{', '.join(LEGAL_ON_STACKED)} there" + ) + return problems + + + +_ANY_CHART_GROUP_RE = re.compile(r"]*(?.*?", re.DOTALL) + +_AXID_RE = re.compile( + r"""\s*]*?\bval=["'](-?\d+)["']\s*(?:/>|>\s*)""" +) + +_AXIS_DECL_RE = re.compile( + r"""]*(?\s*]*?\bval=["'](-?\d+)["']""" +) + +AXID_LIMIT = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 3, "line3DChart": 3, "area3DChart": 3, + "surfaceChart": 3, "surface3DChart": 3, +} + +AXID_MINIMUM = { + "barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2, + "bubbleChart": 2, "radarChart": 2, "stockChart": 2, + "bar3DChart": 2, "area3DChart": 2, "surfaceChart": 2, + "line3DChart": 3, "surface3DChart": 3, +} + + +def _declared_axes(xml: str) -> dict[str, list[str]]: + axes: dict[str, list[str]] = {} + for kind, axid in _AXIS_DECL_RE.findall(xml): + axes.setdefault(kind, []).append(axid) + return axes + + +def _canonical_ids(axes: dict[str, list[str]], limit: int) -> list[str] | None: + category = axes.get("catAx", []) + axes.get("dateAx", []) + value = axes.get("valAx", []) + series = axes.get("serAx", []) + if len(category) != 1 or len(value) != 1 or len(series) > 1: + return None + ids = [category[0], value[0]] + if limit >= 3 and series: + ids.append(series[0]) + return ids + + +def _undeclared_axes(kind: str, block: str, axes: dict[str, list[str]]) -> list[str] | None: + if kind not in AXID_LIMIT: + return None + ids = _AXID_RE.findall(block) + declared = {i for group in axes.values() for i in group} + if len([i for i in ids if i in declared]) >= 2: + return None + return ids + + +def _check_chart_axis_references(part: str, xml: str) -> list[str]: + axes = _declared_axes(xml) + problems: list[str] = [] + declared = {i for group in axes.values() for i in group} + for match in _ANY_CHART_GROUP_RE.finditer(xml): + kind, block = match.group(1), match.group(0) + ids = _undeclared_axes(kind, block, axes) + if ids is None: + continue + if not ids: + problems.append( + f"{part}: declares no this part can resolve; a chart " + f"group needs {AXID_MINIMUM[kind]}, and PowerPoint discards one with fewer" + ) + continue + dead = [i for i in ids if i not in declared] + canonical = _canonical_ids(axes, AXID_LIMIT[kind]) + if canonical is not None and len(canonical) >= AXID_MINIMUM[kind]: + hint = f"Fix: point them at the axes this part declares ({', '.join(canonical)})" + else: + hint = ("Fix: the part declares several axes of a kind -- declare the " + "secondary axes the series expects, or drop them") + detail = (f"of which {', '.join(dead)} name no declared axis" + if dead else f"only {len(ids)} of which this part declares") + problems.append( + f"{part}: references axId {', '.join(ids)}, {detail}, " + f"leaving fewer than two live axes; PowerPoint discards the chart. {hint}" + ) + return problems + + +def _ext_lst_spans(text: str) -> list[tuple[int, int]]: + spans: list[tuple[int, int]] = [] + depth = 0 + start = 0 + for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text): + closing, self_closing = match.group(1), match.group(2) + if self_closing: + continue + if closing: + depth -= 1 + if depth == 0: + spans.append((start, match.end())) + else: + if depth == 0: + start = match.start() + depth += 1 + return spans + + +CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references) + + +def find_chart_problems(files: Mapping[str, bytes]) -> list[str]: + problems: list[str] = [] + for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)): + xml = part_text(files[part]) + for check in CHART_CHECKS: + problems.extend(check(part, xml)) + return problems diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_slide.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_slide.py new file mode 100644 index 0000000000..22f9aee0ff --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_slide.py @@ -0,0 +1,60 @@ +"""Pick the slide-XML schema errors PowerPoint refuses the file over. + +A denylist over lxml's messages, so an unrecognised error class is a miss rather +than a false alarm. +""" + + +from __future__ import annotations + +import re + +SLIDE_PART_RE = re.compile( + r"ppt/(slides|slideLayouts|slideMasters|notesSlides|notesMasters|handoutMasters)" + r"/[^/]+\.xml" +) + +FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"\}tableStyleId': This element is not expected"), + "two in one (the schema allows one)", + ), + ( + re.compile(r"\}srgbClr', attribute 'val'"), + "a colour that is not six hex digits", + ), + ( + re.compile(r"\}txBody': Missing child element"), + "a with no children", + ), + ( + re.compile(r"\}miter', attribute 'lim'"), + 'a line join with lim="NaN"', + ), + ( + re.compile(r"\}uLnTx': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}overrideClrMapping': This element is not expected"), + " in a position the schema forbids", + ), + ( + re.compile(r"\}nvGrpSpPr': Missing child element"), + "a with no children", + ), +) + + +def is_schema_verdict(error: str) -> bool: + return error.startswith("Element ") + + +def fatal_slide_errors(errors: set[str]) -> list[str]: + out = [] + for error in sorted(errors): + for pattern, meaning in FATAL_SLIDE_ERRORS: + if pattern.search(error): + out.append(f"{meaning}: {error}") + break + return out diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_theme.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_theme.py new file mode 100644 index 0000000000..5ef4c3e835 --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/pptx_theme.py @@ -0,0 +1,114 @@ +"""Find masters sharing a theme part in the way PowerPoint refuses to open. + +Reports only; the fix is to move back to directly after + in ppt/presentation.xml. +""" + + +from __future__ import annotations + +import posixpath +import re +from typing import Mapping + +from . import part_text + +THEME_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" + +_MASTER_RE = re.compile( + r"^ppt/(?PslideMasters|notesMasters|handoutMasters)/" + r"(?:slide|notes|handout)Master(?P\d+)\.xml$" +) +_GROUP_ORDER = {"slideMasters": 0, "notesMasters": 1, "handoutMasters": 2} + +_RELATIONSHIP_RE = re.compile( + r"]*?(?:/>|>.*?)", re.DOTALL +) + + +def _sort_key(name: str) -> tuple[int, int]: + m = _MASTER_RE.match(name) + assert m is not None + return (_GROUP_ORDER[m.group("group")], int(m.group("num"))) + + +def _rels_path(part: str) -> str: + directory, base = posixpath.split(part) + return f"{directory}/_rels/{base}.rels" + + +def _resolve(rels_path: str, target: str) -> str: + if target.startswith("/"): + return target.lstrip("/") + part_dir = posixpath.dirname(posixpath.dirname(rels_path)) + return posixpath.normpath(posixpath.join(part_dir, target)) + + +def _theme_rel(files: Mapping[str, bytes], master: str): + rels_path = _rels_path(master) + rels = files.get(rels_path) + if rels is None: + return None + for element in _RELATIONSHIP_RE.findall(part_text(rels)): + if f'Type="{THEME_REL_TYPE}"' not in element: + continue + target = re.search(r'\bTarget="([^"]+)"', element) + if target is None: + continue + return rels_path, element, _resolve(rels_path, target.group(1)) + return None + + +def _masters(files: Mapping[str, bytes]) -> list[str]: + return sorted((n for n in files if _MASTER_RE.match(n)), key=_sort_key) + + +_PRESENTATION = "ppt/presentation.xml" +_NOTES_MASTERS = "ppt/notesMasters/" +_IGNORABLE_RE = re.compile(r"|<\?.*?\?>", re.DOTALL) +_AFTER_SLDIDLST_RE = re.compile( + r"]*/>|[^>]*>.*?)\s*(<[^>\s/]+)", re.DOTALL +) + + +def _notes_master_share_is_inert(files: Mapping[str, bytes]) -> bool: + data = files.get(_PRESENTATION) + if data is None: + return False + match = _AFTER_SLDIDLST_RE.search(_IGNORABLE_RE.sub("", part_text(data))) + return match is not None and match.group(1) == " bool: + return inert_notes and master.startswith(_NOTES_MASTERS) + + +def find_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + ] + + +def live_shared_master_themes(files: Mapping[str, bytes]) -> list[str]: + inert_notes = _notes_master_share_is_inert(files) + return [ + f"{master} shares {theme} with {first}" + for master, _, _, theme, first in _shares(files) + if not _is_inert(master, inert_notes) + ] diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/simplify_redlines.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/simplify_redlines.py deleted file mode 100644 index db963bb998..0000000000 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/helpers/simplify_redlines.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Simplify tracked changes by merging adjacent w:ins or w:del elements. - -Merges adjacent elements from the same author into a single element. -Same for elements. This makes heavily-redlined documents easier to -work with by reducing the number of tracked change wrappers. - -Rules: -- Only merges w:ins with w:ins, w:del with w:del (same element type) -- Only merges if same author (ignores timestamp differences) -- Only merges if truly adjacent (only whitespace between them) -""" - -import xml.etree.ElementTree as ET -import zipfile -from pathlib import Path - -import defusedxml.minidom - -WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" - - -def simplify_redlines(input_dir: str) -> tuple[int, str]: - doc_xml = Path(input_dir) / "word" / "document.xml" - - if not doc_xml.exists(): - return 0, f"Error: {doc_xml} not found" - - try: - dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8")) - root = dom.documentElement - - merge_count = 0 - - containers = _find_elements(root, "p") + _find_elements(root, "tc") - - for container in containers: - merge_count += _merge_tracked_changes_in(container, "ins") - merge_count += _merge_tracked_changes_in(container, "del") - - doc_xml.write_bytes(dom.toxml(encoding="UTF-8")) - return merge_count, f"Simplified {merge_count} tracked changes" - - except Exception as e: - return 0, f"Error: {e}" - - -def _merge_tracked_changes_in(container, tag: str) -> int: - merge_count = 0 - - tracked = [ - child - for child in container.childNodes - if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag) - ] - - if len(tracked) < 2: - return 0 - - i = 0 - while i < len(tracked) - 1: - curr = tracked[i] - next_elem = tracked[i + 1] - - if _can_merge_tracked(curr, next_elem): - _merge_tracked_content(curr, next_elem) - container.removeChild(next_elem) - tracked.pop(i + 1) - merge_count += 1 - else: - i += 1 - - return merge_count - - -def _is_element(node, tag: str) -> bool: - name = node.localName or node.tagName - return name == tag or name.endswith(f":{tag}") - - -def _get_author(elem) -> str: - author = elem.getAttribute("w:author") - if not author: - for attr in elem.attributes.values(): - if attr.localName == "author" or attr.name.endswith(":author"): - return attr.value - return author - - -def _can_merge_tracked(elem1, elem2) -> bool: - if _get_author(elem1) != _get_author(elem2): - return False - - node = elem1.nextSibling - while node and node != elem2: - if node.nodeType == node.ELEMENT_NODE: - return False - if node.nodeType == node.TEXT_NODE and node.data.strip(): - return False - node = node.nextSibling - - return True - - -def _merge_tracked_content(target, source): - while source.firstChild: - child = source.firstChild - source.removeChild(child) - target.appendChild(child) - - -def _find_elements(root, tag: str) -> list: - results = [] - - def traverse(node): - if node.nodeType == node.ELEMENT_NODE: - name = node.localName or node.tagName - if name == tag or name.endswith(f":{tag}"): - results.append(node) - for child in node.childNodes: - traverse(child) - - traverse(root) - return results - - -def get_tracked_change_authors(doc_xml_path: Path) -> dict[str, int]: - if not doc_xml_path.exists(): - return {} - - try: - tree = ET.parse(doc_xml_path) - root = tree.getroot() - except ET.ParseError: - return {} - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - - return authors - - -def _get_authors_from_docx(docx_path: Path) -> dict[str, int]: - try: - with zipfile.ZipFile(docx_path, "r") as zf: - if "word/document.xml" not in zf.namelist(): - return {} - with zf.open("word/document.xml") as f: - tree = ET.parse(f) - root = tree.getroot() - - namespaces = {"w": WORD_NS} - author_attr = f"{{{WORD_NS}}}author" - - authors: dict[str, int] = {} - for tag in ["ins", "del"]: - for elem in root.findall(f".//w:{tag}", namespaces): - author = elem.get(author_attr) - if author: - authors[author] = authors.get(author, 0) + 1 - return authors - except (zipfile.BadZipFile, ET.ParseError): - return {} - - -def infer_author(modified_dir: Path, original_docx: Path, default: str = "Claude") -> str: - modified_xml = modified_dir / "word" / "document.xml" - modified_authors = get_tracked_change_authors(modified_xml) - - if not modified_authors: - return default - - original_authors = _get_authors_from_docx(original_docx) - - new_changes: dict[str, int] = {} - for author, count in modified_authors.items(): - original_count = original_authors.get(author, 0) - diff = count - original_count - if diff > 0: - new_changes[author] = diff - - if not new_changes: - return default - - if len(new_changes) == 1: - return next(iter(new_changes)) - - raise ValueError( - f"Multiple authors added new changes: {new_changes}. " - "Cannot infer which author to validate." - ) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/pack.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/pack.py deleted file mode 100755 index db29ed8b1c..0000000000 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/pack.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Pack a directory into a DOCX, PPTX, or XLSX file. - -Validates with auto-repair, condenses XML formatting, and creates the Office file. - -Usage: - python pack.py [--original ] [--validate true|false] - -Examples: - python pack.py unpacked/ output.docx --original input.docx - python pack.py unpacked/ output.pptx --validate false -""" - -import argparse -import sys -import shutil -import tempfile -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator - -def pack( - input_directory: str, - output_file: str, - original_file: str | None = None, - validate: bool = True, - infer_author_func=None, -) -> tuple[None, str]: - input_dir = Path(input_directory) - output_path = Path(output_file) - suffix = output_path.suffix.lower() - - if not input_dir.is_dir(): - return None, f"Error: {input_dir} is not a directory" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {output_file} must be a .docx, .pptx, or .xlsx file" - - if validate and original_file: - original_path = Path(original_file) - if original_path.exists(): - success, output = _run_validation( - input_dir, original_path, suffix, infer_author_func - ) - if output: - print(output) - if not success: - return None, f"Error: Validation failed for {input_dir}" - - with tempfile.TemporaryDirectory() as temp_dir: - temp_content_dir = Path(temp_dir) / "content" - shutil.copytree(input_dir, temp_content_dir) - - for pattern in ["*.xml", "*.rels"]: - for xml_file in temp_content_dir.rglob(pattern): - _condense_xml(xml_file) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf: - for f in temp_content_dir.rglob("*"): - if f.is_file(): - zf.write(f, f.relative_to(temp_content_dir)) - - return None, f"Successfully packed {input_dir} to {output_file}" - - -def _run_validation( - unpacked_dir: Path, - original_file: Path, - suffix: str, - infer_author_func=None, -) -> tuple[bool, str | None]: - output_lines = [] - validators = [] - - if suffix == ".docx": - author = "Claude" - if infer_author_func: - try: - author = infer_author_func(unpacked_dir, original_file) - except ValueError as e: - print(f"Warning: {e} Using default author 'Claude'.", file=sys.stderr) - - validators = [ - DOCXSchemaValidator(unpacked_dir, original_file), - RedliningValidator(unpacked_dir, original_file, author=author), - ] - elif suffix == ".pptx": - validators = [PPTXSchemaValidator(unpacked_dir, original_file)] - - if not validators: - return True, None - - total_repairs = sum(v.repair() for v in validators) - if total_repairs: - output_lines.append(f"Auto-repaired {total_repairs} issue(s)") - - success = all(v.validate() for v in validators) - - if success: - output_lines.append("All validations PASSED!") - - return success, "\n".join(output_lines) if output_lines else None - - -def _condense_xml(xml_file: Path) -> None: - try: - with open(xml_file, encoding="utf-8") as f: - dom = defusedxml.minidom.parse(f) - - for element in dom.getElementsByTagName("*"): - if element.tagName.endswith(":t"): - continue - - for child in list(element.childNodes): - if ( - child.nodeType == child.TEXT_NODE - and child.nodeValue - and child.nodeValue.strip() == "" - ) or child.nodeType == child.COMMENT_NODE: - element.removeChild(child) - - xml_file.write_bytes(dom.toxml(encoding="UTF-8")) - except Exception as e: - print(f"ERROR: Failed to parse {xml_file.name}: {e}", file=sys.stderr) - raise - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Pack a directory into a DOCX, PPTX, or XLSX file" - ) - parser.add_argument("input_directory", help="Unpacked Office document directory") - parser.add_argument("output_file", help="Output Office file (.docx/.pptx/.xlsx)") - parser.add_argument( - "--original", - help="Original file for validation comparison", - ) - parser.add_argument( - "--validate", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Run validation with auto-repair (default: true)", - ) - args = parser.parse_args() - - _, message = pack( - args.input_directory, - args.output_file, - original_file=args.original, - validate=args.validate, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/soffice.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/soffice.py index c7f7e3289f..0b4c99deca 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/soffice.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/soffice.py @@ -4,20 +4,23 @@ at runtime and applies an LD_PRELOAD shim if needed. Usage: - from office.soffice import run_soffice, get_soffice_env + from office.soffice import run_soffice - # Option 1 – run soffice directly result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"]) - # Option 2 – get env dict for your own subprocess calls - env = get_soffice_env() - subprocess.run(["soffice", ...], env=env) +Call soffice through run_soffice, not through subprocess with get_soffice_env(): +the env dict carries the shim but names no user profile, and a non-root sandbox +cannot bootstrap the default one -- soffice aborts with "User installation could +not be completed" and converts nothing. get_soffice_env() stays public for the +callers that build their own argv (they must pass -env:UserInstallation too). """ +import contextlib import os import socket import subprocess import tempfile +from collections.abc import Iterable from pathlib import Path @@ -32,9 +35,15 @@ def get_soffice_env() -> dict: return env -def run_soffice(args: list[str], **kwargs) -> subprocess.CompletedProcess: - env = get_soffice_env() - return subprocess.run(["soffice"] + args, env=env, **kwargs) +def run_soffice(args: Iterable[str], **kwargs) -> subprocess.CompletedProcess: + args = list(args) + with contextlib.ExitStack() as stack: + if not any(str(a).startswith("-env:UserInstallation") for a in args): + profile = stack.enter_context( + tempfile.TemporaryDirectory(prefix="lo_profile_", ignore_cleanup_errors=True) + ) + args = [f"-env:UserInstallation={Path(profile).as_uri()}"] + args + return subprocess.run(["soffice"] + args, env=get_soffice_env(), **kwargs) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/unpack.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/unpack.py deleted file mode 100755 index 00152533ac..0000000000 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/unpack.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Unpack Office files (DOCX, PPTX, XLSX) for editing. - -Extracts the ZIP archive, pretty-prints XML files, and optionally: -- Merges adjacent runs with identical formatting (DOCX only) -- Simplifies adjacent tracked changes from same author (DOCX only) - -Usage: - python unpack.py [options] - -Examples: - python unpack.py document.docx unpacked/ - python unpack.py presentation.pptx unpacked/ - python unpack.py document.docx unpacked/ --merge-runs false -""" - -import argparse -import sys -import zipfile -from pathlib import Path - -import defusedxml.minidom - -from helpers.merge_runs import merge_runs as do_merge_runs -from helpers.simplify_redlines import simplify_redlines as do_simplify_redlines - -SMART_QUOTE_REPLACEMENTS = { - "\u201c": "“", - "\u201d": "”", - "\u2018": "‘", - "\u2019": "’", -} - - -def unpack( - input_file: str, - output_directory: str, - merge_runs: bool = True, - simplify_redlines: bool = True, -) -> tuple[None, str]: - input_path = Path(input_file) - output_path = Path(output_directory) - suffix = input_path.suffix.lower() - - if not input_path.exists(): - return None, f"Error: {input_file} does not exist" - - if suffix not in {".docx", ".pptx", ".xlsx"}: - return None, f"Error: {input_file} must be a .docx, .pptx, or .xlsx file" - - try: - output_path.mkdir(parents=True, exist_ok=True) - - with zipfile.ZipFile(input_path, "r") as zf: - zf.extractall(output_path) - - xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels")) - for xml_file in xml_files: - _pretty_print_xml(xml_file) - - message = f"Unpacked {input_file} ({len(xml_files)} XML files)" - - if suffix == ".docx": - if simplify_redlines: - simplify_count, _ = do_simplify_redlines(str(output_path)) - message += f", simplified {simplify_count} tracked changes" - - if merge_runs: - merge_count, _ = do_merge_runs(str(output_path)) - message += f", merged {merge_count} runs" - - for xml_file in xml_files: - _escape_smart_quotes(xml_file) - - return None, message - - except zipfile.BadZipFile: - return None, f"Error: {input_file} is not a valid Office file" - except Exception as e: - return None, f"Error unpacking: {e}" - - -def _pretty_print_xml(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - dom = defusedxml.minidom.parseString(content) - xml_file.write_bytes(dom.toprettyxml(indent=" ", encoding="utf-8")) - except Exception: - pass - - -def _escape_smart_quotes(xml_file: Path) -> None: - try: - content = xml_file.read_text(encoding="utf-8") - for char, entity in SMART_QUOTE_REPLACEMENTS.items(): - content = content.replace(char, entity) - xml_file.write_text(content, encoding="utf-8") - except Exception: - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Unpack an Office file (DOCX, PPTX, XLSX) for editing" - ) - parser.add_argument("input_file", help="Office file to unpack") - parser.add_argument("output_directory", help="Output directory") - parser.add_argument( - "--merge-runs", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent runs with identical formatting (DOCX only, default: true)", - ) - parser.add_argument( - "--simplify-redlines", - type=lambda x: x.lower() == "true", - default=True, - metavar="true|false", - help="Merge adjacent tracked changes from same author (DOCX only, default: true)", - ) - args = parser.parse_args() - - _, message = unpack( - args.input_file, - args.output_directory, - merge_runs=args.merge_runs, - simplify_redlines=args.simplify_redlines, - ) - print(message) - - if "Error" in message: - sys.exit(1) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validate.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validate.py index 03b01f6e3b..8fbd2f71ca 100755 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validate.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validate.py @@ -6,7 +6,7 @@ The first argument can be either: - An unpacked directory containing the Office document XML files -- A packed Office file (.docx/.pptx/.xlsx) which will be unpacked to a temp directory +- A packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx template) which will be unpacked to a temp directory Auto-repair fixes: - paraId/durableId values that exceed OOXML limits @@ -19,20 +19,43 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import OOXML_FAMILY, rezip, safe_extract from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator +WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + +def _fail(message: str): + print(f"Error: {message}", file=sys.stderr) + sys.exit(2) + + +def _has_tracked_changes(unpacked_dir: Path) -> bool: + document = unpacked_dir / "word" / "document.xml" + if not document.is_file(): + return False + try: + root = ET.parse(document).getroot() + except (ET.ParseError, DefusedXmlException): + return False + tracked = {f"{{{WORD_NS}}}ins", f"{{{WORD_NS}}}del"} + return any(elem.tag in tracked for elem in root.iter()) + def main(): parser = argparse.ArgumentParser(description="Validate Office document XML files") parser.add_argument( "path", - help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx)", + help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx)", ) parser.add_argument( "--original", required=False, default=None, - help="Path to original file (.docx/.pptx/.xlsx). If omitted, all XSD errors are reported and redlining validation is skipped.", + help="Path to original file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx). If omitted, all XSD errors are reported and redlining validation is skipped.", ) parser.add_argument( "-v", @@ -43,63 +66,102 @@ def main(): parser.add_argument( "--auto-repair", action="store_true", - help="Automatically repair common issues (hex IDs, whitespace preservation)", + help="Automatically repair common issues (hex IDs, whitespace preservation). " + "Modifies the input in place: repairs to a packed file are written back to it.", ) parser.add_argument( "--author", - default="Claude", - help="Author name for redlining validation (default: Claude)", + default=None, + help="The name you are redlining under. Passing it turns on the " + "tracked-change check: any text differing from --original without a " + "/ recording it is reported. Untracked edits carry no " + "author, so the check covers them whoever made them — the name marks " + "the run as redlining work and is not used to filter. Requires " + "--original; docx only.", ) args = parser.parse_args() + if args.author is not None and not args.original: + _fail("--author requires --original") + path = Path(args.path) - assert path.exists(), f"Error: {path} does not exist" + if not path.exists(): + _fail(f"{path} does not exist") original_file = None if args.original: original_file = Path(args.original) - assert original_file.is_file(), f"Error: {original_file} is not a file" - assert original_file.suffix.lower() in [".docx", ".pptx", ".xlsx"], ( - f"Error: {original_file} must be a .docx, .pptx, or .xlsx file" + if not original_file.is_file(): + _fail(f"{original_file} is not a file") + if original_file.suffix.lower() not in OOXML_FAMILY: + _fail(f"{original_file} must be one of: {', '.join(sorted(OOXML_FAMILY))}") + + family = OOXML_FAMILY.get((original_file or path).suffix.lower()) + if family is None: + _fail( + f"Cannot determine file type from {path}. Use --original or provide one of: {', '.join(sorted(OOXML_FAMILY))}." ) - file_extension = (original_file or path).suffix.lower() - assert file_extension in [".docx", ".pptx", ".xlsx"], ( - f"Error: Cannot determine file type from {path}. Use --original or provide a .docx/.pptx/.xlsx file." - ) - - if path.is_file() and path.suffix.lower() in [".docx", ".pptx", ".xlsx"]: - temp_dir = tempfile.mkdtemp() - with zipfile.ZipFile(path, "r") as zf: - zf.extractall(temp_dir) - unpacked_dir = Path(temp_dir) + if args.author is not None and family != "docx": + _fail(f"--author only applies to docx files, not {family}") + + packed_file = None + temp_dir_ctx = None + if path.is_file() and path.suffix.lower() in OOXML_FAMILY: + packed_file = path + temp_dir_ctx = tempfile.TemporaryDirectory() + unpacked_dir = Path(temp_dir_ctx.name) + try: + with zipfile.ZipFile(path, "r") as zf: + safe_extract(zf, unpacked_dir) + except (zipfile.BadZipFile, ValueError, OSError) as e: + _fail(f"cannot unpack {path}: {e}") else: - assert path.is_dir(), f"Error: {path} is not a directory or Office file" + if not path.is_dir(): + _fail(f"{path} is not a directory or Office file") unpacked_dir = path - match file_extension: - case ".docx": + match family: + case "docx": validators = [ DOCXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] - if original_file: + if args.author is not None: validators.append( - RedliningValidator(unpacked_dir, original_file, verbose=args.verbose, author=args.author) + RedliningValidator(unpacked_dir, original_file, verbose=args.verbose) + ) + elif original_file and _has_tracked_changes(unpacked_dir): + print( + "Note: this document has tracked changes; they were not " + "checked against the original (pass --author to check)." ) - case ".pptx": + case "pptx": validators = [ PPTXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose), ] + case "xlsx": + exts = ", ".join(k for k, v in sorted(OOXML_FAMILY.items()) if v == "xlsx") + print( + f"No XSD schema validation is performed for xlsx-family files ({exts}). " + "For formula-error checking, use scripts/recalc.py instead." + ) + sys.exit(0) case _: - print(f"Error: Validation not supported for file type {file_extension}") + print(f"Error: Validation not supported for file type {family}") sys.exit(1) if args.auto_repair: total_repairs = sum(v.repair() for v in validators) if total_repairs: print(f"Auto-repaired {total_repairs} issue(s)") + if packed_file is not None: + rezip(unpacked_dir, packed_file) + print(f"Wrote repaired file to {packed_file}") + + success = all([v.validate() for v in validators]) - success = all(v.validate() for v in validators) + if temp_dir_ctx is not None: + temp_dir_ctx.cleanup() if success: print("All validations PASSED!") diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/base.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/base.py index db4a06a229..19d52a7fe0 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/base.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/base.py @@ -6,8 +6,20 @@ from pathlib import Path import defusedxml.minidom +from functools import lru_cache + import lxml.etree +from helpers import safe_extract + + +@lru_cache(maxsize=None) +def _load_schema(schema_path: str): + with open(schema_path, "rb") as xsd_file: + xsd_doc = lxml.etree.parse( + xsd_file, parser=lxml.etree.XMLParser(), base_url=schema_path + ) + return lxml.etree.XMLSchema(xsd_doc) class BaseSchemaValidator: @@ -119,21 +131,28 @@ def repair_whitespace_preservation(self) -> int: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) - modified = False + pending = [] for elem in dom.getElementsByTagName("*"): - if elem.tagName.endswith(":t") and elem.firstChild: - text = elem.firstChild.nodeValue - if text and (text.startswith((' ', '\t')) or text.endswith((' ', '\t'))): + local_name = elem.tagName.rsplit(":", 1)[-1] + if local_name in ("t", "delText", "instrText", "delInstrText"): + text = "".join( + child.data + for child in elem.childNodes + if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE) + ) + ws = (" ", "\t", "\n", "\r") + if text and (text.startswith(ws) or text.endswith(ws)): if elem.getAttribute("xml:space") != "preserve": elem.setAttribute("xml:space", "preserve") text_preview = repr(text[:30]) + "..." if len(text) > 30 else repr(text) - print(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - repairs += 1 - modified = True + pending.append(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}") - if modified: + if pending: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass @@ -212,6 +231,8 @@ def validate_unique_ids(self): elem.getparent().remove(elem) for elem in root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue tag = ( elem.tag.split("}")[-1].lower() if "}" in elem.tag @@ -326,6 +347,8 @@ def validate_file_references(self): namespaces={"ns": self.PACKAGE_RELATIONSHIPS_NAMESPACE}, ): target = rel.get("Target") + if rel.get("TargetMode") == "External": + continue if target and not target.startswith( ("http", "mailto:") ): @@ -423,6 +446,8 @@ def validate_all_relationship_ids(self): r_ns = self.OFFICE_RELATIONSHIPS_NAMESPACE rid_attrs_to_check = ["id", "embed", "link"] for elem in xml_root.iter(): + if not hasattr(elem, "tag") or callable(elem.tag): + continue for attr_name in rid_attrs_to_check: rid_attr = elem.get(f"{{{r_ns}}}{attr_name}") if not rid_attr: @@ -747,18 +772,16 @@ def _preprocess_for_mc_ignorable(self, xml_doc): return xml_doc - def _validate_single_file_xsd(self, xml_file, base_path): - schema_path = self._get_schema_path(xml_file) + def _preprocess_for_schema(self, xml_doc, relative_path): + return xml_doc + + def _validate_single_file_xsd(self, xml_file, base_path, schema_path=None): + schema_path = schema_path or self._get_schema_path(xml_file) if not schema_path: return None, None try: - with open(schema_path, "rb") as xsd_file: - parser = lxml.etree.XMLParser() - xsd_doc = lxml.etree.parse( - xsd_file, parser=parser, base_url=str(schema_path) - ) - schema = lxml.etree.XMLSchema(xsd_doc) + schema = _load_schema(str(schema_path)) with open(xml_file, "r") as f: xml_doc = lxml.etree.parse(f) @@ -773,6 +796,8 @@ def _validate_single_file_xsd(self, xml_file, base_path): ): xml_doc = self._clean_ignorable_namespaces(xml_doc) + xml_doc = self._preprocess_for_schema(xml_doc, relative_path) + if schema.validate(xml_doc): return True, set() else: @@ -784,7 +809,7 @@ def _validate_single_file_xsd(self, xml_file, base_path): except Exception as e: return False, {str(e)} - def _get_original_file_errors(self, xml_file): + def _get_original_file_errors(self, xml_file, schema_path=None): if self.original_file is None: return set() @@ -798,8 +823,11 @@ def _get_original_file_errors(self, xml_file): with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - with zipfile.ZipFile(self.original_file, "r") as zip_ref: - zip_ref.extractall(temp_path) + try: + with zipfile.ZipFile(self.original_file, "r") as zip_ref: + safe_extract(zip_ref, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() original_xml_file = temp_path / relative_path @@ -807,7 +835,7 @@ def _get_original_file_errors(self, xml_file): return set() is_valid, errors = self._validate_single_file_xsd( - original_xml_file, temp_path + original_xml_file, temp_path, schema_path=schema_path ) return errors if errors else set() diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/docx.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/docx.py index fec405e694..0d18b6979a 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/docx.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/docx.py @@ -6,10 +6,13 @@ import re import tempfile import zipfile +from pathlib import Path import defusedxml.minidom import lxml.etree +from helpers import safe_extract + from .base import BaseSchemaValidator @@ -186,7 +189,7 @@ def count_paragraphs_in_original(self): try: with tempfile.TemporaryDirectory() as temp_dir: with zipfile.ZipFile(original, "r") as zip_ref: - zip_ref.extractall(temp_dir) + safe_extract(zip_ref, Path(temp_dir)) doc_xml_path = temp_dir + "/word/document.xml" root = lxml.etree.parse(doc_xml_path).getroot() @@ -241,9 +244,12 @@ def validate_insertions(self): return True def compare_paragraph_counts(self): - original_count = self.count_paragraphs_in_original() new_count = self.count_paragraphs_in_unpacked() + if self.original_file is None: + print(f"\nParagraphs: {new_count}") + return + original_count = self.count_paragraphs_in_original() diff = new_count - original_count diff_str = f"+{diff}" if diff > 0 else str(diff) print(f"\nParagraphs: {original_count} → {new_count} ({diff_str})") @@ -260,9 +266,15 @@ def validate_id_constraints(self): try: for elem in lxml.etree.parse(str(xml_file)).iter(): if val := elem.get(para_id_attr): - if self._parse_id_value(val, base=16) >= 0x80000000: + try: + if self._parse_id_value(val, base=16) >= 0x80000000: + errors.append( + f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + ) + except ValueError: errors.append( - f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000" + f" {xml_file.name}:{elem.sourceline}: " + f"paraId={val} is not valid hex" ) if val := elem.get(durable_id_attr): @@ -279,13 +291,19 @@ def validate_id_constraints(self): f"durableId={val} must be decimal in numbering.xml" ) else: - if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + try: + if self._parse_id_value(val, base=16) >= 0x7FFFFFFF: + errors.append( + f" {xml_file.name}:{elem.sourceline}: " + f"durableId={val} >= 0x7FFFFFFF" + ) + except ValueError: errors.append( f" {xml_file.name}:{elem.sourceline}: " - f"durableId={val} >= 0x7FFFFFFF" + f"durableId={val} is not valid hex" ) - except Exception: - pass + except lxml.etree.XMLSyntaxError: + continue if errors: print(f"FAILED - {len(errors)} ID constraint violations:") @@ -389,52 +407,54 @@ def repair(self) -> int: return repairs def repair_durableId(self) -> int: + DURABLE_ID_ATTRS = ("w16cid:durableId", "w16cex:durableId") repairs = 0 + renames: dict = {} for xml_file in self.xml_files: try: content = xml_file.read_text(encoding="utf-8") dom = defusedxml.minidom.parseString(content) + is_numbering = xml_file.name == "numbering.xml" + base = 10 if is_numbering else 16 + pending = [] + seen_in_file = set() modified = False for elem in dom.getElementsByTagName("*"): - if not elem.hasAttribute("w16cid:durableId"): - continue + for attr_name in DURABLE_ID_ATTRS: + if not elem.hasAttribute(attr_name): + continue - durable_id = elem.getAttribute("w16cid:durableId") - needs_repair = False - - if xml_file.name == "numbering.xml": + durable_id = elem.getAttribute(attr_name) try: - needs_repair = ( - self._parse_id_value(durable_id, base=10) >= 0x7FFFFFFF - ) + key = self._parse_id_value(durable_id, base=base) + needs_repair = key >= 0x7FFFFFFF except ValueError: + key = durable_id needs_repair = True - else: - try: - needs_repair = ( - self._parse_id_value(durable_id, base=16) >= 0x7FFFFFFF - ) - except ValueError: - needs_repair = True - - if needs_repair: - value = random.randint(1, 0x7FFFFFFE) - if xml_file.name == "numbering.xml": - new_id = str(value) - else: - new_id = f"{value:08X}" - elem.setAttribute("w16cid:durableId", new_id) - print( - f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" - ) - repairs += 1 - modified = True + if needs_repair: + if key in seen_in_file: + value = random.randint(1, 0x7FFFFFFE) + else: + seen_in_file.add(key) + if key not in renames: + renames[key] = random.randint(1, 0x7FFFFFFE) + value = renames[key] + new_id = str(value) if is_numbering else f"{value:08X}" + + elem.setAttribute(attr_name, new_id) + pending.append( + f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}" + ) + modified = True if modified: xml_file.write_bytes(dom.toxml(encoding="UTF-8")) + for message in pending: + print(message) + repairs += len(pending) except Exception: pass diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/pptx.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/pptx.py index 09842aa998..7b53d0d3e4 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/pptx.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/pptx.py @@ -3,6 +3,9 @@ """ import re +from pathlib import Path + +from helpers import opc_target, rels_source_part, safe_extract from .base import BaseSchemaValidator @@ -57,8 +60,171 @@ def validate(self): if not self.validate_no_duplicate_slide_layouts(): all_valid = False + if not self.validate_master_theme_uniqueness(): + all_valid = False + + if not self.validate_charts(): + all_valid = False + + if not self.validate_slides(): + all_valid = False + return all_valid + def _package_map(self) -> dict: + wanted = [] + wanted += list(self.unpacked_dir.glob("[[]Content_Types[]].xml")) + wanted += list(self.unpacked_dir.glob("ppt/presentation.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/*.xml")) + wanted += list(self.unpacked_dir.glob("ppt/theme/_rels/*.rels")) + wanted += list(self.unpacked_dir.glob("ppt/charts/chart*.xml")) + for group in ("slideMasters", "notesMasters", "handoutMasters"): + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/*.xml")) + wanted += list(self.unpacked_dir.glob(f"ppt/{group}/_rels/*.rels")) + return { + p.relative_to(self.unpacked_dir).as_posix(): p.read_bytes() + for p in wanted + if p.is_file() + } + + def validate_master_theme_uniqueness(self): + from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes + + shared = live_shared_master_themes(self._package_map()) + if shared: + print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:") + for message in shared: + print(f" {message}") + if any(m.startswith(_NOTES_MASTERS) for m in shared): + print(" Fix: in ppt/presentation.xml, move back to " + "directly after . PowerPoint reads that happily.") + else: + print(" Fix: give each master its own theme part.") + return False + + if self.verbose: + print("PASSED - No master shares a theme part in a way PowerPoint refuses") + return True + + def validate_charts(self): + from helpers.pptx_chart import find_chart_problems + + problems = find_chart_problems(self._package_map()) + if problems: + print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:") + for message in problems: + print(f" {message}") + return False + + if self.verbose: + print("PASSED - Charts satisfy the constraints PowerPoint enforces") + return True + + def _original_slide_defects(self, schema) -> set[str]: + import tempfile + import zipfile + + from helpers.pptx_slide import SLIDE_PART_RE, fatal_slide_errors + + if self.original_file is None: + return set() + + found: set[str] = set() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + try: + with zipfile.ZipFile(self.original_file, "r") as zf: + safe_extract(zf, temp_path) + except (zipfile.BadZipFile, ValueError, OSError): + return set() + + for part in sorted(temp_path.rglob("*.xml")): + relative = part.relative_to(temp_path).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + part.resolve(), temp_path.resolve(), schema_path=schema + ) + if ok is None or ok or not errors: + continue + found |= set(fatal_slide_errors(set(errors))) + return found + + def validate_slides(self): + from helpers.pptx_slide import ( + SLIDE_PART_RE, + fatal_slide_errors, + is_schema_verdict, + ) + + schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"] + inherited = self._original_slide_defects(schema) + problems: list[str] = [] + broken: list[str] = [] + + for xml_file in self.xml_files: + relative = xml_file.relative_to(self.unpacked_dir).as_posix() + if not SLIDE_PART_RE.fullmatch(relative): + continue + ok, errors = self._validate_single_file_xsd( + xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema + ) + if ok is None or not errors: + continue + + unreadable = [f"{relative}: {e}" for e in errors if not is_schema_verdict(e)] + if unreadable: + broken.extend(unreadable) + continue + if ok: + continue + + for message in fatal_slide_errors(set(errors)): + if message in inherited: + continue + problems.append(f"{relative}: {message}") + + if broken: + print(f"FAILED - Could not check {len(broken)} slide part(s):") + for message in sorted(broken): + print(f" {message[:240]}") + + if problems: + print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:") + for message in sorted(problems): + print(f" {message[:240]}") + + if broken or problems: + return False + + if self.verbose: + print("PASSED - Slide XML has none of the defects PowerPoint refuses") + return True + + def _get_schema_path(self, xml_file): + if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"): + return None + return super()._get_schema_path(xml_file) + + def _preprocess_for_schema(self, xml_doc, relative_path): + if relative_path.as_posix() != "ppt/presentation.xml": + return xml_doc + + root = xml_doc.getroot() + ns = f"{{{self.PRESENTATIONML_NAMESPACE}}}" + notes = root.find(f"{ns}notesMasterIdLst") + slides = root.find(f"{ns}sldIdLst") + if notes is None or slides is None: + return xml_doc + + children = list(root) + if children.index(notes) < children.index(slides): + return xml_doc + + root.remove(notes) + root.insert(list(root).index(slides), notes) + return xml_doc + def validate_uuid_ids(self): import lxml.etree @@ -229,17 +395,17 @@ def validate_notes_slide_references(self): ): rel_type = rel.get("Type", "") if "notesSlide" in rel_type: - target = rel.get("Target", "") - if target: - normalized_target = target.replace("../", "") - + part = opc_target( + rel.get("Target", ""), + rels_source_part(rels_file, self.unpacked_dir), + rel.get("TargetMode", ""), + ) + if part: slide_name = rels_file.stem.replace( ".xml", "" ) - if normalized_target not in notes_slide_references: - notes_slide_references[normalized_target] = [] - notes_slide_references[normalized_target].append( + notes_slide_references.setdefault(part, []).append( (slide_name, rels_file) ) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/redlining.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/redlining.py index 71c81b6bf4..18d0c68be9 100644 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/redlining.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/office/validators/redlining.py @@ -1,5 +1,14 @@ """ Validator for tracked changes in Word documents. + +Detects untracked edits in word/document.xml: text that differs from the +original without a / wrapper recording it. The tracked changes +that are new relative to the original are undone, and the result is compared +against the original; whatever text still differs was edited without being +tracked. + +Only the document body is compared. Headers, footers, footnotes and endnotes +are separate parts and are not checked. """ import subprocess @@ -7,14 +16,18 @@ import zipfile from pathlib import Path +import defusedxml.ElementTree as ET +from defusedxml.common import DefusedXmlException + +from helpers import rendered_text, safe_extract + class RedliningValidator: - def __init__(self, unpacked_dir, original_docx, verbose=False, author="Claude"): + def __init__(self, unpacked_dir, original_docx, verbose=False): self.unpacked_dir = Path(unpacked_dir) self.original_docx = Path(original_docx) self.verbose = verbose - self.author = author self.namespaces = { "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main" } @@ -28,40 +41,12 @@ def validate(self): print(f"FAILED - Modified document.xml not found at {modified_file}") return False - try: - import xml.etree.ElementTree as ET - - tree = ET.parse(modified_file) - root = tree.getroot() - - del_elements = root.findall(".//w:del", self.namespaces) - ins_elements = root.findall(".//w:ins", self.namespaces) - - author_del_elements = [ - elem - for elem in del_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - author_ins_elements = [ - elem - for elem in ins_elements - if elem.get(f"{{{self.namespaces['w']}}}author") == self.author - ] - - if not author_del_elements and not author_ins_elements: - if self.verbose: - print(f"PASSED - No tracked changes by {self.author} found.") - return True - - except Exception: - pass - with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) try: with zipfile.ZipFile(self.original_docx, "r") as zip_ref: - zip_ref.extractall(temp_path) + safe_extract(zip_ref, temp_path) except Exception as e: print(f"FAILED - Error unpacking original docx: {e}") return False @@ -74,18 +59,16 @@ def validate(self): return False try: - import xml.etree.ElementTree as ET - modified_tree = ET.parse(modified_file) modified_root = modified_tree.getroot() original_tree = ET.parse(original_file) original_root = original_tree.getroot() - except ET.ParseError as e: + except (ET.ParseError, DefusedXmlException) as e: print(f"FAILED - Error parsing XML files: {e}") return False - self._remove_author_tracked_changes(original_root) - self._remove_author_tracked_changes(modified_root) + new_changes = self._new_tracked_changes(original_root, modified_root) + self._remove_tracked_changes(modified_root, new_changes) modified_text = self._extract_text_content(modified_root) original_text = self._extract_text_content(original_root) @@ -98,20 +81,91 @@ def validate(self): return False if self.verbose: - print(f"PASSED - All changes by {self.author} are properly tracked") + print( + f"PASSED - All {len(new_changes)} change(s) against the original " + "are properly tracked" + ) return True + def _tracked_change_elements(self, root): + ins_tag = f"{{{self.namespaces['w']}}}ins" + del_tag = f"{{{self.namespaces['w']}}}del" + return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)] + + def _rendered_text(self, elem): + preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" + return rendered_text(elem.text or "", preserve) + + def _text_elements(self, elem): + w = self.namespaces["w"] + return [ + node + for node in elem.iter() + if node.tag in (f"{{{w}}}t", f"{{{w}}}delText") + ] + + def _tracked_change_key(self, elem): + w = self.namespaces["w"] + text = "".join(self._rendered_text(node) for node in self._text_elements(elem)) + return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text) + + def _new_tracked_changes(self, original_root, modified_root): + original = self._tracked_change_elements(original_root) + modified = self._tracked_change_elements(modified_root) + + pool = {} + for elem in original: + pool.setdefault(self._tracked_change_key(elem), []).append(elem) + + matched, leftover = set(), [] + for elem in modified: + bucket = pool.get(self._tracked_change_key(elem)) + if bucket: + matched.add(bucket.pop()) + else: + leftover.append(elem) + + def group(elem): + return self._tracked_change_key(elem)[:3] + + def text_of(elems): + return "".join(self._tracked_change_key(e)[3] for e in elems) + + unmatched_original = {} + for elem in original: + if elem not in matched: + unmatched_original.setdefault(group(elem), []).append(elem) + + by_group = {} + for elem in leftover: + by_group.setdefault(group(elem), []).append(elem) + + new = set() + for key, elems in by_group.items(): + rebuilt = text_of(elems) + if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])): + continue + new.update(elems) + return new + def _generate_detailed_diff(self, original_text, modified_text): error_parts = [ - f"FAILED - Document text doesn't match after removing {self.author}'s tracked changes", + "FAILED - Document text doesn't match after removing the tracked changes", "", "Likely causes:", " 1. Modified text inside another author's or tags", " 2. Made edits without proper tracked changes", " 3. Didn't nest inside when deleting another's insertion", + " 4. Rewrote another author's / and changed its text on", + " the way. A tracked change from the original is recognised by its", + " author, date and text; anything that doesn't reproduce one exactly", + " reads as new, and the text it carried is reported missing.", "", "For pre-redlined documents, use correct patterns:", " - To reject another's INSERTION: Nest inside their ", + " - To reject PART of one: nest around only the runs you reject.", + " Their may be split around it, so long as the pieces keep", + " their author and date and still spell out the same text.", " - To restore another's DELETION: Add new AFTER their ", "", ] @@ -195,15 +249,14 @@ def _get_git_word_diff(self, original_text, modified_text): return None - def _remove_author_tracked_changes(self, root): + def _remove_tracked_changes(self, root, targets): ins_tag = f"{{{self.namespaces['w']}}}ins" del_tag = f"{{{self.namespaces['w']}}}del" - author_attr = f"{{{self.namespaces['w']}}}author" for parent in root.iter(): to_remove = [] for child in parent: - if child.tag == ins_tag and child.get(author_attr) == self.author: + if child.tag == ins_tag and child in targets: to_remove.append(child) for elem in to_remove: parent.remove(elem) @@ -214,7 +267,7 @@ def _remove_author_tracked_changes(self, root): for parent in root.iter(): to_process = [] for child in parent: - if child.tag == del_tag and child.get(author_attr) == self.author: + if child.tag == del_tag and child in targets: to_process.append((child, list(parent).index(child))) for del_elem, del_index in reversed(to_process): @@ -234,8 +287,7 @@ def _extract_text_content(self, root): for p_elem in root.findall(f".//{p_tag}"): text_parts = [] for t_elem in p_elem.findall(f".//{t_tag}"): - if t_elem.text: - text_parts.append(t_elem.text) + text_parts.append(self._rendered_text(t_elem)) paragraph_text = "".join(text_parts) if paragraph_text: paragraphs.append(paragraph_text) diff --git a/src/crates/assembly/core/builtin_skills/xlsx/scripts/recalc.py b/src/crates/assembly/core/builtin_skills/xlsx/scripts/recalc.py index f472e9a5d0..ba6d0c3ad1 100755 --- a/src/crates/assembly/core/builtin_skills/xlsx/scripts/recalc.py +++ b/src/crates/assembly/core/builtin_skills/xlsx/scripts/recalc.py @@ -3,20 +3,29 @@ Recalculates all formulas in an Excel file using LibreOffice """ +import contextlib import json import os import platform +import re +import shutil import subprocess import sys +import tempfile +import time +import zipfile from pathlib import Path -from office.soffice import get_soffice_env +from office.soffice import get_soffice_env, run_soffice from openpyxl import load_workbook -MACRO_DIR_MACOS = "~/Library/Application Support/LibreOffice/4/user/basic/Standard" -MACRO_DIR_LINUX = "~/.config/libreoffice/4/user/basic/Standard" MACRO_FILENAME = "Module1.xba" +SOFFICE_MISSING = "soffice not found on PATH; LibreOffice is required to recalculate" + +MAX_LOCATIONS = 100 + +EXTERNAL_REF_RE = re.compile(r"""(? @@ -39,63 +48,168 @@ def has_gtimeout(): return False -def setup_libreoffice_macro(): - macro_dir = os.path.expanduser( - MACRO_DIR_MACOS if platform.system() == "Darwin" else MACRO_DIR_LINUX - ) - macro_file = os.path.join(macro_dir, MACRO_FILENAME) +def _stamp(path): + st = os.stat(path) + return st.st_mtime_ns, st.st_size - if ( - os.path.exists(macro_file) - and "RecalculateAndSave" in Path(macro_file).read_text() - ): - return True - if not os.path.exists(macro_dir): - subprocess.run( - ["soffice", "--headless", "--terminate_after_init"], +def setup_libreoffice_macro(profile_dir: Path, timeout=30): + url = profile_dir.as_uri() + try: + run_soffice( + ["--headless", "--terminate_after_init", f"-env:UserInstallation={url}"], capture_output=True, - timeout=10, - env=get_soffice_env(), + timeout=timeout, ) - os.makedirs(macro_dir, exist_ok=True) + except FileNotFoundError: + return None, SOFFICE_MISSING + except subprocess.TimeoutExpired: + return None, "LibreOffice timed out creating its profile; formulas were NOT recalculated" + + macro_dir = profile_dir / "user" / "basic" / "Standard" + if not macro_dir.exists(): + return None, "LibreOffice did not create a usable profile; formulas were NOT recalculated" try: - Path(macro_file).write_text(RECALCULATE_MACRO) - return True - except Exception: - return False + (macro_dir / MACRO_FILENAME).write_text(RECALCULATE_MACRO) + except OSError as e: + return None, f"Could not install the recalculation macro: {e}" + + return url, None + + +def external_links_at_risk(filename): + try: + with zipfile.ZipFile(filename) as archive: + names = archive.namelist() + except (zipfile.BadZipFile, OSError): + return [] + if not any(n.startswith("xl/externalLinks/") for n in names): + return [] + + with contextlib.ExitStack() as stack: + formulas = load_workbook(filename, data_only=False) + stack.callback(formulas.close) + values = load_workbook(filename, data_only=True) + stack.callback(values.close) + + external_names = [ + name + for name, dn in formulas.defined_names.items() + if isinstance(getattr(dn, "value", None), str) and EXTERNAL_REF_RE.search(dn.value) + ] + name_re = ( + re.compile(r"\b(" + "|".join(re.escape(n) for n in external_names) + r")\b") + if external_names + else None + ) + at_risk = [] + for sheet in formulas.sheetnames: + ws = formulas[sheet] + if not hasattr(ws, "iter_rows"): + continue + cached = values[sheet] + for row in ws.iter_rows(): + for cell in row: + v = cell.value + if not (isinstance(v, str) and v.startswith("=")): + continue + reaches_out = EXTERNAL_REF_RE.search(v) or (name_re and name_re.search(v)) + if reaches_out and cached[cell.coordinate].value is None: + at_risk.append(f"{sheet}!{cell.coordinate}") + return at_risk -def recalc(filename, timeout=30): + +def recalc(filename, timeout=30, force=False): if not Path(filename).exists(): return {"error": f"File {filename} does not exist"} abs_path = str(Path(filename).absolute()) - if not setup_libreoffice_macro(): - return {"error": "Failed to setup LibreOffice macro"} + if not os.access(abs_path, os.W_OK): + return {"error": f"{filename} is not writable; recalculation rewrites the file in place"} + + try: + get_soffice_env() + except Exception as e: + return {"error": f"Could not prepare the LibreOffice environment: {e}"} + + if not force: + try: + at_risk = external_links_at_risk(filename) + except Exception as e: + return {"error": f"Could not inspect {filename} for external links: {e}"} + if at_risk: + shown = at_risk[:MAX_LOCATIONS] + return { + "error": ( + "Refusing to recalculate: this workbook links to another workbook, and " + f"{len(at_risk)} linked cell(s) have lost their cached value (openpyxl strips " + "these on save). Recalculating would resolve them to #NAME? and delete the " + "external links for good. Copy those cells' values from the original file " + "before saving, or pass --force to accept the loss. Charts and conditional " + "formats can hold external references too, so this list may not be exhaustive." + ), + "external_link_cells": shown, + "external_link_cells_truncated": max(0, len(at_risk) - len(shown)), + } + + with tempfile.TemporaryDirectory( + prefix="recalc-lo-profile-", ignore_cleanup_errors=True + ) as profile_dir: + return _recalc_with_profile(filename, abs_path, timeout, Path(profile_dir)) + + +def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path): + started = time.monotonic() + profile_url, err = setup_libreoffice_macro(profile_dir, timeout=timeout) + if err: + return {"error": err} + + timeout = max(5, int(timeout - (time.monotonic() - started))) + + before = _stamp(abs_path) cmd = [ "soffice", "--headless", "--norestore", + f"-env:UserInstallation={profile_url}", "vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application", abs_path, ] - if platform.system() == "Linux": + if platform.system() == "Linux" and shutil.which("timeout"): cmd = ["timeout", str(timeout)] + cmd elif platform.system() == "Darwin" and has_gtimeout(): cmd = ["gtimeout", str(timeout)] + cmd - result = subprocess.run(cmd, capture_output=True, text=True, env=get_soffice_env()) + timed_out = f"LibreOffice timed out after {timeout}s; formulas were NOT recalculated. Re-run with a longer timeout." - if result.returncode != 0 and result.returncode != 124: - error_msg = result.stderr or "Unknown error during recalculation" - if "Module1" in error_msg or "RecalculateAndSave" not in error_msg: - return {"error": "LibreOffice macro not configured properly"} - return {"error": error_msg} + try: + result = subprocess.run( + cmd, capture_output=True, text=True, env=get_soffice_env(), timeout=timeout + 15 + ) + except subprocess.TimeoutExpired: + return {"error": timed_out} + except FileNotFoundError: + return {"error": SOFFICE_MISSING} + + if result.returncode == 124: + return {"error": timed_out} + + if result.returncode != 0: + detail = (result.stderr or "").strip() or f"soffice exited {result.returncode}" + return {"error": f"LibreOffice failed to recalculate: {detail}"} + + if _stamp(abs_path) == before: + return { + "error": ( + "LibreOffice exited cleanly but never rewrote the file, so nothing was " + "recalculated. Check that no other LibreOffice instance is running, then retry." + ) + } try: wb = load_workbook(filename, data_only=True) @@ -114,6 +228,8 @@ def recalc(filename, timeout=30): for sheet_name in wb.sheetnames: ws = wb[sheet_name] + if not hasattr(ws, "iter_rows"): + continue for row in ws.iter_rows(): for cell in row: if cell.value is not None and isinstance(cell.value, str): @@ -124,8 +240,6 @@ def recalc(filename, timeout=30): total_errors += 1 break - wb.close() - result = { "status": "success" if total_errors == 0 else "errors_found", "total_errors": total_errors, @@ -134,15 +248,19 @@ def recalc(filename, timeout=30): for err_type, locations in error_details.items(): if locations: - result["error_summary"][err_type] = { - "count": len(locations), - "locations": locations[:20], - } + entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]} + if len(locations) > MAX_LOCATIONS: + entry["locations_truncated"] = len(locations) - MAX_LOCATIONS + result["error_summary"][err_type] = entry + + wb.close() wb_formulas = load_workbook(filename, data_only=False) formula_count = 0 for sheet_name in wb_formulas.sheetnames: ws = wb_formulas[sheet_name] + if not hasattr(ws, "iter_rows"): + continue for row in ws.iter_rows(): for cell in row: if ( @@ -162,8 +280,11 @@ def recalc(filename, timeout=30): def main(): - if len(sys.argv) < 2: - print("Usage: python recalc.py [timeout_seconds]") + args = [a for a in sys.argv[1:] if a != "--force"] + force = "--force" in sys.argv[1:] + + if not args: + print("Usage: python recalc.py [timeout_seconds] [--force]") print("\nRecalculates all formulas in an Excel file using LibreOffice") print("\nReturns JSON with error details:") print(" - status: 'success' or 'errors_found'") @@ -171,13 +292,16 @@ def main(): print(" - total_formulas: Number of formulas in the file") print(" - error_summary: Breakdown by error type with locations") print(" - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A") + print("\nOn any failure the JSON has an 'error' key and no 'status'.") + print("--force recalculates even when it would destroy external links.") sys.exit(1) - filename = sys.argv[1] - timeout = int(sys.argv[2]) if len(sys.argv) > 2 else 30 + filename = args[0] + timeout = int(args[1]) if len(args) > 1 else 30 - result = recalc(filename, timeout) + result = recalc(filename, timeout, force=force) print(json.dumps(result, indent=2)) + sys.exit(1 if "error" in result else 0) if __name__ == "__main__": diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/skills/builtin.rs b/src/crates/assembly/core/src/agentic/tools/implementations/skills/builtin.rs index 9505ee9e23..1363f90ee4 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/skills/builtin.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/skills/builtin.rs @@ -364,3 +364,253 @@ async fn desired_file_content( ) -> BitFunResult> { Ok(file.contents().to_vec()) } + +#[cfg(test)] +mod tests { + use super::{collect_files, BUILTIN_SKILLS_DIR}; + + fn embedded_skill_text(path: &str) -> &'static str { + BUILTIN_SKILLS_DIR + .get_file(path) + .unwrap_or_else(|| panic!("Missing embedded built-in skill file: {path}")) + .contents_utf8() + .unwrap_or_else(|| panic!("Built-in skill file is not UTF-8: {path}")) + } + + fn gstack_skill_texts() -> Vec<(String, &'static str)> { + BUILTIN_SKILLS_DIR + .dirs() + .filter_map(|dir| { + let name = dir.path().file_name()?.to_str()?; + if !name.starts_with("gstack-") { + return None; + } + let file = dir.files().find(|file| { + file.path().file_name().and_then(|name| name.to_str()) == Some("SKILL.md") + })?; + Some(( + name.to_string(), + file.contents_utf8() + .unwrap_or_else(|| panic!("{name}/SKILL.md is not UTF-8")), + )) + }) + .collect() + } + + #[test] + fn gstack_direct_skill_paths_resolve_to_bundled_skills() { + for (source, text) in gstack_skill_texts() { + for token in text.split(|ch: char| { + !(ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '/' | '.' | ':')) + }) { + if let Some(path) = token.strip_suffix("/SKILL.md") { + let target = path.rsplit('/').next().unwrap_or(path); + assert!( + BUILTIN_SKILLS_DIR.get_dir(target).is_some(), + "{source}/SKILL.md references missing built-in skill {target}/SKILL.md" + ); + } + if let Some(target) = token.strip_prefix("user::bitfun-system::") { + assert!( + BUILTIN_SKILLS_DIR.get_dir(target).is_some(), + "{source}/SKILL.md references missing stable skill key {token}" + ); + } + } + } + } + + #[test] + fn gstack_does_not_emit_pseudo_bitfun_browser_commands() { + const STALE_BROWSER_GUIDANCE: [&str; 5] = [ + "BitFun browser/computer-use", + "BitFun built-in browser/computer-use", + "external browse binary", + "use `ComputerUse` for browser inspection", + "use `ComputerUse` for browser/desktop testing", + ]; + + for (source, text) in gstack_skill_texts() { + let lowercase = text.to_ascii_lowercase(); + for stale in STALE_BROWSER_GUIDANCE { + assert!( + !text.contains(stale), + "{source}/SKILL.md still contains stale browser guidance: {stale}" + ); + } + assert!( + !text.contains("agent-browser fill @e4 \"[REDACTED]\""), + "{source}/SKILL.md still places a password placeholder in a logged command" + ); + assert!( + !text.contains("agent-browser state load cookies.json"), + "{source}/SKILL.md treats a cookie file as agent-browser storage state" + ); + assert!( + !text.contains("auth save qa-target") && !text.contains("auth login qa-target"), + "{source}/SKILL.md reuses a global auth profile across unrelated targets" + ); + assert!( + !text.contains("CDP_MODE=true"), + "{source}/SKILL.md still infers agent-browser state from the legacy CDP mode" + ); + if text.contains("cookie file or Copy-as-cURL export") { + assert!( + text.contains("agent-browser cookies set --curl cookies.json"), + "{source}/SKILL.md does not import cookie files with the supported command" + ); + } + assert!( + !text.contains("Ask the user to enter it through stdin"), + "{source}/SKILL.md asks for interactive stdin in a non-interactive tool command" + ); + if text.contains("--password-stdin") { + assert!( + text.contains("own interactive terminal"), + "{source}/SKILL.md must route password-stdin setup to the user's terminal" + ); + } + if text.contains("agent-browser open") || text.contains("agent-browser get url") { + assert!( + text.contains("agent-browser --version") + && text.contains("agent-browser skills get core") + && lowercase.contains("once per skill invocation") + && lowercase.contains("before the first browser command"), + "{source}/SKILL.md must verify the CLI and load version-matched guidance" + ); + } + if text.contains("SKETCH_URI") { + assert!( + text.contains("tempfile.mkstemp") + && text.contains(".resolve().as_uri()") + && text.contains("agent-browser --allow-file-access open") + && lowercase.contains("once per skill invocation") + && lowercase.contains("before the first browser command"), + "{source}/SKILL.md does not open local HTML portably and explicitly" + ); + } + } + } + + #[test] + fn gstack_does_not_route_to_unbundled_workflows() { + const ABSENT_WORKFLOWS: [&str; 6] = [ + "plan-devex-review", + "/design-shotgun", + "/design-html", + "/setup-browser-cookies", + "qa/templates/qa-report-template.md", + "qa/references/issue-taxonomy.md", + ]; + + for (source, text) in gstack_skill_texts() { + for absent in ABSENT_WORKFLOWS { + assert!( + !text.contains(absent), + "{source}/SKILL.md routes to unbundled workflow {absent}" + ); + } + } + } + + #[test] + fn agent_browser_uses_dynamic_cli_documentation_only() { + let text = embedded_skill_text("agent-browser/SKILL.md"); + assert!(text.contains("agent-browser skills get core")); + assert!(text.contains("agent-browser skills get core --full")); + assert!(text.contains("agent-browser skills list")); + assert!(text.contains("agent-browser skills get electron")); + assert!(text.contains("agent-browser skills get dogfood")); + assert!(text.contains("npm i -g agent-browser@0.32.3")); + assert!(text.contains("Install only after user approval")); + assert!(text.contains("do not silently switch tools")); + assert!(text.contains("native Rust")); + assert!(!text.contains("npx playwright install-deps")); + + let dir = BUILTIN_SKILLS_DIR + .get_dir("agent-browser") + .expect("agent-browser directory should be embedded"); + assert!( + dir.dirs().next().is_none(), + "dynamic agent-browser stub must not retain static reference/template directories" + ); + } + + #[test] + fn office_helpers_use_validated_archive_extraction() { + for skill in ["docx", "pptx", "xlsx"] { + let helper_path = format!("{skill}/scripts/office/helpers/__init__.py"); + let helper = embedded_skill_text(&helper_path); + assert!( + helper.contains("def safe_extract("), + "{helper_path} lacks safe_extract" + ); + assert!( + helper.contains("stat.S_ISLNK"), + "{helper_path} lacks symlink rejection" + ); + assert!( + helper.contains("MAX_ARCHIVE_TOTAL_SIZE") + && helper.contains("MAX_ARCHIVE_COMPRESSION_RATIO") + && helper.contains("duplicate archive entry"), + "{helper_path} lacks bounded, collision-safe extraction" + ); + + let dir = BUILTIN_SKILLS_DIR + .get_dir(skill) + .unwrap_or_else(|| panic!("Missing embedded Office skill {skill}")); + let mut files = Vec::new(); + collect_files(dir, &mut files); + for file in files { + let text = file.contents_utf8().unwrap_or(""); + assert!( + !text.contains(".extractall("), + "{} still uses unrestricted ZipFile.extractall", + file.path().display() + ); + } + + assert!(dir + .get_file(format!("{skill}/scripts/office/pack.py")) + .is_none()); + assert!(dir + .get_file(format!("{skill}/scripts/office/unpack.py")) + .is_none()); + + if matches!(skill, "docx" | "pptx") { + let skill_text = embedded_skill_text(&format!("{skill}/SKILL.md")); + assert!( + skill_text.contains("safe_extract") && skill_text.contains("rezip"), + "{skill}/SKILL.md must use the cross-platform safe archive helpers" + ); + assert!( + !skill_text.contains("unzip -q") && !skill_text.contains("zip -Xr"), + "{skill}/SKILL.md still recommends unsafe or non-portable archive commands" + ); + } + } + + let comment = embedded_skill_text("docx/scripts/comment.py"); + assert!(comment.contains("author: str = \"BitFun\"")); + assert!(comment.contains("initials: str = \"B\"")); + assert!(comment.contains("default=\"BitFun\"")); + assert!(comment.contains("default=\"B\"")); + + let docx_skill = embedded_skill_text("docx/SKILL.md"); + assert!(docx_skill.contains( + "Use \"BitFun\" as the author for tracked changes and comments unless the user explicitly requests a different name." + )); + + let xlsx_skill = embedded_skill_text("xlsx/SKILL.md"); + assert!(xlsx_skill.contains("years as text (`\"2026\"`, never `2,026`)")); + + let docx_helper = embedded_skill_text("docx/scripts/office/helpers/__init__.py"); + for skill in ["pptx", "xlsx"] { + assert_eq!( + docx_helper, + embedded_skill_text(&format!("{skill}/scripts/office/helpers/__init__.py")), + "Office safe extraction helpers drifted between bundled skills" + ); + } + } +} diff --git a/src/crates/assembly/core/tests/office_archive_safety.py b/src/crates/assembly/core/tests/office_archive_safety.py new file mode 100644 index 0000000000..e73ea02ee9 --- /dev/null +++ b/src/crates/assembly/core/tests/office_archive_safety.py @@ -0,0 +1,117 @@ +import importlib.util +import io +import stat +import sys +import tempfile +import unittest +import warnings +import zipfile +from pathlib import Path + + +sys.dont_write_bytecode = True + + +CORE_ROOT = Path(__file__).resolve().parents[1] +SKILLS_ROOT = CORE_ROOT / "builtin_skills" + + +def load_helpers(skill: str): + path = SKILLS_ROOT / skill / "scripts" / "office" / "helpers" / "__init__.py" + spec = importlib.util.spec_from_file_location(f"{skill}_office_helpers", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load Office helpers from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def archive_bytes(entries, compression=zipfile.ZIP_STORED): + data = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(data, "w", compression=compression) as archive: + for name, content in entries: + archive.writestr(name, content) + data.seek(0) + return data + + +class OfficeArchiveSafetyTests(unittest.TestCase): + def setUp(self): + self.helpers = {skill: load_helpers(skill) for skill in ("docx", "pptx", "xlsx")} + + def assert_rejected(self, data, message): + for skill, helpers in self.helpers.items(): + with self.subTest(skill=skill, message=message), tempfile.TemporaryDirectory() as temp: + data.seek(0) + with zipfile.ZipFile(data) as archive: + with self.assertRaisesRegex(ValueError, message): + helpers.safe_extract(archive, Path(temp)) + + def test_rejects_traversal_absolute_symlink_and_duplicate_targets(self): + self.assert_rejected(archive_bytes([("../escape.txt", b"x")]), "unsafe archive entry") + self.assert_rejected(archive_bytes([("/absolute.txt", b"x")]), "unsafe archive entry") + self.assert_rejected(archive_bytes([(".", b"x")]), "unsafe archive entry") + + symlink = zipfile.ZipInfo("link") + symlink.create_system = 3 + symlink.external_attr = (stat.S_IFLNK | 0o777) << 16 + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr(symlink, "target") + data.seek(0) + self.assert_rejected(data, "symlink archive entry") + + self.assert_rejected( + archive_bytes([("duplicate.txt", b"a"), ("./duplicate.txt", b"b")]), + "duplicate archive entry", + ) + self.assert_rejected( + archive_bytes([("file", b"a"), ("file/child", b"b")]), + "file entry conflicts with child path", + ) + + def test_rejects_member_count_size_total_size_and_compression_ratio_limits(self): + cases = [ + ("MAX_ARCHIVE_MEMBERS", 1, [("a", b""), ("b", b"")], "too many entries"), + ("MAX_ARCHIVE_MEMBER_SIZE", 1, [("large", b"xx")], "entry is too large"), + ("MAX_ARCHIVE_TOTAL_SIZE", 1, [("total", b"xx")], "allowed total size"), + ] + for constant, limit, entries, message in cases: + for skill, helpers in self.helpers.items(): + with self.subTest(skill=skill, constant=constant), tempfile.TemporaryDirectory() as temp: + original = getattr(helpers, constant) + setattr(helpers, constant, limit) + try: + with zipfile.ZipFile(archive_bytes(entries)) as archive: + with self.assertRaisesRegex(ValueError, message): + helpers.safe_extract(archive, Path(temp)) + finally: + setattr(helpers, constant, original) + + for skill, helpers in self.helpers.items(): + with self.subTest(skill=skill, constant="compression_ratio"), tempfile.TemporaryDirectory() as temp: + original = helpers.MAX_ARCHIVE_COMPRESSION_RATIO + helpers.MAX_ARCHIVE_COMPRESSION_RATIO = 1 + try: + data = archive_bytes([("compressed", b"A" * 4096)], zipfile.ZIP_DEFLATED) + with zipfile.ZipFile(data) as archive: + with self.assertRaisesRegex(ValueError, "unsafe compression ratio"): + helpers.safe_extract(archive, Path(temp)) + finally: + helpers.MAX_ARCHIVE_COMPRESSION_RATIO = original + + def test_extracts_valid_archive(self): + for skill, helpers in self.helpers.items(): + with self.subTest(skill=skill), tempfile.TemporaryDirectory() as temp: + with zipfile.ZipFile(archive_bytes([("word/document.xml", b"")])) as archive: + helpers.safe_extract(archive, Path(temp)) + self.assertEqual( + (Path(temp) / "word" / "document.xml").read_bytes(), + b"", + ) + + +if __name__ == "__main__": + unittest.main()