diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 6395d80..e047e82 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,5 +13,5 @@ {"id":"openadapt-evals-hvm","title":"VL model fix PR #18 ready to merge","notes":"2026-02-08: openadapt-ml PR #18 was already merged on 2026-01-29. VL model fix is done.","status":"closed","priority":0,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-01-29T16:17:03.491938-05:00","created_by":"Richard Abrich","updated_at":"2026-02-08T12:55:19.233249-05:00","closed_at":"2026-02-08T12:55:19.233249-05:00","close_reason":"PR #18 already merged 2026-01-29"} {"id":"openadapt-evals-mx8","title":"Analyze evaluation results and publish findings","description":"After demo-conditioned evaluation completes, analyze results: success rates, failure modes, demo impact. Create data-driven roadmap for improvements.","notes":"wright repo (OpenAdaptAI/wright) scaffolding underway. Herald + consilium repos transferred to OpenAdaptAI org. Wright will be the orchestration layer for eval pipeline.","status":"open","priority":1,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-02-14T12:23:06.328838-05:00","created_by":"Richard Abrich","updated_at":"2026-03-02T00:08:08.422633-05:00"} {"id":"openadapt-evals-sz4","title":"RCA: Windows product key prompt recurring issue","status":"closed","priority":0,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-01-20T18:59:36.266286-05:00","created_by":"Richard Abrich","updated_at":"2026-01-20T20:32:06.493102-05:00","closed_at":"2026-01-20T20:32:06.493102-05:00","close_reason":"RCA complete - root cause is VERSION mismatch (CLI=11, Dockerfile=11e). Fix documented in RECURRING_ISSUES.md and WINDOWS_PRODUCT_KEY_RCA.md"} -{"id":"openadapt-evals-vcb","title":"Run demo-conditioned WAA evaluation","description":"Once demos are recorded, run WAA evaluation with demo-conditioned agents (RetrievalAugmentedAgent with real demos). Target: measure improvement over zero-shot baseline. Requires real demos from recording task.","notes":"wright repo created (OpenAdaptAI/wright), scaffolding in progress. Herald + consilium transferred to OpenAdaptAI org.","status":"open","priority":0,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-02-14T12:23:04.624305-05:00","created_by":"Richard Abrich","updated_at":"2026-03-02T00:08:08.423284-05:00"} +{"id":"openadapt-evals-vcb","title":"Run demo-conditioned WAA evaluation","description":"Once demos are recorded, run WAA evaluation with demo-conditioned agents (RetrievalAugmentedAgent with real demos). Target: measure improvement over zero-shot baseline. Requires real demos from recording task.","notes":"2026-03-01: GPU grant applications reviewed and rewritten (11 files). Writing done, blocked on eval results (DC signal on harder tasks). Detailed status tracked in openadapt-internal (private repo).","status":"open","priority":0,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-02-14T12:23:04.624305-05:00","created_by":"Richard Abrich","updated_at":"2026-03-02T01:14:44.513125-05:00"} {"id":"openadapt-evals-wis","title":"Add pre-flight check to detect Windows install issues","status":"closed","priority":1,"issue_type":"task","owner":"richard.abrich@gmail.com","created_at":"2026-01-20T18:59:36.865052-05:00","created_by":"Richard Abrich","updated_at":"2026-01-20T20:32:06.757261-05:00","closed_at":"2026-01-20T20:32:06.757261-05:00","close_reason":"Duplicate of openadapt-evals-0dt"} diff --git a/docs/artifacts/full/step_00_after.png b/docs/artifacts/full/step_00_after.png new file mode 100644 index 0000000..5238ebe Binary files /dev/null and b/docs/artifacts/full/step_00_after.png differ diff --git a/docs/artifacts/full/step_00_before.png b/docs/artifacts/full/step_00_before.png new file mode 100644 index 0000000..b28a343 Binary files /dev/null and b/docs/artifacts/full/step_00_before.png differ diff --git a/docs/artifacts/full/step_01_after.png b/docs/artifacts/full/step_01_after.png new file mode 100644 index 0000000..ac6cead Binary files /dev/null and b/docs/artifacts/full/step_01_after.png differ diff --git a/docs/artifacts/full/step_01_before.png b/docs/artifacts/full/step_01_before.png new file mode 100644 index 0000000..5238ebe Binary files /dev/null and b/docs/artifacts/full/step_01_before.png differ diff --git a/docs/artifacts/full/step_02_after.png b/docs/artifacts/full/step_02_after.png new file mode 100644 index 0000000..b707921 Binary files /dev/null and b/docs/artifacts/full/step_02_after.png differ diff --git a/docs/artifacts/full/step_02_before.png b/docs/artifacts/full/step_02_before.png new file mode 100644 index 0000000..ac6cead Binary files /dev/null and b/docs/artifacts/full/step_02_before.png differ diff --git a/docs/artifacts/full/step_03_after.png b/docs/artifacts/full/step_03_after.png new file mode 100644 index 0000000..9cbbbf4 Binary files /dev/null and b/docs/artifacts/full/step_03_after.png differ diff --git a/docs/artifacts/full/step_03_before.png b/docs/artifacts/full/step_03_before.png new file mode 100644 index 0000000..b707921 Binary files /dev/null and b/docs/artifacts/full/step_03_before.png differ diff --git a/docs/artifacts/full/step_04_after.png b/docs/artifacts/full/step_04_after.png new file mode 100644 index 0000000..d4e8d7b Binary files /dev/null and b/docs/artifacts/full/step_04_after.png differ diff --git a/docs/artifacts/full/step_04_before.png b/docs/artifacts/full/step_04_before.png new file mode 100644 index 0000000..9cbbbf4 Binary files /dev/null and b/docs/artifacts/full/step_04_before.png differ diff --git a/docs/artifacts/full/step_05_after.png b/docs/artifacts/full/step_05_after.png new file mode 100644 index 0000000..338ed3a Binary files /dev/null and b/docs/artifacts/full/step_05_after.png differ diff --git a/docs/artifacts/full/step_05_before.png b/docs/artifacts/full/step_05_before.png new file mode 100644 index 0000000..d4e8d7b Binary files /dev/null and b/docs/artifacts/full/step_05_before.png differ diff --git a/docs/artifacts/full/step_06_after.png b/docs/artifacts/full/step_06_after.png new file mode 100644 index 0000000..45bdc78 Binary files /dev/null and b/docs/artifacts/full/step_06_after.png differ diff --git a/docs/artifacts/full/step_06_before.png b/docs/artifacts/full/step_06_before.png new file mode 100644 index 0000000..338ed3a Binary files /dev/null and b/docs/artifacts/full/step_06_before.png differ diff --git a/docs/artifacts/full/step_07_after.png b/docs/artifacts/full/step_07_after.png new file mode 100644 index 0000000..9e8af20 Binary files /dev/null and b/docs/artifacts/full/step_07_after.png differ diff --git a/docs/artifacts/full/step_07_before.png b/docs/artifacts/full/step_07_before.png new file mode 100644 index 0000000..45bdc78 Binary files /dev/null and b/docs/artifacts/full/step_07_before.png differ diff --git a/docs/artifacts/full/step_08_after.png b/docs/artifacts/full/step_08_after.png new file mode 100644 index 0000000..97a32e8 Binary files /dev/null and b/docs/artifacts/full/step_08_after.png differ diff --git a/docs/artifacts/full/step_08_before.png b/docs/artifacts/full/step_08_before.png new file mode 100644 index 0000000..9e8af20 Binary files /dev/null and b/docs/artifacts/full/step_08_before.png differ diff --git a/docs/artifacts/full/step_09_after.png b/docs/artifacts/full/step_09_after.png new file mode 100644 index 0000000..ad17079 Binary files /dev/null and b/docs/artifacts/full/step_09_after.png differ diff --git a/docs/artifacts/full/step_09_before.png b/docs/artifacts/full/step_09_before.png new file mode 100644 index 0000000..97a32e8 Binary files /dev/null and b/docs/artifacts/full/step_09_before.png differ diff --git a/docs/artifacts/full/step_10_after.png b/docs/artifacts/full/step_10_after.png new file mode 100644 index 0000000..57f6fe2 Binary files /dev/null and b/docs/artifacts/full/step_10_after.png differ diff --git a/docs/artifacts/full/step_10_before.png b/docs/artifacts/full/step_10_before.png new file mode 100644 index 0000000..ad17079 Binary files /dev/null and b/docs/artifacts/full/step_10_before.png differ diff --git a/docs/artifacts/full/step_11_after.png b/docs/artifacts/full/step_11_after.png new file mode 100644 index 0000000..0ad4163 Binary files /dev/null and b/docs/artifacts/full/step_11_after.png differ diff --git a/docs/artifacts/full/step_11_before.png b/docs/artifacts/full/step_11_before.png new file mode 100644 index 0000000..57f6fe2 Binary files /dev/null and b/docs/artifacts/full/step_11_before.png differ diff --git a/docs/artifacts/full/step_12_after.png b/docs/artifacts/full/step_12_after.png new file mode 100644 index 0000000..0b8df70 Binary files /dev/null and b/docs/artifacts/full/step_12_after.png differ diff --git a/docs/artifacts/full/step_12_before.png b/docs/artifacts/full/step_12_before.png new file mode 100644 index 0000000..0ad4163 Binary files /dev/null and b/docs/artifacts/full/step_12_before.png differ diff --git a/docs/artifacts/full/step_13_after.png b/docs/artifacts/full/step_13_after.png new file mode 100644 index 0000000..a033c99 Binary files /dev/null and b/docs/artifacts/full/step_13_after.png differ diff --git a/docs/artifacts/full/step_13_before.png b/docs/artifacts/full/step_13_before.png new file mode 100644 index 0000000..0b8df70 Binary files /dev/null and b/docs/artifacts/full/step_13_before.png differ diff --git a/docs/artifacts/full/step_14_after.png b/docs/artifacts/full/step_14_after.png new file mode 100644 index 0000000..0d47824 Binary files /dev/null and b/docs/artifacts/full/step_14_after.png differ diff --git a/docs/artifacts/full/step_14_before.png b/docs/artifacts/full/step_14_before.png new file mode 100644 index 0000000..a033c99 Binary files /dev/null and b/docs/artifacts/full/step_14_before.png differ diff --git a/docs/artifacts/full/step_15_after.png b/docs/artifacts/full/step_15_after.png new file mode 100644 index 0000000..6ce2d6f Binary files /dev/null and b/docs/artifacts/full/step_15_after.png differ diff --git a/docs/artifacts/full/step_15_before.png b/docs/artifacts/full/step_15_before.png new file mode 100644 index 0000000..0d47824 Binary files /dev/null and b/docs/artifacts/full/step_15_before.png differ diff --git a/docs/artifacts/full/step_16_after.png b/docs/artifacts/full/step_16_after.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/docs/artifacts/full/step_16_after.png differ diff --git a/docs/artifacts/full/step_16_before.png b/docs/artifacts/full/step_16_before.png new file mode 100644 index 0000000..6ce2d6f Binary files /dev/null and b/docs/artifacts/full/step_16_before.png differ diff --git a/docs/artifacts/full/step_17_after.png b/docs/artifacts/full/step_17_after.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/docs/artifacts/full/step_17_after.png differ diff --git a/docs/artifacts/full/step_17_before.png b/docs/artifacts/full/step_17_before.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/docs/artifacts/full/step_17_before.png differ diff --git a/docs/artifacts/full/step_18_after.png b/docs/artifacts/full/step_18_after.png new file mode 100644 index 0000000..69ff4a9 Binary files /dev/null and b/docs/artifacts/full/step_18_after.png differ diff --git a/docs/artifacts/full/step_18_before.png b/docs/artifacts/full/step_18_before.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/docs/artifacts/full/step_18_before.png differ diff --git a/docs/artifacts/full/step_19_after.png b/docs/artifacts/full/step_19_after.png new file mode 100644 index 0000000..10a824b Binary files /dev/null and b/docs/artifacts/full/step_19_after.png differ diff --git a/docs/artifacts/full/step_19_before.png b/docs/artifacts/full/step_19_before.png new file mode 100644 index 0000000..75af98b Binary files /dev/null and b/docs/artifacts/full/step_19_before.png differ diff --git a/docs/artifacts/full/step_20_after.png b/docs/artifacts/full/step_20_after.png new file mode 100644 index 0000000..81a1f90 Binary files /dev/null and b/docs/artifacts/full/step_20_after.png differ diff --git a/docs/artifacts/full/step_20_before.png b/docs/artifacts/full/step_20_before.png new file mode 100644 index 0000000..10a824b Binary files /dev/null and b/docs/artifacts/full/step_20_before.png differ diff --git a/docs/artifacts/thumbnails/step_00_after.png b/docs/artifacts/thumbnails/step_00_after.png new file mode 100644 index 0000000..a02cd08 Binary files /dev/null and b/docs/artifacts/thumbnails/step_00_after.png differ diff --git a/docs/artifacts/thumbnails/step_00_before.png b/docs/artifacts/thumbnails/step_00_before.png new file mode 100644 index 0000000..629d7e2 Binary files /dev/null and b/docs/artifacts/thumbnails/step_00_before.png differ diff --git a/docs/artifacts/thumbnails/step_01_after.png b/docs/artifacts/thumbnails/step_01_after.png new file mode 100644 index 0000000..e97144a Binary files /dev/null and b/docs/artifacts/thumbnails/step_01_after.png differ diff --git a/docs/artifacts/thumbnails/step_01_before.png b/docs/artifacts/thumbnails/step_01_before.png new file mode 100644 index 0000000..a02cd08 Binary files /dev/null and b/docs/artifacts/thumbnails/step_01_before.png differ diff --git a/docs/artifacts/thumbnails/step_02_after.png b/docs/artifacts/thumbnails/step_02_after.png new file mode 100644 index 0000000..4a9e03e Binary files /dev/null and b/docs/artifacts/thumbnails/step_02_after.png differ diff --git a/docs/artifacts/thumbnails/step_02_before.png b/docs/artifacts/thumbnails/step_02_before.png new file mode 100644 index 0000000..e97144a Binary files /dev/null and b/docs/artifacts/thumbnails/step_02_before.png differ diff --git a/docs/artifacts/thumbnails/step_03_after.png b/docs/artifacts/thumbnails/step_03_after.png new file mode 100644 index 0000000..ce1981c Binary files /dev/null and b/docs/artifacts/thumbnails/step_03_after.png differ diff --git a/docs/artifacts/thumbnails/step_03_before.png b/docs/artifacts/thumbnails/step_03_before.png new file mode 100644 index 0000000..4a9e03e Binary files /dev/null and b/docs/artifacts/thumbnails/step_03_before.png differ diff --git a/docs/artifacts/thumbnails/step_04_after.png b/docs/artifacts/thumbnails/step_04_after.png new file mode 100644 index 0000000..959580e Binary files /dev/null and b/docs/artifacts/thumbnails/step_04_after.png differ diff --git a/docs/artifacts/thumbnails/step_04_before.png b/docs/artifacts/thumbnails/step_04_before.png new file mode 100644 index 0000000..ce1981c Binary files /dev/null and b/docs/artifacts/thumbnails/step_04_before.png differ diff --git a/docs/artifacts/thumbnails/step_05_after.png b/docs/artifacts/thumbnails/step_05_after.png new file mode 100644 index 0000000..21cbb1b Binary files /dev/null and b/docs/artifacts/thumbnails/step_05_after.png differ diff --git a/docs/artifacts/thumbnails/step_05_before.png b/docs/artifacts/thumbnails/step_05_before.png new file mode 100644 index 0000000..959580e Binary files /dev/null and b/docs/artifacts/thumbnails/step_05_before.png differ diff --git a/docs/artifacts/thumbnails/step_06_after.png b/docs/artifacts/thumbnails/step_06_after.png new file mode 100644 index 0000000..4e64436 Binary files /dev/null and b/docs/artifacts/thumbnails/step_06_after.png differ diff --git a/docs/artifacts/thumbnails/step_06_before.png b/docs/artifacts/thumbnails/step_06_before.png new file mode 100644 index 0000000..21cbb1b Binary files /dev/null and b/docs/artifacts/thumbnails/step_06_before.png differ diff --git a/docs/artifacts/thumbnails/step_07_after.png b/docs/artifacts/thumbnails/step_07_after.png new file mode 100644 index 0000000..2be5e55 Binary files /dev/null and b/docs/artifacts/thumbnails/step_07_after.png differ diff --git a/docs/artifacts/thumbnails/step_07_before.png b/docs/artifacts/thumbnails/step_07_before.png new file mode 100644 index 0000000..4e64436 Binary files /dev/null and b/docs/artifacts/thumbnails/step_07_before.png differ diff --git a/docs/artifacts/thumbnails/step_08_after.png b/docs/artifacts/thumbnails/step_08_after.png new file mode 100644 index 0000000..e51f42d Binary files /dev/null and b/docs/artifacts/thumbnails/step_08_after.png differ diff --git a/docs/artifacts/thumbnails/step_08_before.png b/docs/artifacts/thumbnails/step_08_before.png new file mode 100644 index 0000000..2be5e55 Binary files /dev/null and b/docs/artifacts/thumbnails/step_08_before.png differ diff --git a/docs/artifacts/thumbnails/step_09_after.png b/docs/artifacts/thumbnails/step_09_after.png new file mode 100644 index 0000000..d6ff229 Binary files /dev/null and b/docs/artifacts/thumbnails/step_09_after.png differ diff --git a/docs/artifacts/thumbnails/step_09_before.png b/docs/artifacts/thumbnails/step_09_before.png new file mode 100644 index 0000000..e51f42d Binary files /dev/null and b/docs/artifacts/thumbnails/step_09_before.png differ diff --git a/docs/artifacts/thumbnails/step_10_after.png b/docs/artifacts/thumbnails/step_10_after.png new file mode 100644 index 0000000..10d9751 Binary files /dev/null and b/docs/artifacts/thumbnails/step_10_after.png differ diff --git a/docs/artifacts/thumbnails/step_10_before.png b/docs/artifacts/thumbnails/step_10_before.png new file mode 100644 index 0000000..d6ff229 Binary files /dev/null and b/docs/artifacts/thumbnails/step_10_before.png differ diff --git a/docs/artifacts/thumbnails/step_11_after.png b/docs/artifacts/thumbnails/step_11_after.png new file mode 100644 index 0000000..9e488cc Binary files /dev/null and b/docs/artifacts/thumbnails/step_11_after.png differ diff --git a/docs/artifacts/thumbnails/step_11_before.png b/docs/artifacts/thumbnails/step_11_before.png new file mode 100644 index 0000000..10d9751 Binary files /dev/null and b/docs/artifacts/thumbnails/step_11_before.png differ diff --git a/docs/artifacts/thumbnails/step_12_after.png b/docs/artifacts/thumbnails/step_12_after.png new file mode 100644 index 0000000..3642d64 Binary files /dev/null and b/docs/artifacts/thumbnails/step_12_after.png differ diff --git a/docs/artifacts/thumbnails/step_12_before.png b/docs/artifacts/thumbnails/step_12_before.png new file mode 100644 index 0000000..9e488cc Binary files /dev/null and b/docs/artifacts/thumbnails/step_12_before.png differ diff --git a/docs/artifacts/thumbnails/step_13_after.png b/docs/artifacts/thumbnails/step_13_after.png new file mode 100644 index 0000000..3ab6e1b Binary files /dev/null and b/docs/artifacts/thumbnails/step_13_after.png differ diff --git a/docs/artifacts/thumbnails/step_13_before.png b/docs/artifacts/thumbnails/step_13_before.png new file mode 100644 index 0000000..3642d64 Binary files /dev/null and b/docs/artifacts/thumbnails/step_13_before.png differ diff --git a/docs/artifacts/thumbnails/step_14_after.png b/docs/artifacts/thumbnails/step_14_after.png new file mode 100644 index 0000000..c4273dc Binary files /dev/null and b/docs/artifacts/thumbnails/step_14_after.png differ diff --git a/docs/artifacts/thumbnails/step_14_before.png b/docs/artifacts/thumbnails/step_14_before.png new file mode 100644 index 0000000..3ab6e1b Binary files /dev/null and b/docs/artifacts/thumbnails/step_14_before.png differ diff --git a/docs/artifacts/thumbnails/step_15_after.png b/docs/artifacts/thumbnails/step_15_after.png new file mode 100644 index 0000000..6343b64 Binary files /dev/null and b/docs/artifacts/thumbnails/step_15_after.png differ diff --git a/docs/artifacts/thumbnails/step_15_before.png b/docs/artifacts/thumbnails/step_15_before.png new file mode 100644 index 0000000..c4273dc Binary files /dev/null and b/docs/artifacts/thumbnails/step_15_before.png differ diff --git a/docs/artifacts/thumbnails/step_16_after.png b/docs/artifacts/thumbnails/step_16_after.png new file mode 100644 index 0000000..8d3a369 Binary files /dev/null and b/docs/artifacts/thumbnails/step_16_after.png differ diff --git a/docs/artifacts/thumbnails/step_16_before.png b/docs/artifacts/thumbnails/step_16_before.png new file mode 100644 index 0000000..6343b64 Binary files /dev/null and b/docs/artifacts/thumbnails/step_16_before.png differ diff --git a/docs/artifacts/thumbnails/step_17_after.png b/docs/artifacts/thumbnails/step_17_after.png new file mode 100644 index 0000000..8d3a369 Binary files /dev/null and b/docs/artifacts/thumbnails/step_17_after.png differ diff --git a/docs/artifacts/thumbnails/step_17_before.png b/docs/artifacts/thumbnails/step_17_before.png new file mode 100644 index 0000000..8d3a369 Binary files /dev/null and b/docs/artifacts/thumbnails/step_17_before.png differ diff --git a/docs/artifacts/thumbnails/step_18_after.png b/docs/artifacts/thumbnails/step_18_after.png new file mode 100644 index 0000000..2a6c757 Binary files /dev/null and b/docs/artifacts/thumbnails/step_18_after.png differ diff --git a/docs/artifacts/thumbnails/step_18_before.png b/docs/artifacts/thumbnails/step_18_before.png new file mode 100644 index 0000000..8d3a369 Binary files /dev/null and b/docs/artifacts/thumbnails/step_18_before.png differ diff --git a/docs/artifacts/thumbnails/step_19_after.png b/docs/artifacts/thumbnails/step_19_after.png new file mode 100644 index 0000000..33456de Binary files /dev/null and b/docs/artifacts/thumbnails/step_19_after.png differ diff --git a/docs/artifacts/thumbnails/step_19_before.png b/docs/artifacts/thumbnails/step_19_before.png new file mode 100644 index 0000000..793eedb Binary files /dev/null and b/docs/artifacts/thumbnails/step_19_before.png differ diff --git a/docs/artifacts/thumbnails/step_20_after.png b/docs/artifacts/thumbnails/step_20_after.png new file mode 100644 index 0000000..2151cd8 Binary files /dev/null and b/docs/artifacts/thumbnails/step_20_after.png differ diff --git a/docs/artifacts/thumbnails/step_20_before.png b/docs/artifacts/thumbnails/step_20_before.png new file mode 100644 index 0000000..33456de Binary files /dev/null and b/docs/artifacts/thumbnails/step_20_before.png differ diff --git a/docs/demo_review.md b/docs/demo_review.md new file mode 100644 index 0000000..85c8fa1 --- /dev/null +++ b/docs/demo_review.md @@ -0,0 +1,376 @@ +# Demo Pipeline Review + +**Task ID:** `04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS` + +**Instruction:** In a new sheet with 4 headers "Year", "CA changes", "FA changes", and "OA changes", calculate the annual changes for the Current Assets, Fixed Assets, and Other Assets columns. Set the results as percentage type. + +**Steps:** 21 + +**Recorded at:** 2026-03-01T23:31:25.670893+00:00 + +## Text vs VLM Comparison (First 3 Steps) + +| Step | Ground Truth | Text-Only Demo | VLM-Enriched Demo | +|------|-------------|----------------|-------------------| +| 1 | Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet" | Action: Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet" | Observation: The spreadsheet is open to "Sheet1," which contains financial data with columns for years, assets, and liabilities.
Intent: To create a new sheet for calculating and displaying annual changes in asset categories.
Action: Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet".
Result: A new, blank sheet named "Sheet2" is added to the workbook and displayed. | +| 2 | Click cell A1 and type "Year" | Action: Click cell A1 and type "Year" | Observation: The new sheet is blank, with cell A1 selected and ready for input.
Intent: To create a header row for organizing annual asset changes.
Action: Click cell A1 and type "Year"
Result: The text "Year" now appears in cell A1. | +| 3 | Press Tab and type "CA changes" | Action: Press Tab and type "CA changes" | Observation: The new sheet contains only the header "Year" in cell A1, with the cursor positioned in cell B1.
Intent: To add the next header, "CA changes," as part of setting up the required columns for annual asset changes.
Action: Press Tab and type "CA changes"
Result: The header "CA changes" is entered in cell B1, and the sheet now displays two headers: "Year" and "CA changes." | + +### Step 1: Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet" + +[![before](artifacts/thumbnails/step_00_before.png)](artifacts/full/step_00_before.png) [![after](artifacts/thumbnails/step_00_after.png)](artifacts/full/step_00_after.png) + +**Ground truth:** Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet" + +**Text demo:** Action: Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet" + +**VLM demo:** + +> Observation: The spreadsheet is open to "Sheet1," which contains financial data with columns for years, assets, and liabilities. +> Intent: To create a new sheet for calculating and displaying annual changes in asset categories. +> Action: Right-click on the "Sheet1" tab at the bottom and select "Insert Sheet" or "New Sheet". +> Result: A new, blank sheet named "Sheet2" is added to the workbook and displayed. + +--- + +### Step 2: Click cell A1 and type "Year" + +[![before](artifacts/thumbnails/step_01_before.png)](artifacts/full/step_01_before.png) [![after](artifacts/thumbnails/step_01_after.png)](artifacts/full/step_01_after.png) + +**Ground truth:** Click cell A1 and type "Year" + +**Text demo:** Action: Click cell A1 and type "Year" + +**VLM demo:** + +> Observation: The new sheet is blank, with cell A1 selected and ready for input. +> Intent: To create a header row for organizing annual asset changes. +> Action: Click cell A1 and type "Year" +> Result: The text "Year" now appears in cell A1. + +--- + +### Step 3: Press Tab and type "CA changes" + +[![before](artifacts/thumbnails/step_02_before.png)](artifacts/full/step_02_before.png) [![after](artifacts/thumbnails/step_02_after.png)](artifacts/full/step_02_after.png) + +**Ground truth:** Press Tab and type "CA changes" + +**Text demo:** Action: Press Tab and type "CA changes" + +**VLM demo:** + +> Observation: The new sheet contains only the header "Year" in cell A1, with the cursor positioned in cell B1. +> Intent: To add the next header, "CA changes," as part of setting up the required columns for annual asset changes. +> Action: Press Tab and type "CA changes" +> Result: The header "CA changes" is entered in cell B1, and the sheet now displays two headers: "Year" and "CA changes." + +--- + +### Step 4: Press Tab and type "FA changes" + +[![before](artifacts/thumbnails/step_03_before.png)](artifacts/full/step_03_before.png) [![after](artifacts/thumbnails/step_03_after.png)](artifacts/full/step_03_after.png) + +**Ground truth:** Press Tab and type "FA changes" + +**Text demo:** Action: Press Tab and type "FA changes" + +**VLM demo:** + +> Observation: The new sheet contains two headers, "Year" in cell A1 and "CA changes" in cell B1, with the cursor positioned in cell C1. +> Intent: To continue setting up the required headers for calculating annual changes in asset categories. +> Action: Press Tab and type "FA changes". +> Result: The header "FA changes" is added to cell C1. + +--- + +### Step 5: Press Tab and type "OA changes" + +[![before](artifacts/thumbnails/step_04_before.png)](artifacts/full/step_04_before.png) [![after](artifacts/thumbnails/step_04_after.png)](artifacts/full/step_04_after.png) + +**Ground truth:** Press Tab and type "OA changes" + +**Text demo:** Action: Press Tab and type "OA changes" + +**VLM demo:** + +> Observation: The new sheet contains three headers: "Year", "CA changes", and "FA changes" in the first row. +> Intent: To complete the set of required headers by adding "OA changes" as the fourth column. +> Action: Press Tab and type "OA changes". +> Result: The "OA changes" header is added in cell D1, completing the four required headers in the first row. + +--- + +### Step 6: Click cell A2 and type "2015" + +[![before](artifacts/thumbnails/step_05_before.png)](artifacts/full/step_05_before.png) [![after](artifacts/thumbnails/step_05_after.png)](artifacts/full/step_05_after.png) + +**Ground truth:** Click cell A2 and type "2015" + +**Text demo:** Action: Click cell A2 and type "2015" + +**VLM demo:** + +> Observation: The new sheet contains four headers: "Year", "CA changes", "FA changes", and "OA changes", with all cells below the headers empty. +> Intent: To begin entering annual data by specifying the first year in the "Year" column. +> Action: Click cell A2 and type "2015". +> Result: The value "2015" appears in cell A2 under the "Year" header. + +--- + +### Step 7: Press Enter and type "2016" + +[![before](artifacts/thumbnails/step_06_before.png)](artifacts/full/step_06_before.png) [![after](artifacts/thumbnails/step_06_after.png)](artifacts/full/step_06_after.png) + +**Ground truth:** Press Enter and type "2016" + +**Text demo:** Action: Press Enter and type "2016" + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes" in row 1, with cell A2 currently empty. +> Intent: To begin entering the years for which asset changes will be calculated. +> Action: Press Enter and type "2016" in cell A2. +> Result: The value "2016" appears in cell A2, starting the list of years under the "Year" header. + +--- + +### Step 8: Press Enter and type "2017" + +[![before](artifacts/thumbnails/step_07_before.png)](artifacts/full/step_07_before.png) [![after](artifacts/thumbnails/step_07_after.png)](artifacts/full/step_07_after.png) + +**Ground truth:** Press Enter and type "2017" + +**Text demo:** Action: Press Enter and type "2017" + +**VLM demo:** + +> Observation: The new sheet contains the headers "Year", "CA changes", "FA changes", and "OA changes" in row 1, and "2016" is entered in cell A2. +> Intent: To begin listing the years for which annual changes will be calculated. +> Action: Press Enter and type "2017". +> Result: "2017" is entered into cell A3, directly below "2016" in the "Year" column. + +--- + +### Step 9: Press Enter and type "2018" + +[![before](artifacts/thumbnails/step_08_before.png)](artifacts/full/step_08_before.png) [![after](artifacts/thumbnails/step_08_after.png)](artifacts/full/step_08_after.png) + +**Ground truth:** Press Enter and type "2018" + +**Text demo:** Action: Press Enter and type "2018" + +**VLM demo:** + +> Observation: The new sheet contains the headers "Year", "CA changes", "FA changes", and "OA changes", with the years 2016 and 2017 already listed in the "Year" column. +> Intent: To continue populating the "Year" column with the next chronological year for annual change calculations. +> Action: Press Enter and type "2018". +> Result: The value "2018" appears in the next row of the "Year" column. + +--- + +### Step 10: Press Enter and type "2019" + +[![before](artifacts/thumbnails/step_09_before.png)](artifacts/full/step_09_before.png) [![after](artifacts/thumbnails/step_09_after.png)](artifacts/full/step_09_after.png) + +**Ground truth:** Press Enter and type "2019" + +**Text demo:** Action: Press Enter and type "2019" + +**VLM demo:** + +> Observation: The new sheet contains four headers ("Year", "CA changes", "FA changes", "OA changes") and a column of years from 2015 to 2018 under "Year". +> Intent: To add the next year in the sequence to the "Year" column in preparation for calculating annual changes. +> Action: Press Enter and type "2019". +> Result: The value "2019" appears in the next row under the "Year" column. + +--- + +### Step 11: Click cell B2 and type "=(Sheet1.B3-Sheet1.B2)/Sheet1.B2" + +[![before](artifacts/thumbnails/step_10_before.png)](artifacts/full/step_10_before.png) [![after](artifacts/thumbnails/step_10_after.png)](artifacts/full/step_10_after.png) + +**Ground truth:** Click cell B2 and type "=(Sheet1.B3-Sheet1.B2)/Sheet1.B2" + +**Text demo:** Action: Click cell B2 and type "=(Sheet1.B3-Sheet1.B2)/Sheet1.B2" + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes", with years listed in column A, but no formulas or data in the other columns. +> Intent: To calculate the annual percentage change for Current Assets using data from Sheet1. +> Action: Click cell B2 and type "=(Sheet1.B3-Sheet1.B2)/Sheet1.B2". +> Result: Cell B2 now contains a formula to compute the percentage change in Current Assets between two years from Sheet1. + +--- + +### Step 12: Press Enter + +[![before](artifacts/thumbnails/step_11_before.png)](artifacts/full/step_11_before.png) [![after](artifacts/thumbnails/step_11_after.png)](artifacts/full/step_11_after.png) + +**Ground truth:** Press Enter + +**Text demo:** Action: Press Enter + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes", and the first row under "Year" is filled with "2016", while the "CA changes" cell (B2) contains a formula referencing another sheet. +> Intent: To confirm and apply the formula entered in cell B2 for calculating annual changes in Current Assets. +> Action: Press Enter +> Result: The formula in cell B2 is executed, displaying the calculated percentage change for "CA changes" in 2016. + +--- + +### Step 13: Click cell B2, then drag the fill handle down to B6 + +[![before](artifacts/thumbnails/step_12_before.png)](artifacts/full/step_12_before.png) [![after](artifacts/thumbnails/step_12_after.png)](artifacts/full/step_12_after.png) + +**Ground truth:** Click cell B2, then drag the fill handle down to B6 + +**Text demo:** Action: Click cell B2, then drag the fill handle down to B6 + +**VLM demo:** + +> Observation: The sheet displays headers "Year", "CA changes", "FA changes", and "OA changes", with cell B2 containing a formula and the rest of column B empty. +> Intent: To copy the formula in B2 down the column so that annual changes for "CA changes" are calculated for each year. +> Action: Click cell B2, then drag the fill handle down to B6. +> Result: The formula from B2 is filled down to B6, calculating "CA changes" for all listed years. + +--- + +### Step 14: Click cell C2 and type "=(Sheet1.C3-Sheet1.C2)/Sheet1.C2" + +[![before](artifacts/thumbnails/step_13_before.png)](artifacts/full/step_13_before.png) [![after](artifacts/thumbnails/step_13_after.png)](artifacts/full/step_13_after.png) + +**Ground truth:** Click cell C2 and type "=(Sheet1.C3-Sheet1.C2)/Sheet1.C2" + +**Text demo:** Action: Click cell C2 and type "=(Sheet1.C3-Sheet1.C2)/Sheet1.C2" + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes", and cell C2 is empty. +> Intent: To calculate the annual change for Fixed Assets as a percentage using data from Sheet1. +> Action: Click cell C2 and type "=(Sheet1.C3-Sheet1.C2)/Sheet1.C2" +> Result: Cell C2 now contains the formula to compute the percentage change in Fixed Assets between two years from Sheet1. + +--- + +### Step 15: Press Enter + +[![before](artifacts/thumbnails/step_14_before.png)](artifacts/full/step_14_before.png) [![after](artifacts/thumbnails/step_14_after.png)](artifacts/full/step_14_after.png) + +**Ground truth:** Press Enter + +**Text demo:** Action: Press Enter + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes" in row 1, and the first row of data is being entered in row 2. +> Intent: To confirm or complete the entry of a formula or value in the spreadsheet. +> Action: Press Enter. +> Result: The cell selection moves from the current cell (D2) to the next cell below (D3), confirming the entry in D2. + +--- + +### Step 16: Click cell C2, then drag the fill handle down to C6 + +[![before](artifacts/thumbnails/step_15_before.png)](artifacts/full/step_15_before.png) [![after](artifacts/thumbnails/step_15_after.png)](artifacts/full/step_15_after.png) + +**Ground truth:** Click cell C2, then drag the fill handle down to C6 + +**Text demo:** Action: Click cell C2, then drag the fill handle down to C6 + +**VLM demo:** + +> Observation: The sheet displays headers "Year", "CA changes", "FA changes", and "OA changes" with a formula present in cell C2, but the rest of column C is empty. +> Intent: To apply the formula in C2 to the rest of the rows in the "CA changes" column for consistent annual change calculations. +> Action: Click cell C2, then drag the fill handle down to C6. +> Result: The formula from C2 is copied down to C6, populating the "CA changes" column with calculated values for each year. + +--- + +### Step 17: Click cell D2 and type "=(Sheet1.D3-Sheet1.D2)/Sheet1.D2" + +[![before](artifacts/thumbnails/step_16_before.png)](artifacts/full/step_16_before.png) [![after](artifacts/thumbnails/step_16_after.png)](artifacts/full/step_16_after.png) + +**Ground truth:** Click cell D2 and type "=(Sheet1.D3-Sheet1.D2)/Sheet1.D2" + +**Text demo:** Action: Click cell D2 and type "=(Sheet1.D3-Sheet1.D2)/Sheet1.D2" + +**VLM demo:** + +> Observation: The new sheet contains headers "Year", "CA changes", "FA changes", and "OA changes" in row 1, with all other cells empty. +> Intent: To calculate the annual percentage change for the "OA changes" column using data from Sheet1. +> Action: Click cell D2 and type "=(Sheet1.D3-Sheet1.D2)/Sheet1.D2" +> Result: Cell D2 is populated with the formula to compute the percentage change for "OA changes" between two years. + +--- + +### Step 18: Press Enter + +[![before](artifacts/thumbnails/step_17_before.png)](artifacts/full/step_17_before.png) [![after](artifacts/thumbnails/step_17_after.png)](artifacts/full/step_17_after.png) + +**Ground truth:** Press Enter + +**Text demo:** Action: Press Enter + +**VLM demo:** + +> Observation: The formula `=(C3-C2)/C2` is being entered in cell D3 to calculate the percentage change for "OA changes". +> Intent: To compute the annual percentage change for Other Assets in the new sheet. +> Action: Press Enter +> Result: The formula is executed in cell D3, displaying the calculated percentage change for Other Assets. + +--- + +### Step 19: Click cell D2, then drag the fill handle down to D6 + +[![before](artifacts/thumbnails/step_18_before.png)](artifacts/full/step_18_before.png) [![after](artifacts/thumbnails/step_18_after.png)](artifacts/full/step_18_after.png) + +**Ground truth:** Click cell D2, then drag the fill handle down to D6 + +**Text demo:** Action: Click cell D2, then drag the fill handle down to D6 + +**VLM demo:** + +> Observation: The sheet displays headers "Year", "CA changes", "FA changes", and "OA changes" with a formula present only in cell D2. +> Intent: To copy the formula in D2 down the "OA changes" column for all relevant rows. +> Action: Click cell D2, then drag the fill handle down to D6. +> Result: The formula from D2 is filled down through D6, calculating values for each row in the "OA changes" column. + +--- + +### Step 20: Click and drag to select cells B2:D6 + +[![before](artifacts/thumbnails/step_19_before.png)](artifacts/full/step_19_before.png) [![after](artifacts/thumbnails/step_19_after.png)](artifacts/full/step_19_after.png) + +**Ground truth:** Click and drag to select cells B2:D6 + +**Text demo:** Action: Click and drag to select cells B2:D6 + +**VLM demo:** + +> Observation: The new sheet contains four headers ("Year", "CA changes", "FA changes", "OA changes") and calculated annual changes for each asset type in columns B, C, and D. +> Intent: To select the range of calculated annual changes and their corresponding years for further formatting or analysis. +> Action: Click and drag to select cells B2:D6. +> Result: Cells B2:D6 are highlighted, indicating they are selected for the next operation. + +--- + +### Step 21: Click the % button in the toolbar (or press Ctrl+Shift+5) + +[![before](artifacts/thumbnails/step_20_before.png)](artifacts/full/step_20_before.png) [![after](artifacts/thumbnails/step_20_after.png)](artifacts/full/step_20_after.png) + +**Ground truth:** Click the % button in the toolbar (or press Ctrl+Shift+5) + +**Text demo:** Action: Click the % button in the toolbar (or press Ctrl+Shift+5) + +**VLM demo:** + +> Observation: The annual changes for "CA changes", "FA changes", and "OA changes" are displayed as decimal values in columns B, C, and D. +> Intent: To format the annual change values as percentages for better readability. +> Action: Click the % button in the toolbar (or press Ctrl+Shift+5). +> Result: The values in columns B, C, and D are now displayed as percentages. + +--- + +*Generated by `scripts/generate_demo_review.py` from recording `04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS`* diff --git a/scripts/generate_demo_review.py b/scripts/generate_demo_review.py new file mode 100644 index 0000000..8c741c5 --- /dev/null +++ b/scripts/generate_demo_review.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Generate a markdown review artifact for the demo recording pipeline. + +Reads a WAA recording (meta.json + screenshots), creates thumbnail images, +and produces a markdown file showing the pipeline output for each step. +The markdown is suitable for embedding in docs or PR descriptions and +renders on GitHub with relative image paths. Thumbnails link to full-resolution +originals when available. + +Usage: + python scripts/generate_demo_review.py \ + --recording waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS \ + --text-demo demo_prompts/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS.txt \ + --vlm-demo demo_prompts_vlm/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS.txt \ + --output docs/demo_review.md +""" +from __future__ import annotations + +import json +import re +import shutil +import sys +from pathlib import Path + +from PIL import Image + + +THUMBNAIL_WIDTH = 600 + + +def _parse_demo_steps(demo_text: str) -> dict[int, str]: + """Parse a demo .txt file into a dict mapping step number -> step content. + + Handles both text-only and VLM-enriched formats. Returns the full text + block for each step (everything between "Step N:" markers). + """ + steps: dict[int, str] = {} + # Split on "Step N:" headers, capturing the step number + parts = re.split(r'^(Step \d+:)\s*$', demo_text, flags=re.MULTILINE) + + # parts looks like: [preamble, "Step 1:", content, "Step 2:", content, ...] + for i in range(1, len(parts) - 1, 2): + header = parts[i] # e.g. "Step 3:" + content = parts[i + 1] + step_num = int(re.search(r'\d+', header).group()) + # Strip trailing blank lines / separators but preserve internal structure + content = content.strip() + # Remove trailing "---" if it's the last step + if content.endswith("---"): + content = content[:-3].strip() + steps[step_num] = content + + return steps + + +def _create_thumbnail(src: Path, dst: Path, width: int = THUMBNAIL_WIDTH) -> None: + """Resize an image to the given width, preserving aspect ratio.""" + with Image.open(src) as img: + if img.width <= width: + shutil.copy2(src, dst) + return + ratio = width / img.width + new_height = int(img.height * ratio) + resized = img.resize((width, new_height), Image.LANCZOS) + resized.save(dst, optimize=True) + + +def _relpath(target: Path, start: Path) -> str: + """Compute a relative path from start to target, suitable for markdown.""" + try: + return str(target.resolve().relative_to(start.resolve())) + except ValueError: + import os + return os.path.relpath(target.resolve(), start.resolve()) + + +def _escape_md(text: str) -> str: + """Minimal escaping so that text doesn't break markdown tables.""" + return text.replace("|", "\\|").replace("\n", "
") + + +def _indent_block(text: str, prefix: str = "> ") -> str: + """Indent every line of text with the given prefix.""" + return "\n".join(prefix + line for line in text.split("\n")) + + +def main( + recording: str, + text_demo: str | None = None, + vlm_demo: str | None = None, + output: str = "docs/demo_review.md", + thumbnail_width: int = THUMBNAIL_WIDTH, +) -> None: + """Generate a markdown review of the demo pipeline output. + + Args: + recording: Path to the recording directory (contains meta.json + PNGs). + text_demo: Path to the text-only demo .txt file. + vlm_demo: Path to the VLM-enriched demo .txt file. + output: Output path for the generated markdown file. + thumbnail_width: Width in pixels for thumbnail images. + """ + recording_dir = Path(recording) + output_path = Path(output) + + # --- Validate inputs --- + meta_path = recording_dir / "meta.json" + if not meta_path.exists(): + print(f"Error: meta.json not found in {recording_dir}") + sys.exit(1) + + meta = json.loads(meta_path.read_text(encoding="utf-8")) + task_id = meta["task_id"] + instruction = meta["instruction"] + num_steps = meta.get("num_steps", len(meta.get("steps", []))) + steps = meta.get("steps", []) + recorded_at = meta.get("recorded_at", "unknown") + + # --- Parse demo files --- + text_steps: dict[int, str] = {} + vlm_steps: dict[int, str] = {} + + if text_demo: + text_demo_path = Path(text_demo) + if text_demo_path.exists(): + text_steps = _parse_demo_steps( + text_demo_path.read_text(encoding="utf-8") + ) + else: + print(f"Warning: text demo not found at {text_demo_path}") + + if vlm_demo: + vlm_demo_path = Path(vlm_demo) + if vlm_demo_path.exists(): + vlm_steps = _parse_demo_steps( + vlm_demo_path.read_text(encoding="utf-8") + ) + else: + print(f"Warning: VLM demo not found at {vlm_demo_path}") + + # --- Create thumbnails and copy full-res originals --- + thumb_dir = output_path.parent / "artifacts" / "thumbnails" + thumb_dir.mkdir(parents=True, exist_ok=True) + + full_dir = output_path.parent / "artifacts" / "full" + full_dir.mkdir(parents=True, exist_ok=True) + + thumbnail_map: dict[str, Path] = {} + full_map: dict[str, Path] = {} + for i in range(num_steps): + for suffix in ("before", "after"): + name = f"step_{i:02d}_{suffix}" + src = recording_dir / f"{name}.png" + if src.exists(): + dst = thumb_dir / f"{name}.png" + _create_thumbnail(src, dst, width=thumbnail_width) + thumbnail_map[name] = dst + # Copy full-resolution original + full_dst = full_dir / f"{name}.png" + shutil.copy2(src, full_dst) + full_map[name] = full_dst + + print(f"Created {len(thumbnail_map)} thumbnails in {thumb_dir}") + if full_map: + print(f"Copied {len(full_map)} full-resolution images to {full_dir}") + + # --- Build markdown --- + md_dir = output_path.parent + md_dir.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + + # Header + lines.append("# Demo Pipeline Review") + lines.append("") + lines.append(f"**Task ID:** `{task_id}`") + lines.append("") + lines.append(f"**Instruction:** {instruction}") + lines.append("") + lines.append(f"**Steps:** {num_steps}") + lines.append("") + lines.append(f"**Recorded at:** {recorded_at}") + lines.append("") + + # --- Comparison table (first 3 steps) --- + compare_count = min(3, num_steps) + if text_steps or vlm_steps: + lines.append("## Text vs VLM Comparison (First 3 Steps)") + lines.append("") + lines.append( + "| Step | Ground Truth | Text-Only Demo | VLM-Enriched Demo |" + ) + lines.append("|------|-------------|----------------|-------------------|") + + for i in range(compare_count): + step_num = i + 1 + gt = steps[i].get("suggested_step", "") if i < len(steps) else "" + text_content = _escape_md(text_steps.get(step_num, "*(not available)*")) + vlm_content = _escape_md(vlm_steps.get(step_num, "*(not available)*")) + gt_escaped = _escape_md(gt) + lines.append( + f"| {step_num} | {gt_escaped} | {text_content} | {vlm_content} |" + ) + + lines.append("") + + # --- Per-step details (expanded, not collapsed) --- + for i in range(num_steps): + step_num = i + 1 + gt = steps[i].get("suggested_step", f"(step {step_num})") if i < len(steps) else f"(step {step_num})" + + lines.append(f"### Step {step_num}: {gt}") + lines.append("") + + # Screenshots — thumbnails that link to full-resolution when available + before_key = f"step_{i:02d}_before" + after_key = f"step_{i:02d}_after" + has_before = before_key in thumbnail_map + has_after = after_key in thumbnail_map + + if has_before or has_after: + if has_before and has_after: + before_thumb = _relpath(thumbnail_map[before_key], md_dir) + after_thumb = _relpath(thumbnail_map[after_key], md_dir) + if before_key in full_map: + before_full = _relpath(full_map[before_key], md_dir) + after_full = _relpath(full_map[after_key], md_dir) + lines.append( + f"[![before]({before_thumb})]({before_full}) " + f"[![after]({after_thumb})]({after_full})" + ) + else: + lines.append( + f"![before]({before_thumb}) " + f"![after]({after_thumb})" + ) + elif has_before: + before_thumb = _relpath(thumbnail_map[before_key], md_dir) + if before_key in full_map: + before_full = _relpath(full_map[before_key], md_dir) + lines.append(f"[![before]({before_thumb})]({before_full})") + else: + lines.append(f"![before]({before_thumb})") + elif has_after: + after_thumb = _relpath(thumbnail_map[after_key], md_dir) + if after_key in full_map: + after_full = _relpath(full_map[after_key], md_dir) + lines.append(f"[![after]({after_thumb})]({after_full})") + else: + lines.append(f"![after]({after_thumb})") + + lines.append("") + + # Ground truth + lines.append(f"**Ground truth:** {gt}") + lines.append("") + + # Text-only demo output + if text_steps: + text_content = text_steps.get(step_num) + if text_content: + lines.append(f"**Text demo:** {text_content}") + else: + lines.append("**Text demo:** *(not available)*") + lines.append("") + + # VLM-enriched demo output + if vlm_steps: + vlm_content = vlm_steps.get(step_num) + if vlm_content: + lines.append("**VLM demo:**") + lines.append("") + lines.append(_indent_block(vlm_content)) + else: + lines.append("**VLM demo:** *(not available)*") + lines.append("") + + lines.append("---") + lines.append("") + + # --- Footer --- + lines.append( + f"*Generated by `scripts/generate_demo_review.py` from recording " + f"`{recording_dir.name}`*" + ) + lines.append("") + + # Write output + md_text = "\n".join(lines) + output_path.write_text(md_text, encoding="utf-8") + print(f"Wrote {len(md_text)} bytes to {output_path}") + print(f" {num_steps} steps, {len(thumbnail_map)} thumbnails") + if text_steps: + print(f" Text-only demo: {len(text_steps)} steps parsed") + if vlm_steps: + print(f" VLM-enriched demo: {len(vlm_steps)} steps parsed") + + +if __name__ == "__main__": + import fire + + fire.Fire(main) diff --git a/scripts/record_waa_demos.py b/scripts/record_waa_demos.py index d61ff67..afbb30c 100644 --- a/scripts/record_waa_demos.py +++ b/scripts/record_waa_demos.py @@ -1356,9 +1356,10 @@ def _auto_start_socat(vm_ip: str) -> bool: capture_output=True, text=True, timeout=30, ) if result.returncode != 0: - print(f" ERROR: socat proxy setup failed: {result.stderr.strip()}") - return False - print(" Socat proxy established (VM:5051 -> container:5050).") + print(f" WARNING: socat setup returned non-zero: {result.stderr.strip()}") + # Not fatal — socat may already be running + else: + print(" Socat proxy established (VM:5051 -> container:5050).") return True diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta.json b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta.json new file mode 100644 index 0000000..b7f8a40 --- /dev/null +++ b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta.json @@ -0,0 +1,143 @@ +{ + "task_id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS", + "instruction": "In a new sheet with 4 headers \"Year\", \"CA changes\", \"FA changes\", and \"OA changes\", calculate the annual changes for the Current Assets, Fixed Assets, and Other Assets columns. Set the results as percentage type.", + "num_steps": 21, + "steps": [ + { + "action_hint": null, + "suggested_step": "Right-click on the \"Sheet1\" tab at the bottom and select \"Insert Sheet\" or \"New Sheet\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell A1 and type \"Year\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"CA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"FA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"OA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell A2 and type \"2015\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2016\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2017\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2018\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2019\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell B2 and type \"=(Sheet1.B3-Sheet1.B2)/Sheet1.B2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell B2, then drag the fill handle down to B6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell C2 and type \"=(Sheet1.C3-Sheet1.C2)/Sheet1.C2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell C2, then drag the fill handle down to C6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell D2 and type \"=(Sheet1.D3-Sheet1.D2)/Sheet1.D2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell D2, then drag the fill handle down to D6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click and drag to select cells B2:D6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click the % button in the toolbar (or press Ctrl+Shift+5)", + "step_was_refined": false + } + ], + "step_plans": [ + { + "at_step_idx": 0, + "trigger": "initial", + "steps": [ + "Right-click on the \"Sheet1\" tab at the bottom and select \"Insert Sheet\" or \"New Sheet\"", + "Click cell A1 and type \"Year\"", + "Press Tab and type \"CA changes\"", + "Press Tab and type \"FA changes\"", + "Press Tab and type \"OA changes\"", + "Click cell A2 and type \"2015\"", + "Press Enter and type \"2016\"", + "Press Enter and type \"2017\"", + "Press Enter and type \"2018\"", + "Press Enter and type \"2019\"", + "Click cell B2 and type \"=(Sheet1.B3-Sheet1.B2)/Sheet1.B2\"", + "Press Enter", + "Click cell B2, then drag the fill handle down to B6", + "Click cell C2 and type \"=(Sheet1.C3-Sheet1.C2)/Sheet1.C2\"", + "Press Enter", + "Click cell C2, then drag the fill handle down to C6", + "Click cell D2 and type \"=(Sheet1.D3-Sheet1.D2)/Sheet1.D2\"", + "Press Enter", + "Click cell D2, then drag the fill handle down to D6", + "Click and drag to select cells B2:D6", + "Click the % button in the toolbar (or press Ctrl+Shift+5)" + ] + } + ], + "server_url": "http://localhost:5001", + "recorded_at": "2026-03-01T23:31:25.670893+00:00" +} \ No newline at end of file diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta_refined.json b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta_refined.json new file mode 100644 index 0000000..e059e3d --- /dev/null +++ b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/meta_refined.json @@ -0,0 +1,145 @@ +{ + "task_id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS", + "instruction": "In a new sheet with 4 headers \"Year\", \"CA changes\", \"FA changes\", and \"OA changes\", calculate the annual changes for the Current Assets, Fixed Assets, and Other Assets columns. Set the results as percentage type.", + "num_steps": 21, + "steps": [ + { + "action_hint": null, + "suggested_step": "Right-click on the \"Sheet1\" tab at the bottom and select \"Insert Sheet\" or \"New Sheet\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell A1 and type \"Year\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"CA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"FA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Tab and type \"OA changes\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell A2 and type \"2015\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2016\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2017\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2018\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter and type \"2019\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell B2 and type \"=(Sheet1.B3-Sheet1.B2)/Sheet1.B2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell B2, then drag the fill handle down to B6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell C2 and type \"=(Sheet1.C3-Sheet1.C2)/Sheet1.C2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell C2, then drag the fill handle down to C6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell D2 and type \"=(Sheet1.D3-Sheet1.D2)/Sheet1.D2\"", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Press Enter", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click cell D2, then drag the fill handle down to D6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click and drag to select cells B2:D6", + "step_was_refined": false + }, + { + "action_hint": null, + "suggested_step": "Click the % button in the toolbar (or press Ctrl+Shift+5)", + "step_was_refined": false + } + ], + "step_plans": [ + { + "at_step_idx": 0, + "trigger": "initial", + "steps": [ + "Right-click on the \"Sheet1\" tab at the bottom and select \"Insert Sheet\" or \"New Sheet\"", + "Click cell A1 and type \"Year\"", + "Press Tab and type \"CA changes\"", + "Press Tab and type \"FA changes\"", + "Press Tab and type \"OA changes\"", + "Click cell A2 and type \"2015\"", + "Press Enter and type \"2016\"", + "Press Enter and type \"2017\"", + "Press Enter and type \"2018\"", + "Press Enter and type \"2019\"", + "Click cell B2 and type \"=(Sheet1.B3-Sheet1.B2)/Sheet1.B2\"", + "Press Enter", + "Click cell B2, then drag the fill handle down to B6", + "Click cell C2 and type \"=(Sheet1.C3-Sheet1.C2)/Sheet1.C2\"", + "Press Enter", + "Click cell C2, then drag the fill handle down to C6", + "Click cell D2 and type \"=(Sheet1.D3-Sheet1.D2)/Sheet1.D2\"", + "Press Enter", + "Click cell D2, then drag the fill handle down to D6", + "Click and drag to select cells B2:D6", + "Click the % button in the toolbar (or press Ctrl+Shift+5)" + ] + } + ], + "server_url": "http://localhost:5001", + "recorded_at": "2026-03-01T23:31:25.670893+00:00", + "refined_at": "2026-03-01T23:52:52.952200+00:00", + "original_num_steps": 21 +} \ No newline at end of file diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/refinement_log.json b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/refinement_log.json new file mode 100644 index 0000000..605f91f --- /dev/null +++ b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/refinement_log.json @@ -0,0 +1,11 @@ +{ + "task_id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS", + "recording_dir": "/Users/abrichr/oa/src/openadapt-evals/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS", + "refined_at": "2026-03-01T23:52:52.953346+00:00", + "total_steps": 21, + "steps_flagged": 0, + "steps_accepted": 0, + "steps_edited": 0, + "steps_rejected": 0, + "entries": [] +} \ No newline at end of file diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_after.png new file mode 100644 index 0000000..5238ebe Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_before.png new file mode 100644 index 0000000..b28a343 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_00_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_after.png new file mode 100644 index 0000000..ac6cead Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_before.png new file mode 100644 index 0000000..5238ebe Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_01_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_after.png new file mode 100644 index 0000000..b707921 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_before.png new file mode 100644 index 0000000..ac6cead Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_02_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_after.png new file mode 100644 index 0000000..9cbbbf4 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_before.png new file mode 100644 index 0000000..b707921 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_03_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_after.png new file mode 100644 index 0000000..d4e8d7b Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_before.png new file mode 100644 index 0000000..9cbbbf4 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_04_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_after.png new file mode 100644 index 0000000..338ed3a Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_before.png new file mode 100644 index 0000000..d4e8d7b Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_05_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_after.png new file mode 100644 index 0000000..45bdc78 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_before.png new file mode 100644 index 0000000..338ed3a Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_06_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_after.png new file mode 100644 index 0000000..9e8af20 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_before.png new file mode 100644 index 0000000..45bdc78 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_07_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_after.png new file mode 100644 index 0000000..97a32e8 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_before.png new file mode 100644 index 0000000..9e8af20 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_08_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_after.png new file mode 100644 index 0000000..ad17079 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_before.png new file mode 100644 index 0000000..97a32e8 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_09_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_after.png new file mode 100644 index 0000000..57f6fe2 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_before.png new file mode 100644 index 0000000..ad17079 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_10_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_after.png new file mode 100644 index 0000000..0ad4163 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_before.png new file mode 100644 index 0000000..57f6fe2 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_11_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_after.png new file mode 100644 index 0000000..0b8df70 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_before.png new file mode 100644 index 0000000..0ad4163 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_12_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_after.png new file mode 100644 index 0000000..a033c99 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_before.png new file mode 100644 index 0000000..0b8df70 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_13_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_after.png new file mode 100644 index 0000000..0d47824 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_before.png new file mode 100644 index 0000000..a033c99 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_14_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_after.png new file mode 100644 index 0000000..6ce2d6f Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_before.png new file mode 100644 index 0000000..0d47824 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_15_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_after.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_before.png new file mode 100644 index 0000000..6ce2d6f Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_16_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_after.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_before.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_17_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_after.png new file mode 100644 index 0000000..69ff4a9 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_before.png new file mode 100644 index 0000000..9f74534 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_18_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_after.png new file mode 100644 index 0000000..10a824b Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_before.png new file mode 100644 index 0000000..75af98b Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_19_before.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_after.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_after.png new file mode 100644 index 0000000..81a1f90 Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_after.png differ diff --git a/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_before.png b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_before.png new file mode 100644 index 0000000..10a824b Binary files /dev/null and b/waa_recordings/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f-WOS/step_20_before.png differ