From 823ac71e941e0d3e04d27a6f2cb94d9d2b2a2389 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 22 Oct 2025 16:03:59 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A4=96=20Add=20Chromatic=20for=20visu?= =?UTF-8?q?al=20regression=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add chromatic package for visual diff reviews on Storybook - Create GitHub Actions workflow for automated PR checks - Add chromatic config file with sensible defaults - Add comprehensive setup guide (CHROMATIC_SETUP.md) - Add 'make chromatic' command for local testing Requires CHROMATIC_PROJECT_TOKEN secret to be set in GitHub. Generated with `cmux` --- .github/workflows/chromatic.yml | 32 ++++++ CHROMATIC_SETUP.md | 172 ++++++++++++++++++++++++++++++++ Makefile | 5 +- bun.lock | 3 + chromatic.config.json | 9 ++ package.json | 6 +- 6 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/chromatic.yml create mode 100644 CHROMATIC_SETUP.md create mode 100644 chromatic.config.json diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.yml new file mode 100644 index 000000000..137738bc1 --- /dev/null +++ b/.github/workflows/chromatic.yml @@ -0,0 +1,32 @@ +name: Chromatic + +on: + push: + branches: + - main + pull_request: + branches: ["**"] + +jobs: + chromatic: + name: Visual Regression Testing + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Required for Chromatic to track changes + + - uses: ./.github/actions/setup-cmux + + - name: Generate version file + run: ./scripts/generate-version.sh + + - name: Run Chromatic + uses: chromaui/action@latest + with: + projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} + exitZeroOnChanges: true + onlyChanged: true + buildScriptName: storybook:build + diff --git a/CHROMATIC_SETUP.md b/CHROMATIC_SETUP.md new file mode 100644 index 000000000..7750297d9 --- /dev/null +++ b/CHROMATIC_SETUP.md @@ -0,0 +1,172 @@ +# 🎨 Chromatic Setup Guide + +This guide explains how to set up and use Chromatic for visual regression testing in cmux. + +## What is Chromatic? + +Chromatic is a visual testing platform that: +- Captures screenshots of your Storybook components +- Detects visual changes between commits +- Provides a UI review workflow for accepting/rejecting changes +- Integrates with GitHub PRs to show visual diffs + +## Initial Setup + +### 1. Create a Chromatic Account + +1. Go to [chromatic.com](https://www.chromatic.com/) +2. Sign in with your GitHub account +3. Click "Add project" and select the `coder/cmux` repository + +### 2. Get Your Project Token + +After creating the project, you'll receive a project token that looks like: +``` +chpt_1234567890abcdef +``` + +### 3. Configure GitHub Secrets + +Add the Chromatic project token to your GitHub repository: + +1. Go to repository **Settings** → **Secrets and variables** → **Actions** +2. Click **New repository secret** +3. Name: `CHROMATIC_PROJECT_TOKEN` +4. Value: Your Chromatic project token from step 2 +5. Click **Add secret** + +### 4. Update Configuration File + +Edit `chromatic.config.json` and replace the placeholder with your actual project ID: + +```json +{ + "projectId": "your-actual-project-id-here" +} +``` + +You can find your project ID in the Chromatic dashboard URL or in the setup instructions. + +## Usage + +### Running Chromatic Locally + +Test your Storybook components with Chromatic before pushing: + +```bash +# Set your project token (one-time setup) +export CHROMATIC_PROJECT_TOKEN=chpt_your_token_here + +# Run Chromatic +make chromatic +# or +bun run chromatic +``` + +### Automated PR Checks + +Chromatic automatically runs on every pull request via GitHub Actions. The workflow: + +1. Builds your Storybook +2. Captures screenshots of all stories +3. Compares them against the baseline (main branch) +4. Posts a status check and comment on your PR with: + - Number of visual changes detected + - Link to review changes in Chromatic UI + - Direct comparison images + +### Reviewing Changes + +When Chromatic detects visual changes: + +1. Click the **"View changes in Chromatic"** link in the PR +2. Review each component change in the Chromatic UI +3. Accept or reject changes: + - ✅ **Accept**: Updates the baseline for future comparisons + - ❌ **Reject**: Keeps the existing baseline (treat as a regression) + +### Workflow Integration + +The Chromatic check is **informational** (won't block PRs) thanks to `exitZeroOnChanges: true`. This allows you to: +- Review visual changes at your own pace +- Merge PRs even with pending visual reviews +- Accept/reject changes after merging if needed + +To make it a **required check** (block PRs with unreviewed changes): +1. Update `.github/workflows/chromatic.yml`: Remove `exitZeroOnChanges: true` +2. Add Chromatic as a required status check in GitHub branch protection + +## Best Practices + +### Component Development + +1. **Create stories for visual states**: Add Storybook stories for each significant visual state +2. **Keep stories focused**: Each story should demonstrate one specific state or variant +3. **Use controls**: Make stories interactive with Storybook controls for easier testing + +### Managing Changes + +1. **Accept intentional changes**: When you update styling, accept the Chromatic changes +2. **Investigate unexpected changes**: If Chromatic catches changes you didn't make, investigate before accepting +3. **Batch reviews**: Review multiple small changes together rather than one at a time + +### Performance + +- Chromatic uses **smart diffing** - only changed stories are captured +- The workflow uses `onlyChanged: true` to speed up builds +- First run takes longer (captures all baselines) +- Subsequent runs are much faster + +## Troubleshooting + +### "Project token not found" Error + +Make sure you've added `CHROMATIC_PROJECT_TOKEN` to GitHub Secrets and it matches your Chromatic project token exactly. + +### Workflow Fails to Build Storybook + +Check that: +1. `make storybook-build` works locally +2. All dependencies are installed +3. No TypeScript errors in story files + +### Visual Differences in Fonts/Colors + +This can happen due to OS-level rendering differences. Chromatic uses consistent rendering, but local previews might look different. Trust the Chromatic UI for accurate comparisons. + +### Too Many Changes Detected + +If you've refactored significantly: +1. Review and accept legitimate changes in batches +2. Consider using Chromatic's "Accept all" feature for major style updates +3. Use Chromatic's branch comparison to see changes against your feature branch + +## Configuration Reference + +### chromatic.config.json + +```json +{ + "projectId": "your-project-id", + "buildScriptName": "storybook:build", // npm script to build Storybook + "storybookBuildDir": "storybook-static", // Where Storybook builds to + "exitZeroOnChanges": true, // Don't fail on visual changes + "exitOnceUploaded": true, // Exit after upload completes + "autoAcceptChanges": false // Don't auto-accept changes +} +``` + +### Workflow Options + +In `.github/workflows/chromatic.yml`: + +- `onlyChanged: true` - Only test changed stories (faster) +- `exitZeroOnChanges: true` - Don't block PRs on visual changes +- `buildScriptName` - The npm/make script that builds Storybook + +## Resources + +- [Chromatic Documentation](https://www.chromatic.com/docs/) +- [Storybook Documentation](https://storybook.js.org/docs) +- [Visual Testing Best Practices](https://www.chromatic.com/docs/test) + diff --git a/Makefile b/Makefile index 3a217d5ba..3fbb97d42 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ include fmt.mk .PHONY: test test-unit test-integration test-watch test-coverage test-e2e .PHONY: dist dist-mac dist-win dist-linux .PHONY: docs docs-build docs-watch -.PHONY: storybook storybook-build test-storybook +.PHONY: storybook storybook-build test-storybook chromatic .PHONY: benchmark-terminal .PHONY: ensure-deps .PHONY: check-eager-imports check-bundle-size check-startup @@ -252,6 +252,9 @@ storybook-build: node_modules/.installed src/version.ts ## Build static Storyboo test-storybook: node_modules/.installed ## Run Storybook interaction tests (requires Storybook to be running or built) @bun x test-storybook +chromatic: node_modules/.installed ## Run Chromatic for visual regression testing + @bun x chromatic --exit-zero-on-changes + ## Benchmarks benchmark-terminal: ## Run Terminal-Bench with the cmux agent (use TB_DATASET/TB_SAMPLE_SIZE/TB_ARGS to customize) @TB_DATASET=$${TB_DATASET:-terminal-bench-core==0.1.1}; \ diff --git a/bun.lock b/bun.lock index b3f43700e..ca8a63105 100644 --- a/bun.lock +++ b/bun.lock @@ -58,6 +58,7 @@ "@typescript/native-preview": "^7.0.0-dev.20251014.1", "@vitejs/plugin-react": "^4.0.0", "babel-plugin-react-compiler": "^1.0.0", + "chromatic": "^13.3.1", "cmdk": "^1.0.0", "concurrently": "^8.2.0", "dotenv": "^17.2.3", @@ -1074,6 +1075,8 @@ "chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="], + "chromatic": ["chromatic@13.3.1", "", { "peerDependencies": { "@chromatic-com/cypress": "^0.*.* || ^1.0.0", "@chromatic-com/playwright": "^0.*.* || ^1.0.0" }, "optionalPeers": ["@chromatic-com/cypress", "@chromatic-com/playwright"], "bin": { "chroma": "dist/bin.js", "chromatic": "dist/bin.js", "chromatic-cli": "dist/bin.js" } }, "sha512-qJ/el70Wo7jFgiXPpuukqxCEc7IKiH/e8MjTzIF9uKw+3XZ6GghOTTLC7lGfeZtosiQBMkRlYet77tC4KKHUng=="], + "chromium-pickle-js": ["chromium-pickle-js@0.2.0", "", {}, "sha512-1R5Fho+jBq0DDydt+/vHWj5KJNJCKdARKOCwZUen84I5BreWoLqRLANH1U87eJy1tiASPtMnGqJJq0ZsLoRPOw=="], "ci-info": ["ci-info@3.9.0", "", {}, "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ=="], diff --git a/chromatic.config.json b/chromatic.config.json new file mode 100644 index 000000000..d1342d59f --- /dev/null +++ b/chromatic.config.json @@ -0,0 +1,9 @@ +{ + "projectId": "REPLACE_WITH_YOUR_PROJECT_ID", + "buildScriptName": "storybook:build", + "storybookBuildDir": "storybook-static", + "exitZeroOnChanges": true, + "exitOnceUploaded": true, + "autoAcceptChanges": false +} + diff --git a/package.json b/package.json index cadb9bba3..54bb6a86c 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "docs:watch": "make docs-watch", "storybook": "make storybook", "storybook:build": "make storybook-build", - "test:storybook": "make test-storybook" + "test:storybook": "make test-storybook", + "chromatic": "make chromatic" }, "dependencies": { "@ai-sdk/anthropic": "^2.0.29", @@ -98,6 +99,7 @@ "@typescript/native-preview": "^7.0.0-dev.20251014.1", "@vitejs/plugin-react": "^4.0.0", "babel-plugin-react-compiler": "^1.0.0", + "chromatic": "^13.3.1", "cmdk": "^1.0.0", "concurrently": "^8.2.0", "dotenv": "^17.2.3", @@ -105,10 +107,10 @@ "electron-builder": "^24.6.0", "electron-devtools-installer": "^4.0.0", "electron-mock-ipc": "^0.3.12", + "escape-html": "^1.0.3", "eslint": "^9.36.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-react-hooks": "^5.2.0", - "escape-html": "^1.0.3", "jest": "^30.1.3", "mermaid": "^11.12.0", "playwright": "^1.56.0", From 9911f9019565342be8a934a2dc0f484cc703f073 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 22 Oct 2025 16:06:10 -0400 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=A4=96=20Update=20project=20ID=20and?= =?UTF-8?q?=20remove=20setup=20guide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated with `cmux` --- CHROMATIC_SETUP.md | 172 ------------------------------------------ chromatic.config.json | 2 +- 2 files changed, 1 insertion(+), 173 deletions(-) delete mode 100644 CHROMATIC_SETUP.md diff --git a/CHROMATIC_SETUP.md b/CHROMATIC_SETUP.md deleted file mode 100644 index 7750297d9..000000000 --- a/CHROMATIC_SETUP.md +++ /dev/null @@ -1,172 +0,0 @@ -# 🎨 Chromatic Setup Guide - -This guide explains how to set up and use Chromatic for visual regression testing in cmux. - -## What is Chromatic? - -Chromatic is a visual testing platform that: -- Captures screenshots of your Storybook components -- Detects visual changes between commits -- Provides a UI review workflow for accepting/rejecting changes -- Integrates with GitHub PRs to show visual diffs - -## Initial Setup - -### 1. Create a Chromatic Account - -1. Go to [chromatic.com](https://www.chromatic.com/) -2. Sign in with your GitHub account -3. Click "Add project" and select the `coder/cmux` repository - -### 2. Get Your Project Token - -After creating the project, you'll receive a project token that looks like: -``` -chpt_1234567890abcdef -``` - -### 3. Configure GitHub Secrets - -Add the Chromatic project token to your GitHub repository: - -1. Go to repository **Settings** → **Secrets and variables** → **Actions** -2. Click **New repository secret** -3. Name: `CHROMATIC_PROJECT_TOKEN` -4. Value: Your Chromatic project token from step 2 -5. Click **Add secret** - -### 4. Update Configuration File - -Edit `chromatic.config.json` and replace the placeholder with your actual project ID: - -```json -{ - "projectId": "your-actual-project-id-here" -} -``` - -You can find your project ID in the Chromatic dashboard URL or in the setup instructions. - -## Usage - -### Running Chromatic Locally - -Test your Storybook components with Chromatic before pushing: - -```bash -# Set your project token (one-time setup) -export CHROMATIC_PROJECT_TOKEN=chpt_your_token_here - -# Run Chromatic -make chromatic -# or -bun run chromatic -``` - -### Automated PR Checks - -Chromatic automatically runs on every pull request via GitHub Actions. The workflow: - -1. Builds your Storybook -2. Captures screenshots of all stories -3. Compares them against the baseline (main branch) -4. Posts a status check and comment on your PR with: - - Number of visual changes detected - - Link to review changes in Chromatic UI - - Direct comparison images - -### Reviewing Changes - -When Chromatic detects visual changes: - -1. Click the **"View changes in Chromatic"** link in the PR -2. Review each component change in the Chromatic UI -3. Accept or reject changes: - - ✅ **Accept**: Updates the baseline for future comparisons - - ❌ **Reject**: Keeps the existing baseline (treat as a regression) - -### Workflow Integration - -The Chromatic check is **informational** (won't block PRs) thanks to `exitZeroOnChanges: true`. This allows you to: -- Review visual changes at your own pace -- Merge PRs even with pending visual reviews -- Accept/reject changes after merging if needed - -To make it a **required check** (block PRs with unreviewed changes): -1. Update `.github/workflows/chromatic.yml`: Remove `exitZeroOnChanges: true` -2. Add Chromatic as a required status check in GitHub branch protection - -## Best Practices - -### Component Development - -1. **Create stories for visual states**: Add Storybook stories for each significant visual state -2. **Keep stories focused**: Each story should demonstrate one specific state or variant -3. **Use controls**: Make stories interactive with Storybook controls for easier testing - -### Managing Changes - -1. **Accept intentional changes**: When you update styling, accept the Chromatic changes -2. **Investigate unexpected changes**: If Chromatic catches changes you didn't make, investigate before accepting -3. **Batch reviews**: Review multiple small changes together rather than one at a time - -### Performance - -- Chromatic uses **smart diffing** - only changed stories are captured -- The workflow uses `onlyChanged: true` to speed up builds -- First run takes longer (captures all baselines) -- Subsequent runs are much faster - -## Troubleshooting - -### "Project token not found" Error - -Make sure you've added `CHROMATIC_PROJECT_TOKEN` to GitHub Secrets and it matches your Chromatic project token exactly. - -### Workflow Fails to Build Storybook - -Check that: -1. `make storybook-build` works locally -2. All dependencies are installed -3. No TypeScript errors in story files - -### Visual Differences in Fonts/Colors - -This can happen due to OS-level rendering differences. Chromatic uses consistent rendering, but local previews might look different. Trust the Chromatic UI for accurate comparisons. - -### Too Many Changes Detected - -If you've refactored significantly: -1. Review and accept legitimate changes in batches -2. Consider using Chromatic's "Accept all" feature for major style updates -3. Use Chromatic's branch comparison to see changes against your feature branch - -## Configuration Reference - -### chromatic.config.json - -```json -{ - "projectId": "your-project-id", - "buildScriptName": "storybook:build", // npm script to build Storybook - "storybookBuildDir": "storybook-static", // Where Storybook builds to - "exitZeroOnChanges": true, // Don't fail on visual changes - "exitOnceUploaded": true, // Exit after upload completes - "autoAcceptChanges": false // Don't auto-accept changes -} -``` - -### Workflow Options - -In `.github/workflows/chromatic.yml`: - -- `onlyChanged: true` - Only test changed stories (faster) -- `exitZeroOnChanges: true` - Don't block PRs on visual changes -- `buildScriptName` - The npm/make script that builds Storybook - -## Resources - -- [Chromatic Documentation](https://www.chromatic.com/docs/) -- [Storybook Documentation](https://storybook.js.org/docs) -- [Visual Testing Best Practices](https://www.chromatic.com/docs/test) - diff --git a/chromatic.config.json b/chromatic.config.json index d1342d59f..89522bddf 100644 --- a/chromatic.config.json +++ b/chromatic.config.json @@ -1,5 +1,5 @@ { - "projectId": "REPLACE_WITH_YOUR_PROJECT_ID", + "projectId": "68e30fca49979473fc9abc73", "buildScriptName": "storybook:build", "storybookBuildDir": "storybook-static", "exitZeroOnChanges": true, From b77c8c0386e8ff5b052ed77ed45ee764b12c7aae Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 22 Oct 2025 16:08:26 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=A4=96=20Fix=20Chromatic=20workflow?= =?UTF-8?q?=20-=20build=20Storybook=20first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated with `cmux` --- .github/workflows/chromatic.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.yml index 137738bc1..71ab48b5c 100644 --- a/.github/workflows/chromatic.yml +++ b/.github/workflows/chromatic.yml @@ -22,11 +22,13 @@ jobs: - name: Generate version file run: ./scripts/generate-version.sh + - name: Build Storybook + run: make storybook-build + - name: Run Chromatic uses: chromaui/action@latest with: projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} exitZeroOnChanges: true onlyChanged: true - buildScriptName: storybook:build From c6314d645acd5f2301d0823c95fa0612b97f12d7 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 22 Oct 2025 18:26:12 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A4=96=20Fix=20Chromatic=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Skip workflow for forked PRs (no access to secrets) - Build Storybook with --stats-json for TurboSnap optimization Generated with `cmux` --- .github/workflows/chromatic.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.yml index 71ab48b5c..f925e615d 100644 --- a/.github/workflows/chromatic.yml +++ b/.github/workflows/chromatic.yml @@ -11,6 +11,8 @@ jobs: chromatic: name: Visual Regression Testing runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }} + # Skip for forked PRs since they don't have access to secrets + if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' steps: - name: Checkout code uses: actions/checkout@v4 @@ -23,7 +25,7 @@ jobs: run: ./scripts/generate-version.sh - name: Build Storybook - run: make storybook-build + run: bun x storybook build --stats-json - name: Run Chromatic uses: chromaui/action@latest