Skip to content

Commit 83d3952

Browse files
karangattukarangattu
authored
feat(cli): Add AI support to shiny add test (#2041)
1 parent c78c8f1 commit 83d3952

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+5182
-118
lines changed

‎.github/workflows/conventional-commits.yaml‎ renamed to ‎.github/workflows/verify-conventional-commits.yaml‎

File renamed without changes.

‎.github/workflows/verify-test-generation-prompts.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: Verify test generation prompts
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- ".github/workflows/verify-test-generation-prompts.yml"
7+
- "shiny/pytest/_generate/**"
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: "prompt-test-generation-${{ github.event.pull_request.number || 'dispatch' }}"
12+
cancel-in-progress: true
13+
14+
env:
15+
PYTHON_VERSION: "3.13"
16+
ATTEMPTS: 3
17+
PYTHONUNBUFFERED: 1
18+
19+
jobs:
20+
verify-test-generation-prompts:
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 30
23+
24+
steps:
25+
- name: Checkout repository
26+
uses: actions/checkout@v4
27+
with:
28+
fetch-depth: 0
29+
30+
- name: Set up Python
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: ${{ env.PYTHON_VERSION }}
34+
35+
- name: Setup py-shiny
36+
id: install
37+
uses: ./.github/py-shiny/setup
38+
39+
- name: Install Test Generator Dependencies
40+
run: |
41+
make ci-install-ai-deps
42+
43+
- name: Run Evaluation and Tests 3 Times
44+
env:
45+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
46+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
47+
PYTHONUNBUFFERED: 1
48+
timeout-minutes: 25
49+
run: |
50+
make run-test-ai-evaluation
51+
52+
- name: Upload test results
53+
if: always()
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: test-results-${{ github.run_id }}
57+
path: |
58+
test-results-inspect-ai/
59+
retention-days: 7
60+
61+
- name: Process Results
62+
timeout-minutes: 2
63+
run: |
64+
# Results are already averaged by the bash script, just verify they exist
65+
if [ ! -f "test-results-inspect-ai/summary.json" ]; then
66+
echo "No averaged summary found at test-results-inspect-ai/summary.json"
67+
ls -la test-results-inspect-ai/
68+
exit 1
69+
else
70+
echo "Using averaged results from all attempts"
71+
cat test-results-inspect-ai/summary.json
72+
fi
73+
74+
- name: Check Quality Gate
75+
timeout-minutes: 2
76+
run: |
77+
if [ ! -f "test-results-inspect-ai/summary.json" ]; then
78+
echo "Summary file not found at test-results-inspect-ai/summary.json"
79+
ls -la test-results-inspect-ai/
80+
exit 1
81+
else
82+
echo "Found summary file, checking quality gate..."
83+
python tests/inspect-ai/utils/scripts/quality_gate.py test-results-inspect-ai/
84+
fi
85+
86+
- name: Prepare Comment Body
87+
if: github.event_name == 'pull_request'
88+
timeout-minutes: 1
89+
run: |
90+
python tests/inspect-ai/scripts/prepare_comment.py test-results-inspect-ai/summary.json
91+
92+
- name: Comment PR Results
93+
if: github.event_name == 'pull_request'
94+
uses: marocchino/sticky-pull-request-comment@v2
95+
with:
96+
header: inspect-ai-results
97+
path: comment_body.txt

‎.github/workflows/verify-testing-docs-on-change.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
name: Verify testing documentation for changes
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- ".github/workflows/verify-testing-docs-on-change.yml"
7+
- "docs/_quartodoc-testing.yml"
8+
- "shiny/playwright/controller/**"
9+
10+
permissions:
11+
contents: write
12+
pull-requests: write
13+
14+
jobs:
15+
verify-testing-docs:
16+
runs-on: ubuntu-latest
17+
if: github.event_name == 'pull_request'
18+
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v4
22+
with:
23+
fetch-depth: 0
24+
25+
- name: Setup py-shiny
26+
id: install
27+
uses: ./.github/py-shiny/setup
28+
29+
- name: Install dependencies
30+
run: |
31+
make ci-install-docs
32+
33+
- name: Update testing docs and check for changes
34+
id: check-docs-changes
35+
run: |
36+
# Store the current state of the documentation file
37+
cp shiny/pytest/_generate/_data/testing-documentation.json testing-documentation-before.json
38+
39+
# Run the make command to update testing docs
40+
make update-testing-docs
41+
42+
if [[ ! -f testing-documentation-before.json || ! -f shiny/pytest/_generate/_data/testing-documentation.json ]]; then
43+
echo "One or both documentation files are missing."
44+
exit 1
45+
fi
46+
47+
# Check if the documentation file has changed
48+
if diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json > /dev/null 2>&1; then
49+
echo "docs_changed=true" >> $GITHUB_OUTPUT
50+
echo "The generated documentation is out of sync with the current controller changes."
51+
echo "\n\n"
52+
diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json || true
53+
echo "\n\n"
54+
else
55+
echo "docs_changed=false" >> $GITHUB_OUTPUT
56+
echo "Documentation file is up to date"
57+
fi
58+
59+
- name: Comment on PR about testing docs update
60+
if: steps.check-docs-changes.outputs.docs_changed == 'true'
61+
uses: marocchino/sticky-pull-request-comment@v2
62+
with:
63+
header: testing-docs-update
64+
message: |
65+
🚨 **Testing Documentation Out of Sync**
66+
67+
We detected changes in the `shiny/playwright/controller` directory that affect the testing documentation used by the `shiny add test` command.
68+
69+
**The generated documentation is out of sync with your controller changes. Please run:**
70+
71+
```bash
72+
make update-testing-docs
73+
```
74+
75+
**Then commit the updated `shiny/pytest/_generate/_data/testing-documentation.json` file.**
76+
77+
<details><summary>Additional details</summary>
78+
79+
The updated documentation file ensures that the AI test generator has access to the latest controller API documentation.
80+
81+
</details>
82+
83+
❌ **This check will fail until the documentation is updated and committed.**
84+
85+
---
86+
*This comment was automatically generated by the `verify-testing-docs-on-change.yml` workflow.*
87+
88+
- name: Remove comment when no controller changes or docs are up to date
89+
if: steps.check-docs-changes.outputs.docs_changed == 'false'
90+
uses: marocchino/sticky-pull-request-comment@v2
91+
with:
92+
header: testing-docs-update
93+
delete: true

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,10 @@ shiny_bookmarks/
123123

124124
# setuptools_scm
125125
shiny/_version.py
126+
127+
# Other
128+
tests/inspect-ai/apps/*/test_*.py
129+
test-results.xml
130+
results-inspect-ai/
131+
test-results-inspect-ai/
132+
tests/inspect-ai/scripts/test_metadata.json

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1313

1414
### New features
1515

16+
* Added AI-powered test generator for Shiny applications. Use `shiny add test` to automatically generate comprehensive Playwright tests for your apps using AI models from Anthropic or OpenAI. (#2041)
17+
1618
* `navset_card_*()` now has a `full_screen` option to support `card()`'s existing full-screen functionality. (#1451)
1719

1820
* Added `ui.insert_nav_panel()`, `ui.remove_nav_panel()`, and `ui.update_nav_panel()` to support dynamic navigation. (#90)

‎Makefile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,35 @@ docs-quartodoc: FORCE
123123
@echo "-------- Making quartodoc docs --------"
124124
@cd docs && make quartodoc
125125

126+
install-repomix: install-npm FORCE ## Install repomix if not already installed
127+
@echo "-------- Installing repomix if needed --------"
128+
@if ! command -v repomix > /dev/null 2>&1; then \
129+
echo "Installing repomix..."; \
130+
npm install -g repomix; \
131+
else \
132+
echo "repomix is already installed"; \
133+
fi
134+
135+
update-testing-docs-repomix: install-repomix FORCE ## Generate repomix output for testing docs
136+
@echo "-------- Generating repomix output for testing docs --------"
137+
repomix docs/api/testing -o tests/inspect-ai/utils/scripts/repomix-output-testing.xml
138+
139+
update-testing-docs-process: FORCE ## Process repomix output to generate testing documentation JSON
140+
@echo "-------- Processing testing documentation --------"
141+
python tests/inspect-ai/utils/scripts/process_docs.py --input tests/inspect-ai/utils/scripts/repomix-output-testing.xml --output shiny/pytest/_generate/_data/testing-documentation.json
142+
@echo "-------- Cleaning up temporary files --------"
143+
rm -f tests/inspect-ai/utils/scripts/repomix-output-testing.xml
144+
145+
update-testing-docs: docs update-testing-docs-repomix update-testing-docs-process FORCE ## Update testing documentation (full pipeline)
146+
@echo "-------- Testing documentation update complete --------"
147+
148+
ci-install-ai-deps: FORCE
149+
uv pip install -e ".[dev,test,testgen]"
150+
$(MAKE) install-playwright
151+
152+
run-test-ai-evaluation: FORCE ## Run the AI evaluation script for tests
153+
@echo "-------- Running AI evaluation for tests --------"
154+
bash ./tests/inspect-ai/scripts/run-test-evaluation.sh
126155

127156
install-npm: FORCE
128157
$(if $(shell which npm), @echo -n, $(error Please install node.js and npm first. See https://nodejs.org/en/download/ for instructions.))

‎pyproject.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,13 @@ doc = [
124124
"quartodoc>=0.8.1",
125125
"griffe>=1.3.2",
126126
]
127+
testgen = [
128+
"chatlas[anthropic,openai]",
129+
"openai>=1.104.1",
130+
"anthropic>=0.62.0",
131+
"inspect-ai>=0.3.129",
132+
"pytest-timeout",
133+
]
127134

128135

129136
[project.urls]

‎pyrightconfig.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010
"docs",
1111
"tests/playwright/deploys/*/app.py",
1212
"shiny/templates",
13-
"tests/playwright/ai_generated_apps",
13+
"tests/playwright/ai_generated_apps/*/*/app*.py",
14+
"tests/inspect-ai/apps/*/app*.py",
15+
"shiny/pytest/_generate/_main.py",
16+
"tests/inspect-ai/scripts/evaluation.py"
1417
],
1518
"typeCheckingMode": "strict",
1619
"reportImportCycles": "none",

‎shiny/_main.py‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -533,11 +533,10 @@ def add() -> None:
533533
@add.command(
534534
help="""Add a test file for a specified Shiny app.
535535
536-
Add an empty test file for a specified app. You will be prompted with a destination
537-
folder. If you don't provide a destination folder, it will be added in the current
538-
working directory based on the app name.
536+
Generate a comprehensive test file for a specified app using AI. The generator
537+
will analyze your app code and create appropriate test cases with assertions.
539538
540-
After creating the shiny app file, you can use `pytest` to run the tests:
539+
After creating the test file, you can use `pytest` to run the tests:
541540
542541
pytest TEST_FILE
543542
"""
@@ -546,22 +545,37 @@ def add() -> None:
546545
"--app",
547546
"-a",
548547
type=str,
549-
help="Please provide the path to the app file for which you want to create a test file.",
548+
help="Path to the app file for which you want to generate a test file.",
550549
)
551550
@click.option(
552551
"--test-file",
553552
"-t",
554553
type=str,
555-
help="Please provide the name of the test file you want to create. The basename of the test file should start with `test_` and be unique across all test files.",
554+
help="Path for the generated test file. If not provided, will be auto-generated.",
555+
)
556+
@click.option(
557+
"--provider",
558+
type=click.Choice(["anthropic", "openai"]),
559+
default="anthropic",
560+
help="AI provider to use for test generation.",
561+
)
562+
@click.option(
563+
"--model",
564+
type=str,
565+
help="Specific model to use (optional). Examples: haiku3.5, sonnet, gpt-5, gpt-5-mini",
556566
)
557567
# Param for app.py, param for test_name
558568
def test(
559-
app: Path | None,
560-
test_file: Path | None,
569+
app: str | None,
570+
test_file: str | None,
571+
provider: str,
572+
model: str | None,
561573
) -> None:
562-
from ._main_add_test import add_test_file
574+
from ._main_generate_test import generate_test_file
563575

564-
add_test_file(app_file=app, test_file=test_file)
576+
generate_test_file(
577+
app_file=app, output_file=test_file, provider=provider, model=model
578+
)
565579

566580

567581
@main.command(

0 commit comments

Comments
 (0)