test(CI): Add merge queue support for faster PR testing (#1265)

posit-dev · Mar 28, 2024 · 6b1dcfb · 6b1dcfb
1 parent acd611d
commit 6b1dcfb
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 30 deletions.
diff --git a/.github/py-shiny/pytest-browsers/action.yaml b/.github/py-shiny/pytest-browsers/action.yaml
@@ -0,0 +1,30 @@
+name: 'Custom merge queue browsers'
+description: 'Trim down pytest browsers for any github event other than merge_group.'
+inputs:
+  all-browsers:
+    description: 'Force all pytest browsers to used when testing'
+    required: false
+    default: 'false'
+outputs:
+  browsers:
+    description: 'pytest browsers to use'
+    value: ${{ steps.browsers.outputs.browsers }}
+runs:
+  using: "composite"
+  steps:
+      - name: Determine browsers to use
+        shell: bash
+        id: browsers
+        run: |
+          if [ "${{ inputs.all-browsers }}" == "true" ]; then
+            echo "Using all browsers!"
+            exit 0
+          fi
+
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "Using chrome browser only!"
+            echo 'browsers=PYTEST_BROWSERS="--browser chromium"' >> "$GITHUB_OUTPUT"
+          fi
+
+          echo "No custom pytest browsers!"
+          exit 0
diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml
@@ -5,17 +5,18 @@ on:
   push:
     branches: ["main"]
   pull_request:
+  merge_group:
 
 jobs:
-  build:
+  build-docs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.12"]
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -42,6 +43,7 @@ jobs:
           make quartodoc
 
       - name: Build site
+        if: ${{ github.event_name != 'pull_request' || startsWith(github.head_ref, 'docs')  }}
         run: |
           cd docs
           make site
@@ -52,15 +54,14 @@ jobs:
         with:
           path: "docs/_site"
 
-
   deploy:
     if: github.ref == 'refs/heads/main'
-    needs: build
+    needs: build-docs
 
     # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
     permissions:
-      pages: write      # to deploy to Pages
-      id-token: write   # to verify the deployment originates from an appropriate source
+      pages: write # to deploy to Pages
+      id-token: write # to verify the deployment originates from an appropriate source
 
     # Deploy to the github-pages environment
     environment:

diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -5,13 +5,14 @@ on:
   push:
     branches: ["main", "rc-*"]
   pull_request:
+  merge_group:
   release:
     types: [published]
   schedule:
     - cron: "0 8 * * *"
 
 jobs:
-  build:
+  check:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -21,7 +22,7 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         id: install
         uses: ./.github/py-shiny/setup
@@ -58,16 +59,24 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         uses: ./.github/py-shiny/setup
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Determine browsers for testing
+        uses: ./.github/py-shiny/pytest-browsers
+        id: browsers
+        with:
+          all-browsers: ${{ startsWith(github.head_ref, 'playwright') }}
+      - name: Display browser
+        shell: bash
+        run: echo '${{ steps.browsers.outputs.browsers }}'
       - name: Run End-to-End tests
         timeout-minutes: 20
         run: |
-          make playwright-shiny SUB_FILE=". -vv"
+          make playwright-shiny SUB_FILE=". -vv" ${{ steps.browsers.outputs.browsers }}
       - uses: actions/upload-artifact@v4
         if: failure()
         with:
@@ -85,14 +94,14 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         uses: ./.github/py-shiny/setup
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: "18"
           cache: npm
@@ -102,10 +111,15 @@ jobs:
         run: |
           npm ci
 
+      - name: Determine browsers for testing
+        uses: ./.github/py-shiny/pytest-browsers
+        id: browsers
+        with:
+          all-browsers: ${{ startsWith(github.head_ref, 'playwright') }}
       - name: Run example app tests
         timeout-minutes: 20
         run: |
-          make playwright-examples SUB_FILE=". -vv"
+          make playwright-examples SUB_FILE=". -vv" ${{ steps.browsers.outputs.browsers }}
       - uses: actions/upload-artifact@v4
         if: failure()
         with:
@@ -124,7 +138,7 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         uses: ./.github/py-shiny/setup
         with:
@@ -146,7 +160,7 @@ jobs:
 
   playwright-deploys:
     needs: [playwright-deploys-precheck]
-    if: github.event_name != 'release' && (github.event_name == 'push' || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'deploy')))
+    if: github.event_name != 'release' && (github.event_name == 'push' || startsWith(github.head_ref, 'deploy'))
     # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR)
     # Only one is allowed to run at a time because it is deploying to the same server location.
     concurrency: playwright-deploys
@@ -159,7 +173,7 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         uses: ./.github/py-shiny/setup
         with:
@@ -172,7 +186,7 @@ jobs:
         run: |
           make playwright-deploys SUB_FILE=". -vv"
 
-      - name: Deploy apps and run tests (on `push` or on `pull_request` w/ `deploy**` branch)
+      - name: Deploy apps and run tests (on `push` or `deploy**` branches)
         env:
           DEPLOY_APPS: "true"
           DEPLOY_CONNECT_SERVER_URL: "https://rsc.radixu.com/"
@@ -197,9 +211,9 @@ jobs:
     name: "Deploy to PyPI"
     runs-on: ubuntu-latest
     if: github.event_name == 'release'
-    needs: [build]
+    needs: [check]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: "Set up Python 3.10"
         uses: actions/setup-python@v4
         with:
@@ -233,7 +247,7 @@ jobs:
 
   testrail-reporting-nightly:
     runs-on: ${{ matrix.os }}
-    if: ${{ github.event_name == 'schedule' || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'testrail')) }}
+    if: ${{ github.event_name == 'schedule' || startsWith(github.head_ref, 'testrail') }}
     strategy:
       matrix:
         python-version:
@@ -246,14 +260,14 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup py-shiny
         uses: ./.github/py-shiny/setup
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: "18"
           cache: npm

diff --git a/.github/workflows/verify-js-built.yaml b/.github/workflows/verify-js-built.yaml
@@ -1,9 +1,10 @@
-name: Build
+name: Verify built assets
 
 on:
   push:
     branches: ["main", "rc-*"]
   pull_request:
+  merge_group:
 
 jobs:
   verify_js_built:

diff --git a/Makefile b/Makefile
@@ -144,7 +144,7 @@ clean-js: FORCE
 
 # Default `SUB_FILE` to empty
 SUB_FILE:=
-
+PYTEST_BROWSERS:= --browser webkit --browser firefox --browser chromium
 install-playwright: FORCE
 	playwright install --with-deps
 
@@ -157,15 +157,15 @@ install-rsconnect: FORCE
 
 # end-to-end tests with playwright; (SUB_FILE="" within tests/playwright/shiny/)
 playwright-shiny: install-playwright
-	pytest tests/playwright/shiny/$(SUB_FILE)
+	pytest tests/playwright/shiny/$(SUB_FILE) $(PYTEST_BROWSERS)
 
 # end-to-end tests on deployed apps with playwright; (SUB_FILE="" within tests/playwright/deploys/)
 playwright-deploys: install-playwright install-rsconnect
-	pytest tests/playwright/deploys/$(SUB_FILE)
+	pytest tests/playwright/deploys/$(SUB_FILE) $(PYTEST_BROWSERS)
 
 # end-to-end tests on all py-shiny examples with playwright; (SUB_FILE="" within tests/playwright/examples/)
 playwright-examples: install-playwright
-	pytest tests/playwright/examples/$(SUB_FILE)
+	pytest tests/playwright/examples/$(SUB_FILE) $(PYTEST_BROWSERS)
 
 playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed; (SUB_FILE="" within tests/playwright/)
 	pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE)
@@ -175,10 +175,10 @@ playwright-show-trace: ## Show trace of failed tests
 
 # end-to-end tests with playwright and generate junit report
 testrail-junit: install-playwright install-trcli
-	pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml
+	pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml $(PYTEST_BROWSERS)
 
 coverage: FORCE ## check combined code coverage (must run e2e last)
-	pytest --cov-report term-missing --cov=shiny tests/pytest/ tests/playwright/shiny/$(SUB_FILE)
+	pytest --cov-report term-missing --cov=shiny tests/pytest/ tests/playwright/shiny/$(SUB_FILE) $(PYTEST_BROWSERS)
 	coverage html
 	$(BROWSER) htmlcov/index.html
 

diff --git a/pytest.ini b/pytest.ini
@@ -1,4 +1,5 @@
 [pytest]
 asyncio_mode=strict
 testpaths=tests/pytest/
-addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto   --tracing=retain-on-failure --video=retain-on-failure
+; Note: Browsers are set within `./Makefile`
+addopts = --strict-markers --durations=6 --durations-min=5.0 --numprocesses auto --tracing=retain-on-failure --video=retain-on-failure