test: Add make playwright-debug to help debug playwright tests (#1097)

Makefile

@@ -104,6 +104,9 @@ playwright-deploys: install-playwright install-rsconnect ## end-to-end tests on
 playwright-examples: install-playwright ## end-to-end tests on examples with playwright
 	pytest tests/playwright/examples/$(SUB_FILE)
 
+playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed
+	pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE)
+
 testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report
 	pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml
 

pytest.ini

@@ -4,6 +4,3 @@ testpaths=tests/pytest/
 ; ; Debug version of options
 ; addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vv
 addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto
-markers =
-    examples: Suite of tests to validate that examples do not produce errors (deselect with '-m "not examples"')
-    integrationtest: Suite of tests that check deploys and integration with other services (deselect with '-m "not integrationtest"')

tests/playwright/README.md

@@ -2,7 +2,7 @@
 
 This directory contains end-to-end (i.e. browser based) tests for Shiny for Python.
 
-The Python files directly in this subdirectory are for Pytest fixtures and helper code
+The Python files directly in this subdirectory (and `./utils`) are for Pytest fixtures and helper code
 to make test writing easier. (Eventually this logic may move to the `shiny` package
 itself or its own dedicated package, so that Shiny app authors can set up their own e2e
 tests against their apps.)
@@ -17,14 +17,21 @@ optional, because the tests may also be for apps in the `../examples` or `../shi
 The following commands can be run from the repo root:
 
 ```sh
-# Run all e2e tests
+# Run tests related to shiny (in `./shiny` folder)
 make playwright-shiny
+# Run tests related to examples (in `./examples` folder)
+make playwright-examples
+# Run tests on apps that should be deployed (in `./deploys` folder)
+make playwright-deploys
 
 # Run just the tests in playwright/shiny/async/
 make playwright-shiny SUB_FILE=e2e/async
 
 # Run just the tests in playwright/shiny/async/, in headed mode
 make playwright-shiny SUB_FILE="--headed e2e/async"
+
+# Run **all** tests on with pytest options best for debugging
+make playwright-debug
 ```
 
 ## Shiny app fixtures
@@ -73,3 +80,5 @@ def test_airmass(page: Page, airmass_app: ShinyAppProc):
     plot = page.locator("#plot")
     expect(plot).to_have_class(re.compile(r"\bshiny-bound-output\b"))
 ```
+
+However, it is suggested to alter the example app for testing and use a local app for testing.

tests/playwright/examples/example_apps.py

@@ -12,6 +12,7 @@
 
 is_interactive = hasattr(sys, "ps1")
 reruns = 1 if is_interactive else 3
+reruns_delay = 0
 
 
 def get_apps(path: str) -> typing.List[str]:
@@ -71,8 +72,6 @@ def get_apps(path: str) -> typing.List[str]:
     # TODO-garrick-future: Remove after fixing sidebar max_height_mobile warning
     "UserWarning: The `shiny.ui.sidebar(max_height_mobile=)`",
     "res = self.fn(*self.args, **self.kwargs)",
-    # if shiny express app detected
-    "Detected Shiny Express app",
     # pandas >= 2.2.0
     # https://github.com/pandas-dev/pandas/blame/5740667a55aabffc660936079268cee2f2800225/pandas/core/groupby/groupby.py#L1129
     "FutureWarning: When grouping with a length-1 list-like",
@@ -161,6 +160,9 @@ def validate_example(page: Page, ex_app_path: str) -> None:
 
     def on_console_msg(msg: ConsoleMessage) -> None:
         if msg.type == "error":
+            # Do not report missing favicon errors
+            if msg.location["url"].endswith("favicon.ico"):
+                return
             console_errors.append(msg.text)
 
     page.on("console", on_console_msg)

tests/playwright/examples/test_api_examples.py

@@ -1,10 +1,9 @@
 import pytest
-from example_apps import get_apps, reruns, validate_example
+from example_apps import get_apps, reruns, reruns_delay, validate_example
 from playwright.sync_api import Page
 
 
-@pytest.mark.examples
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("ex_app_path", get_apps("shiny/api-examples"))
 def test_api_examples(page: Page, ex_app_path: str) -> None:
     validate_example(page, ex_app_path)

tests/playwright/examples/test_examples.py

@@ -1,9 +1,9 @@
 import pytest
-from example_apps import get_apps, reruns, validate_example
+from example_apps import get_apps, reruns, reruns_delay, validate_example
 from playwright.sync_api import Page
 
 
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("ex_app_path", get_apps("examples"))
 def test_examples(page: Page, ex_app_path: str) -> None:
     validate_example(page, ex_app_path)

tests/playwright/examples/test_shiny_create.py

@@ -3,7 +3,7 @@
 import tempfile
 
 import pytest
-from example_apps import get_apps, reruns, validate_example
+from example_apps import get_apps, reruns, reruns_delay, validate_example
 from playwright.sync_api import Page
 
 
@@ -41,33 +41,29 @@ def subprocess_create(
         subprocess.run(cmd, check=True)
 
 
-@pytest.mark.examples
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("ex_app_path", get_apps("shiny/templates/app-templates"))
 def test_template_examples(page: Page, ex_app_path: str) -> None:
     validate_example(page, ex_app_path)
 
 
-@pytest.mark.examples
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("app_template", ["basic-app", "dashboard", "multi-page"])
 def test_create_core(app_template: str, page: Page):
     with tempfile.TemporaryDirectory("example_apps") as tmpdir:
         subprocess_create(app_template, dest_dir=tmpdir)
         validate_example(page, f"{tmpdir}/app.py")
 
 
-@pytest.mark.examples
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("app_template", ["basic-app"])
 def test_create_express(app_template: str, page: Page):
     with tempfile.TemporaryDirectory("example_apps") as tmpdir:
         subprocess_create(app_template, dest_dir=tmpdir, mode="express")
         validate_example(page, f"{tmpdir}/app.py")
 
 
-@pytest.mark.examples
-@pytest.mark.flaky(reruns=reruns, reruns_delay=1)
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.parametrize("app_template", ["js-input", "js-output", "js-react"])
 def test_create_js(app_template: str):
     with tempfile.TemporaryDirectory("example_apps") as tmpdir:

tests/playwright/playwright-pytest.ini

@@ -0,0 +1,13 @@
+[pytest]
+asyncio_mode=strict
+# --strict-markers: Markers not registered in the `markers` section of the configuration file raise errors
+# --durations <k>: show top `k` slowest durations
+# --durations-min <n>: Require that the top `k` slowest durations are longer than `n` seconds
+# --browser <name>: browser type to run on playwright
+# --numprocesses auto: number of testing workers. auto is number of (virtual) cores
+# --video=retain-on-failure: playwright saves recording of any failed test
+# -vv: Extra extra verbose output
+# # --headed: Headed browser testing
+# # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings...
+# --maxfail=1: Stop after 1 failure has occurred
+addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vvv --maxfail=1 --headed

tests/playwright/shiny/components/data_frame/test_data_frame.py

@@ -7,10 +7,9 @@
 import pytest
 from conftest import ShinyAppProc, create_example_fixture, expect_to_change
 from controls import InputSelect, InputSwitch
+from examples.example_apps import reruns, reruns_delay
 from playwright.sync_api import Locator, Page, expect
 
-RERUNS = 3
-
 data_frame_app = create_example_fixture("dataframe")
 
 
@@ -47,7 +46,7 @@ def do():
     return do
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_grid_mode(
     page: Page, data_frame_app: ShinyAppProc, grid: Locator, grid_container: Locator
 ):
@@ -61,7 +60,7 @@ def test_grid_mode(
     expect(grid_container).to_have_class(re.compile(r"\bshiny-data-grid-grid\b"))
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_summary_navigation(
     page: Page,
     data_frame_app: ShinyAppProc,
@@ -81,7 +80,7 @@ def test_summary_navigation(
     expect(summary).to_have_text(re.compile("^Viewing rows \\d+ through 20 of 20$"))
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_full_width(page: Page, data_frame_app: ShinyAppProc, grid_container: Locator):
     page.goto(data_frame_app.url)
 
@@ -101,7 +100,7 @@ def get_width() -> float:
     InputSwitch(page, "fullwidth").toggle()
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_table_switch(
     page: Page,
     data_frame_app: ShinyAppProc,
@@ -134,7 +133,7 @@ def test_table_switch(
     )
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_sort(
     page: Page,
     data_frame_app: ShinyAppProc,
@@ -171,7 +170,7 @@ def test_sort(
     expect(first_cell_depth).to_have_text("67.6")
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_multi_selection(
     page: Page, data_frame_app: ShinyAppProc, grid_container: Locator, snapshot: Any
 ):
@@ -201,7 +200,7 @@ def detail_text():
     assert detail_text() == snapshot
 
 
-@pytest.mark.flaky(reruns=RERUNS)
+@pytest.mark.flaky(reruns=reruns, delay=reruns_delay)
 def test_single_selection(
     page: Page, data_frame_app: ShinyAppProc, grid_container: Locator, snapshot: Any
 ):

tests/pytest/test_utils.py

@@ -139,7 +139,7 @@ async def mutate_registrations():
 
 # Timeout within 2 seconds
 @pytest.mark.timeout(2)
-@pytest.mark.flaky(reruns=3, reruns_delay=1)
+@pytest.mark.flaky(reruns=3)
 def test_random_port():
     assert random_port(9000, 9000) == 9000